mirror of
https://github.com/ikechan8370/chatgpt-plugin.git
synced 2025-12-16 13:27:08 +00:00
feat(bym): 好像应该没问题支持了多图片处理？
This commit is contained in:
ycxom
parent
e3fe14c6e6
commit
6033d93028
3 changed files with 172 additions and 60 deletions
78
apps/bym.js
78
apps/bym.js
|
|
@ -134,34 +134,50 @@ export class bym extends plugin {
|
|||
let opt = {
|
||||
maxOutputTokens: 500,
|
||||
temperature: 1,
|
||||
replyPureTextCallback: e.reply
|
||||
replyPureTextCallback: e.reply,
|
||||
images: []
|
||||
}
|
||||
let imgs = await getImg(e)
|
||||
// 处理图片
|
||||
if (!e.msg) {
|
||||
if (imgs?.length > 0) {
|
||||
let image = imgs[0]
|
||||
const response = await fetch(image)
|
||||
const base64Image = Buffer.from(await response.arrayBuffer())
|
||||
opt.image = base64Image.toString('base64')
|
||||
e.msg = '[图片]'
|
||||
// 并行处理多张图片
|
||||
opt.images = await Promise.all(imgs.map(async image => {
|
||||
try {
|
||||
const response = await fetch(image)
|
||||
const base64Image = Buffer.from(await response.arrayBuffer())
|
||||
return base64Image.toString('base64')
|
||||
} catch (error) {
|
||||
logger.error(`处理图片失败: ${error}`)
|
||||
return null
|
||||
}
|
||||
})).then(results => results.filter(Boolean))
|
||||
|
||||
e.msg = `[${opt.images.length}张图片]`
|
||||
} else {
|
||||
return setTimeout(async () => {
|
||||
e.msg = '我单纯只是at了你，根据群聊内容回应'
|
||||
await bymGo()
|
||||
}, 3000);
|
||||
|
||||
}, 3000)
|
||||
}
|
||||
} else if (imgs?.length > 0 && !opt.images.length) {
|
||||
// 处理有消息且有图片的情况
|
||||
opt.images = await Promise.all(imgs.map(async image => {
|
||||
try {
|
||||
const response = await fetch(image)
|
||||
const base64Image = Buffer.from(await response.arrayBuffer())
|
||||
return base64Image.toString('base64')
|
||||
} catch (error) {
|
||||
logger.error(`处理图片失败: ${error}`)
|
||||
return null
|
||||
}
|
||||
})).then(results => results.filter(Boolean))
|
||||
}
|
||||
if (!opt.image && imgs?.length > 0) {
|
||||
let image = imgs[0]
|
||||
const response = await fetch(image)
|
||||
const base64Image = Buffer.from(await response.arrayBuffer())
|
||||
opt.image = base64Image.toString('base64')
|
||||
}
|
||||
|
||||
logger.info('[bymGo] 开始处理回复')
|
||||
|
||||
let previousRole = ALLRole
|
||||
if (opt.image && !context.isAtBot && !NotToImg && !e.at && Config.AutoToDownImg) {
|
||||
if (opt.images?.length > 0 && !context.isAtBot && !NotToImg && !e.at && Config.AutoToDownImg) {
|
||||
ALLRole = 'downimg'
|
||||
}
|
||||
|
||||
|
|
@ -252,7 +268,7 @@ export class bym extends plugin {
|
|||
|
||||
switch (user_role) {
|
||||
case "downimg":
|
||||
Role = '现在看到的是一张图片，若你觉得是一张表情包，并不是通知，或其他的图片，注意辨别图片文字是否为通知；单纯是表情包，请发送 DOWNIMG: 命名该表情。 不需要发送过多的参数，只需要发送格式DOWNIMG: 命名该表情，注意不需要携带后缀，请以你的角度觉得如果要发这个表情包要用什么名字来命名； 若不是表情包等，及发送NOTIMG';
|
||||
Role = `现在看到的是${opt.images.length}张图片（从第1张到第${opt.images.length}张），请依次查看各张图片。若觉得是表情包，并不是通知或其他类型的图片，请发送 DOWNIMG: 命名该表情。不需要发送过多的参数，只需要发送格式DOWNIMG: 命名该表情，注意不需要携带后缀；若不是表情包等，请发送NOTIMG并对图片内容进行分析描述。注意：请从第1张图片开始依次描述。`;
|
||||
break;
|
||||
case "default":
|
||||
Role = `你的名字是"${Config.assistantLabel}"，你在一个qq群里，群号是${group},当前和你说话的人群名片是${card}, qq号是${sender}, 请你结合用户的发言和聊天记录作出回应，要求表现得随性一点，最好参与讨论，混入其中。不要过分插科打诨，不知道说什么可以复读群友的话。要求你做搜索、发图、发视频和音乐等操作时要使用工具。不可以直接发[图片]这样蒙混过关。要求优先使用中文进行对话。` +
|
||||
|
|
@ -297,9 +313,6 @@ export class bym extends plugin {
|
|||
if (Config.azSerpKey) {
|
||||
tools.push(new SerpTool())
|
||||
}
|
||||
if (Config.azSerpKey) {
|
||||
tools.push(new SerpTool())
|
||||
}
|
||||
if (e.group.is_admin || e.group.is_owner) {
|
||||
tools.push(new EditCardTool())
|
||||
tools.push(new JinyanTool())
|
||||
|
|
@ -321,7 +334,7 @@ export class bym extends plugin {
|
|||
let text = rsp.text
|
||||
let texts = text.split(/(?<!\?)[。？\n](?!\?)/)
|
||||
for (let t of texts) {
|
||||
if (!t) {
|
||||
if (!t || !t.trim()) {
|
||||
continue
|
||||
}
|
||||
t = t.trim()
|
||||
|
|
@ -329,11 +342,32 @@ export class bym extends plugin {
|
|||
t += '？'
|
||||
}
|
||||
const processed = await imageTool.processText(t, {
|
||||
image: opt.image
|
||||
images: opt.images // 传入图片数组而不是单个图片
|
||||
})
|
||||
|
||||
if (!processed) {
|
||||
// 处理工具返回结果
|
||||
if (processed && typeof processed === 'object') {
|
||||
if (processed.switchRole) {
|
||||
ALLRole = processed.switchRole
|
||||
}
|
||||
if (processed.continueProcess) {
|
||||
// 根据是否是重新处理来设置不同的消息
|
||||
if (processed.reprocess) {
|
||||
e.msg = `[重新处理第${processed.currentIndex + 1}张图片的内容]`
|
||||
} else {
|
||||
e.msg = `[处理第${processed.currentIndex + 1}张图片（共${opt.images.length}张）]`
|
||||
}
|
||||
await bymGo(true)
|
||||
return false
|
||||
} else if (processed.needResponse) {
|
||||
await bymGo(true)
|
||||
return false
|
||||
}
|
||||
} else {
|
||||
let finalMsg = await convertFaces(t, true, e)
|
||||
if (!finalMsg || (typeof finalMsg === 'string' && !finalMsg.trim())) {
|
||||
continue
|
||||
}
|
||||
logger.info(JSON.stringify(finalMsg))
|
||||
if (Math.floor(Math.random() * 100) < 10) {
|
||||
await e.reply(finalMsg, true, {
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ export const HarmBlockThreshold = {
|
|||
*/
|
||||
|
||||
export class CustomGoogleGeminiClient extends GoogleGeminiClient {
|
||||
constructor (props) {
|
||||
constructor(props) {
|
||||
super(props)
|
||||
this.model = props.model
|
||||
this.baseUrl = props.baseUrl || BASEURL
|
||||
|
|
@ -99,7 +99,8 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
|
|||
* onProgress: function?,
|
||||
* functionResponse: FunctionResponse?,
|
||||
* system: string?,
|
||||
* image: string?,
|
||||
* image: string?, // 保留旧版单图片支持
|
||||
* images: string[], // 新增多图片支持
|
||||
* maxOutputTokens: number?,
|
||||
* temperature: number?,
|
||||
* topP: number?,
|
||||
|
|
@ -111,7 +112,7 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
|
|||
* }} opt
|
||||
* @returns {Promise<{conversationId: string?, parentMessageId: string, text: string, id: string}>}
|
||||
*/
|
||||
async sendMessage (text, opt = {}) {
|
||||
async sendMessage(text, opt = {}) {
|
||||
let history = await this.getHistory(opt.parentMessageId)
|
||||
let systemMessage = opt.system
|
||||
if (systemMessage) {
|
||||
|
|
@ -138,26 +139,40 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
|
|||
const idModel = crypto.randomUUID()
|
||||
const thisMessage = opt.functionResponse
|
||||
? {
|
||||
role: 'user',
|
||||
parts: [{
|
||||
functionResponse: opt.functionResponse
|
||||
}],
|
||||
id: idThis,
|
||||
parentMessageId: opt.parentMessageId || undefined
|
||||
}
|
||||
role: 'user',
|
||||
parts: [{
|
||||
functionResponse: opt.functionResponse
|
||||
}],
|
||||
id: idThis,
|
||||
parentMessageId: opt.parentMessageId || undefined
|
||||
}
|
||||
: {
|
||||
role: 'user',
|
||||
parts: [{ text }],
|
||||
id: idThis,
|
||||
parentMessageId: opt.parentMessageId || undefined
|
||||
role: 'user',
|
||||
parts: [{ text }],
|
||||
id: idThis,
|
||||
parentMessageId: opt.parentMessageId || undefined
|
||||
}
|
||||
if (opt.image || opt.images) {
|
||||
// 兼容旧版单图片
|
||||
if (opt.image) {
|
||||
thisMessage.parts.push({
|
||||
inline_data: {
|
||||
mime_type: 'image/jpeg',
|
||||
data: opt.image
|
||||
}
|
||||
})
|
||||
}
|
||||
// 处理多图片
|
||||
if (opt.images && Array.isArray(opt.images)) {
|
||||
for (let imageData of opt.images) {
|
||||
thisMessage.parts.push({
|
||||
inline_data: {
|
||||
mime_type: 'image/jpeg',
|
||||
data: imageData
|
||||
}
|
||||
})
|
||||
}
|
||||
if (opt.image) {
|
||||
thisMessage.parts.push({
|
||||
inline_data: {
|
||||
mime_type: 'image/jpeg',
|
||||
data: opt.image
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
history.push(_.cloneDeep(thisMessage))
|
||||
let url = `${this.baseUrl}/v1beta/models/${this.model}:generateContent`
|
||||
|
|
@ -220,7 +235,7 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
|
|||
if (opt.codeExecution) {
|
||||
body.tools.push({ code_execution: {} })
|
||||
}
|
||||
if (opt.image) {
|
||||
if (opt.image || (opt.images && opt.images.length > 0)) {
|
||||
delete body.tools
|
||||
}
|
||||
body.contents.forEach(content => {
|
||||
|
|
@ -365,7 +380,7 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
|
|||
* @param {Content} responseContent
|
||||
* @returns {{final: string, responseContent}}
|
||||
*/
|
||||
function handleSearchResponse (responseContent) {
|
||||
function handleSearchResponse(responseContent) {
|
||||
let final = ''
|
||||
|
||||
// 遍历每个 part 并处理
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ export class ImageProcessTool extends AbstractTool {
|
|||
name = 'imageProcess'
|
||||
#availableImages = []
|
||||
#initialized = false
|
||||
#currentImageIndex = 0
|
||||
#totalImages = 0
|
||||
#needReprocess = false
|
||||
|
||||
parameters = {
|
||||
properties: {
|
||||
|
|
@ -63,7 +66,7 @@ export class ImageProcessTool extends AbstractTool {
|
|||
|
||||
|
||||
|
||||
async initializeImageList() {
|
||||
async initializeImageList() {
|
||||
try {
|
||||
this.#availableImages = await fileImgList()
|
||||
this.#initialized = true
|
||||
|
|
@ -88,6 +91,19 @@ async initializeImageList() {
|
|||
}
|
||||
|
||||
async processText(text, options = {}) {
|
||||
// 初始化或重置图片处理信息
|
||||
if (options.images?.length > 0) {
|
||||
this.#totalImages = options.images.length
|
||||
// 只在新的处理周期时重置索引
|
||||
if (!this.#needReprocess && this.#currentImageIndex >= this.#totalImages) {
|
||||
this.#currentImageIndex = 0
|
||||
}
|
||||
} else if (options.image) {
|
||||
this.#totalImages = 1
|
||||
if (!this.#needReprocess) {
|
||||
this.#currentImageIndex = 0
|
||||
}
|
||||
}
|
||||
|
||||
const commands = {
|
||||
get: {
|
||||
|
|
@ -101,21 +117,54 @@ async initializeImageList() {
|
|||
notImage: {
|
||||
regex: /NOTIMG(.*)/i,
|
||||
handler: async (match) => {
|
||||
await this.handleNotImage(match[1]?.trim())
|
||||
const result = await this.handleNotImage()
|
||||
if (result.success) {
|
||||
// 标记需要重新处理当前图片
|
||||
this.#needReprocess = true
|
||||
return {
|
||||
success: true,
|
||||
continueProcess: true,
|
||||
currentIndex: this.#currentImageIndex, // 保持当前索引不变
|
||||
switchRole: result.switchRole,
|
||||
needResponse: result.needResponse,
|
||||
reprocess: true // 添加重新处理标记
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
},
|
||||
download: {
|
||||
regex: /DOWNIMG:\s*(.+)/i,
|
||||
handler: async (match) => {
|
||||
await this.handleDownloadImage(match[1].trim(), options.image)
|
||||
const baseName = match[1].trim()
|
||||
if (options.images) {
|
||||
const imageName = this.#totalImages === 1
|
||||
? baseName
|
||||
: `${baseName}_${this.#currentImageIndex + 1}`
|
||||
await this.handleDownloadImage(imageName, options.images[this.#currentImageIndex])
|
||||
|
||||
// 处理完当前图片后，重置重新处理标记
|
||||
this.#needReprocess = false
|
||||
|
||||
// 只有在不是重新处理时才增加索引
|
||||
if (this.#currentImageIndex < this.#totalImages - 1) {
|
||||
this.#currentImageIndex++
|
||||
return {
|
||||
success: true,
|
||||
continueProcess: true,
|
||||
currentIndex: this.#currentImageIndex
|
||||
}
|
||||
}
|
||||
} else if (options.image) {
|
||||
await this.handleDownloadImage(baseName, options.image)
|
||||
}
|
||||
return true
|
||||
}
|
||||
},
|
||||
list: {
|
||||
regex: /^(?:表情包列表|查看表情包|列出表情包)$/i,
|
||||
handler: async () => {
|
||||
await this.e.reply(this.getImagesPrompt())
|
||||
await this.e.reply(await this.getImagesPrompt())
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
|
@ -125,14 +174,14 @@ async initializeImageList() {
|
|||
const match = text.match(regex)
|
||||
if (match) {
|
||||
try {
|
||||
return await handler(match)
|
||||
const result = await handler(match)
|
||||
return result
|
||||
} catch (error) {
|
||||
await this.e.reply(`处理失败: ${error.message}`)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
async handleGetImage(imageName) {
|
||||
|
|
@ -157,7 +206,6 @@ async initializeImageList() {
|
|||
if (!imageName || !imageData) {
|
||||
throw new Error('需要提供图片名称和数据')
|
||||
}
|
||||
|
||||
try {
|
||||
const text = `DOWNIMG: ${imageName}`
|
||||
await downImg(this.e, imageData, text)
|
||||
|
|
@ -170,14 +218,22 @@ async initializeImageList() {
|
|||
|
||||
async handleNotImage() {
|
||||
try {
|
||||
this.ALLRole = this.previousRole
|
||||
await this.bymGo(true)
|
||||
return true
|
||||
return {
|
||||
success: true,
|
||||
switchRole: this.previousRole,
|
||||
needResponse: true
|
||||
}
|
||||
} catch (error) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
getCurrentProgress() {
|
||||
return {
|
||||
currentIndex: this.#currentImageIndex,
|
||||
totalImages: this.#totalImages,
|
||||
isProcessing: this.#currentImageIndex < this.#totalImages
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
description = `图片处理工具：支持发送本地表情包、保存新表情包、切换处理模式。
|
||||
|
|
@ -185,20 +241,27 @@ async initializeImageList() {
|
|||
- GETIMG: <表情包名称> - 发送指定表情包
|
||||
- DOWNIMG: <名称> - 保存当前图片为表情包
|
||||
- NOTIMG - 切换到文本模式
|
||||
- 表情包列表 - 查看所有可用表情包`
|
||||
- 表情包列表 - 查看所有可用表情包
|
||||
\n\n多图片处理说明：
|
||||
- 当处理多张图片时，会自动为每张图片添加序号后缀
|
||||
- 例如：DOWNIMG: happy 命令处理多张图片时会保存为 happy_1, happy_2 等\n`
|
||||
// 添加 getSystemPrompt 方法
|
||||
async getSystemPrompt() {
|
||||
const images = await this.getAvailableImages()
|
||||
const progress = this.getCurrentProgress()
|
||||
let prompt = `${this.description}\n`
|
||||
|
||||
|
||||
if (progress.isProcessing) {
|
||||
prompt += `\n当前正在处理第 ${progress.currentIndex + 1}/${progress.totalImages} 张图片\n`
|
||||
}
|
||||
|
||||
if (images.length > 0) {
|
||||
prompt += `\n当前可用的表情包：\n${images.join('\n')}`
|
||||
prompt += `\n使用 GETIMG: <表情包名称> 来发送表情包`
|
||||
} else {
|
||||
logger.warn('[ImageProcessTool] 没有可用的表情包')
|
||||
prompt += '\n当前没有可用的表情包'
|
||||
}
|
||||
|
||||
|
||||
return prompt
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue