From 6033d930280d49d25696ec8c76d61798f1d26bfd Mon Sep 17 00:00:00 2001 From: ycxom Date: Wed, 1 Jan 2025 23:35:50 +0800 Subject: [PATCH] =?UTF-8?q?feat(bym):=20=E5=A5=BD=E5=83=8F=E5=BA=94?= =?UTF-8?q?=E8=AF=A5=E6=B2=A1=E9=97=AE=E9=A2=98=E6=94=AF=E6=8C=81=E4=BA=86?= =?UTF-8?q?=E5=A4=9A=E5=9B=BE=E7=89=87=E5=A4=84=E7=90=86=EF=BC=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/bym.js | 78 ++++++++++++++++++------- client/CustomGoogleGeminiClient.js | 61 ++++++++++++-------- utils/tools/ImageTool.js | 93 +++++++++++++++++++++++++----- 3 files changed, 172 insertions(+), 60 deletions(-) diff --git a/apps/bym.js b/apps/bym.js index ac6cf93..953b6d7 100644 --- a/apps/bym.js +++ b/apps/bym.js @@ -134,34 +134,50 @@ export class bym extends plugin { let opt = { maxOutputTokens: 500, temperature: 1, - replyPureTextCallback: e.reply + replyPureTextCallback: e.reply, + images: [] } let imgs = await getImg(e) + // 处理图片 if (!e.msg) { if (imgs?.length > 0) { - let image = imgs[0] - const response = await fetch(image) - const base64Image = Buffer.from(await response.arrayBuffer()) - opt.image = base64Image.toString('base64') - e.msg = '[图片]' + // 并行处理多张图片 + opt.images = await Promise.all(imgs.map(async image => { + try { + const response = await fetch(image) + const base64Image = Buffer.from(await response.arrayBuffer()) + return base64Image.toString('base64') + } catch (error) { + logger.error(`处理图片失败: ${error}`) + return null + } + })).then(results => results.filter(Boolean)) + + e.msg = `[${opt.images.length}张图片]` } else { return setTimeout(async () => { e.msg = '我单纯只是at了你，根据群聊内容回应' await bymGo() - }, 3000); - + }, 3000) } + } else if (imgs?.length > 0 && !opt.images.length) { + // 处理有消息且有图片的情况 + opt.images = await Promise.all(imgs.map(async image => { + try { + const response = await fetch(image) + const base64Image = Buffer.from(await response.arrayBuffer()) + return base64Image.toString('base64') + } catch (error) { + logger.error(`处理图片失败: ${error}`) + return null + } + })).then(results => results.filter(Boolean)) } - if (!opt.image && imgs?.length > 0) { - let image = imgs[0] - const response = await fetch(image) - const base64Image = Buffer.from(await response.arrayBuffer()) - opt.image = base64Image.toString('base64') - } + logger.info('[bymGo] 开始处理回复') let previousRole = ALLRole - if (opt.image && !context.isAtBot && !NotToImg && !e.at && Config.AutoToDownImg) { + if (opt.images?.length > 0 && !context.isAtBot && !NotToImg && !e.at && Config.AutoToDownImg) { ALLRole = 'downimg' } @@ -252,7 +268,7 @@ export class bym extends plugin { switch (user_role) { case "downimg": - Role = '现在看到的是一张图片，若你觉得是一张表情包，并不是通知，或其他的图片，注意辨别图片文字是否为通知；单纯是表情包，请发送 DOWNIMG: 命名该表情。 不需要发送过多的参数，只需要发送格式DOWNIMG: 命名该表情，注意不需要携带后缀，请以你的角度觉得如果要发这个表情包要用什么名字来命名； 若不是表情包等，及发送NOTIMG'; + Role = `现在看到的是${opt.images.length}张图片（从第1张到第${opt.images.length}张），请依次查看各张图片。若觉得是表情包，并不是通知或其他类型的图片，请发送 DOWNIMG: 命名该表情。不需要发送过多的参数，只需要发送格式DOWNIMG: 命名该表情，注意不需要携带后缀；若不是表情包等，请发送NOTIMG并对图片内容进行分析描述。注意：请从第1张图片开始依次描述。`; break; case "default": Role = `你的名字是"${Config.assistantLabel}"，你在一个qq群里，群号是${group},当前和你说话的人群名片是${card}, qq号是${sender}, 请你结合用户的发言和聊天记录作出回应，要求表现得随性一点，最好参与讨论，混入其中。不要过分插科打诨，不知道说什么可以复读群友的话。要求你做搜索、发图、发视频和音乐等操作时要使用工具。不可以直接发[图片]这样蒙混过关。要求优先使用中文进行对话。` + @@ -297,9 +313,6 @@ export class bym extends plugin { if (Config.azSerpKey) { tools.push(new SerpTool()) } - if (Config.azSerpKey) { - tools.push(new SerpTool()) - } if (e.group.is_admin || e.group.is_owner) { tools.push(new EditCardTool()) tools.push(new JinyanTool()) @@ -321,7 +334,7 @@ export class bym extends plugin { let text = rsp.text let texts = text.split(/(?} */ - async sendMessage (text, opt = {}) { + async sendMessage(text, opt = {}) { let history = await this.getHistory(opt.parentMessageId) let systemMessage = opt.system if (systemMessage) { @@ -138,26 +139,40 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient { const idModel = crypto.randomUUID() const thisMessage = opt.functionResponse ? { - role: 'user', - parts: [{ - functionResponse: opt.functionResponse - }], - id: idThis, - parentMessageId: opt.parentMessageId || undefined - } + role: 'user', + parts: [{ + functionResponse: opt.functionResponse + }], + id: idThis, + parentMessageId: opt.parentMessageId || undefined + } : { - role: 'user', - parts: [{ text }], - id: idThis, - parentMessageId: opt.parentMessageId || undefined + role: 'user', + parts: [{ text }], + id: idThis, + parentMessageId: opt.parentMessageId || undefined + } + if (opt.image || opt.images) { + // 兼容旧版单图片 + if (opt.image) { + thisMessage.parts.push({ + inline_data: { + mime_type: 'image/jpeg', + data: opt.image + } + }) + } + // 处理多图片 + if (opt.images && Array.isArray(opt.images)) { + for (let imageData of opt.images) { + thisMessage.parts.push({ + inline_data: { + mime_type: 'image/jpeg', + data: imageData + } + }) } - if (opt.image) { - thisMessage.parts.push({ - inline_data: { - mime_type: 'image/jpeg', - data: opt.image - } - }) + } } history.push(_.cloneDeep(thisMessage)) let url = `${this.baseUrl}/v1beta/models/${this.model}:generateContent` @@ -220,7 +235,7 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient { if (opt.codeExecution) { body.tools.push({ code_execution: {} }) } - if (opt.image) { + if (opt.image || (opt.images && opt.images.length > 0)) { delete body.tools } body.contents.forEach(content => { @@ -365,7 +380,7 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient { * @param {Content} responseContent * @returns {{final: string, responseContent}} */ -function handleSearchResponse (responseContent) { +function handleSearchResponse(responseContent) { let final = '' // 遍历每个 part 并处理 diff --git a/utils/tools/ImageTool.js b/utils/tools/ImageTool.js index 5885214..4900e66 100644 --- a/utils/tools/ImageTool.js +++ b/utils/tools/ImageTool.js @@ -5,6 +5,9 @@ export class ImageProcessTool extends AbstractTool { name = 'imageProcess' #availableImages = [] #initialized = false + #currentImageIndex = 0 + #totalImages = 0 + #needReprocess = false parameters = { properties: { @@ -63,7 +66,7 @@ export class ImageProcessTool extends AbstractTool { -async initializeImageList() { + async initializeImageList() { try { this.#availableImages = await fileImgList() this.#initialized = true @@ -88,6 +91,19 @@ async initializeImageList() { } async processText(text, options = {}) { + // 初始化或重置图片处理信息 + if (options.images?.length > 0) { + this.#totalImages = options.images.length + // 只在新的处理周期时重置索引 + if (!this.#needReprocess && this.#currentImageIndex >= this.#totalImages) { + this.#currentImageIndex = 0 + } + } else if (options.image) { + this.#totalImages = 1 + if (!this.#needReprocess) { + this.#currentImageIndex = 0 + } + } const commands = { get: { @@ -101,21 +117,54 @@ async initializeImageList() { notImage: { regex: /NOTIMG(.*)/i, handler: async (match) => { - await this.handleNotImage(match[1]?.trim()) + const result = await this.handleNotImage() + if (result.success) { + // 标记需要重新处理当前图片 + this.#needReprocess = true + return { + success: true, + continueProcess: true, + currentIndex: this.#currentImageIndex, // 保持当前索引不变 + switchRole: result.switchRole, + needResponse: result.needResponse, + reprocess: true // 添加重新处理标记 + } + } return true } }, download: { regex: /DOWNIMG:\s*(.+)/i, handler: async (match) => { - await this.handleDownloadImage(match[1].trim(), options.image) + const baseName = match[1].trim() + if (options.images) { + const imageName = this.#totalImages === 1 + ? baseName + : `${baseName}_${this.#currentImageIndex + 1}` + await this.handleDownloadImage(imageName, options.images[this.#currentImageIndex]) + + // 处理完当前图片后，重置重新处理标记 + this.#needReprocess = false + + // 只有在不是重新处理时才增加索引 + if (this.#currentImageIndex < this.#totalImages - 1) { + this.#currentImageIndex++ + return { + success: true, + continueProcess: true, + currentIndex: this.#currentImageIndex + } + } + } else if (options.image) { + await this.handleDownloadImage(baseName, options.image) + } return true } }, list: { regex: /^(?:表情包列表|查看表情包|列出表情包)$/i, handler: async () => { - await this.e.reply(this.getImagesPrompt()) + await this.e.reply(await this.getImagesPrompt()) return true } } @@ -125,14 +174,14 @@ async initializeImageList() { const match = text.match(regex) if (match) { try { - return await handler(match) + const result = await handler(match) + return result } catch (error) { await this.e.reply(`处理失败: ${error.message}`) return true } } } - return null } async handleGetImage(imageName) { @@ -157,7 +206,6 @@ async initializeImageList() { if (!imageName || !imageData) { throw new Error('需要提供图片名称和数据') } - try { const text = `DOWNIMG: ${imageName}` await downImg(this.e, imageData, text) @@ -170,14 +218,22 @@ async initializeImageList() { async handleNotImage() { try { - this.ALLRole = this.previousRole - await this.bymGo(true) - return true + return { + success: true, + switchRole: this.previousRole, + needResponse: true + } } catch (error) { throw error } } - + getCurrentProgress() { + return { + currentIndex: this.#currentImageIndex, + totalImages: this.#totalImages, + isProcessing: this.#currentImageIndex < this.#totalImages + } + } description = `图片处理工具：支持发送本地表情包、保存新表情包、切换处理模式。 @@ -185,20 +241,27 @@ async initializeImageList() { - GETIMG: <表情包名称> - 发送指定表情包 - DOWNIMG: <名称> - 保存当前图片为表情包 - NOTIMG - 切换到文本模式 -- 表情包列表 - 查看所有可用表情包` +- 表情包列表 - 查看所有可用表情包 +\n\n多图片处理说明： +- 当处理多张图片时，会自动为每张图片添加序号后缀 +- 例如：DOWNIMG: happy 命令处理多张图片时会保存为 happy_1, happy_2 等\n` // 添加 getSystemPrompt 方法 async getSystemPrompt() { const images = await this.getAvailableImages() + const progress = this.getCurrentProgress() let prompt = `${this.description}\n` - + + if (progress.isProcessing) { + prompt += `\n当前正在处理第 ${progress.currentIndex + 1}/${progress.totalImages} 张图片\n` + } + if (images.length > 0) { prompt += `\n当前可用的表情包：\n${images.join('\n')}` prompt += `\n使用 GETIMG: <表情包名称> 来发送表情包` } else { - logger.warn('[ImageProcessTool] 没有可用的表情包') prompt += '\n当前没有可用的表情包' } - + return prompt } }