From 4eaf218d662d109f3ec9f04ed9ea9729c67f022a Mon Sep 17 00:00:00 2001
From: ikechan8370 <geyinchibuaa@gmail.com>
Date: Sat, 16 Mar 2024 00:36:27 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20add=20#=E8=AF=86=E5=9B=BE?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 apps/entertainment.js              | 33 ++++++++++++++++++++++++++++++
 client/CustomGoogleGeminiClient.js |  3 +++
 2 files changed, 36 insertions(+)

diff --git a/apps/entertainment.js b/apps/entertainment.js
index b0ba5b3..2e7ac8a 100644
--- a/apps/entertainment.js
+++ b/apps/entertainment.js
@@ -12,6 +12,7 @@ import AzureTTS from '../utils/tts/microsoft-azure.js'
 import VoiceVoxTTS from '../utils/tts/voicevox.js'
 import { URL } from 'node:url'
 import { getBots } from '../utils/bot.js'
+import {CustomGoogleGeminiClient} from "../client/CustomGoogleGeminiClient.js";
 
 let useSilk = false
 try {
@@ -69,6 +70,10 @@ export class Entertainment extends plugin {
         {
           reg: '^#url(：|:)',
           fnc: 'screenshotUrl'
+        },
+        {
+          reg: '^#(识图|图片识别|VQA|vqa)',
+          fnc: 'vqa'
         }
       ]
     })
@@ -598,4 +603,32 @@ ${translateLangLabels}
     }
     return true
   }
+
+  async vqa (e) {
+    if (!Config.geminiKey) {
+      e.reply('需要配置Gemini密钥以使用识图')
+      return
+    }
+    let img = await getImg(e)
+    if (!img?.[0]) {
+      await this.reply('请发送或引用一张图片', e.isGroup)
+      return false
+    }
+    let client = new CustomGoogleGeminiClient({
+      e,
+      userId: e.sender.user_id,
+      key: Config.geminiKey,
+      model: 'gemini-pro-vision',
+      baseUrl: Config.geminiBaseUrl,
+      debug: Config.debug
+    })
+    const response = await fetch(img[0])
+    const base64Image = Buffer.from(await response.arrayBuffer())
+    let msg = e.msg.replace(/#(识图|图片识别|VQA|vqa)/, '') || 'describe this image in Simplified Chinese'
+    let res = await client.sendMessage(msg, {
+      image: base64Image.toString('base64')
+    })
+    await e.reply(res.text)
+    return true
+  }
 }
diff --git a/client/CustomGoogleGeminiClient.js b/client/CustomGoogleGeminiClient.js
index 9276a4e..3c1ca03 100644
--- a/client/CustomGoogleGeminiClient.js
+++ b/client/CustomGoogleGeminiClient.js
@@ -157,6 +157,9 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
         }
       ]
     }
+    if (opt.image) {
+      delete body.tools
+    }
     body.contents.forEach(content => {
       delete content.id
       delete content.parentMessageId