fix: 修复Gemini的图片和工具支持

client/CustomGoogleGeminiClient.js

@@ -66,11 +66,23 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
   }
 
   /**
-     *
+   *
-     * @param text
+   * @param text
-     * @param {{conversationId: string?, parentMessageId: string?, stream: boolean?, onProgress: function?, functionResponse: FunctionResponse?, system: string?, image: string?}} opt
+   * @param {{
-     * @returns {Promise<{conversationId: string?, parentMessageId: string, text: string, id: string}>}
+   *     conversationId: string?,
-     */
+   *     parentMessageId: string?,
+   *     stream: boolean?,
+   *     onProgress: function?,
+   *     functionResponse: FunctionResponse?,
+   *     system: string?,
+   *     image: string?,
+   *     maxOutputTokens: number?,
+   *     temperature: number?,
+   *     topP: number?,
+   *     tokK: number?
+   * }} opt
+   * @returns {Promise<{conversationId: string?, parentMessageId: string, text: string, id: string}>}
+   */
   async sendMessage (text, opt = {}) {
     let history = await this.getHistory(opt.parentMessageId)
     let systemMessage = opt.system
@@ -98,7 +110,7 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
     const idModel = crypto.randomUUID()
     const thisMessage = opt.functionResponse
       ? {
-          role: 'function',
+          role: 'user',
           parts: [{
             functionResponse: opt.functionResponse
           }],
@@ -120,7 +132,7 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
       })
     }
     history.push(_.cloneDeep(thisMessage))
-    let url = `${this.baseUrl}/v1beta/models/${this.model}:generateContent?key=${this._key}`
+    let url = `${this.baseUrl}/v1beta/models/${this.model}:generateContent`
     let body = {
       // 不去兼容官方的简单格式了，直接用，免得function还要转换
       /**
@@ -146,14 +158,15 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
         }
       ],
       generationConfig: {
-        maxOutputTokens: 1000,
+        maxOutputTokens: opt.maxOutputTokens || 1000,
-        temperature: 0.9,
+        temperature: opt.temperature || 0.9,
-        topP: 0.95,
+        topP: opt.topP || 0.95,
-        topK: 16
+        topK: opt.tokK || 16
       },
       tools: [
         {
           functionDeclarations: this.tools.map(tool => tool.function())
+          // codeExecution: {}
         }
       ]
     }
@@ -167,7 +180,10 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
     })
     let result = await newFetch(url, {
       method: 'POST',
-      body: JSON.stringify(body)
+      body: JSON.stringify(body),
+      headers: {
+        'x-goog-api-key': this._key
+      }
     })
     if (result.status !== 200) {
       throw new Error(await result.text())
@@ -177,8 +193,8 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
      */
     let responseContent
     /**
-       * @type {{candidates: Array<{content: Content}>}}
+     * @type {{candidates: Array<{content: Content}>}}
-       */
+     */
     let response = await result.json()
     if (this.debug) {
       console.log(JSON.stringify(response))
@@ -193,8 +209,8 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
         const funcName = functionCall.name
         let chosenTool = this.tools.find(t => t.name === funcName)
         /**
-           * @type {FunctionResponse}
+         * @type {FunctionResponse}
-           */
+         */
         let functionResponse = {
           name: funcName,
           response: {
@@ -259,7 +275,7 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient {
       await this.upsertMessage(respMessage)
     }
     return {
-      text: responseContent.parts[0].text,
+      text: responseContent.parts[0].text.trim(),
       conversationId: '',
       parentMessageId: idThis,
       id: idModel

model/core.js

@@ -735,17 +735,15 @@ class Core {
         parentMessageId: conversation.parentMessageId,
         conversationId: conversation.conversationId
       }
-      if (Config.geminiModel.includes('vision')) {
+      const image = await getImg(e)
-        const image = await getImg(e)
+      let imageUrl = image ? image[0] : undefined
-        let imageUrl = image ? image[0] : undefined
+      if (imageUrl) {
-        if (imageUrl) {
+        let md5 = imageUrl.split(/[/-]/).find(s => s.length === 32)?.toUpperCase()
-          let md5 = imageUrl.split(/[/-]/).find(s => s.length === 32)?.toUpperCase()
+        let imageLoc = await getOrDownloadFile(`ocr/${md5}.png`, imageUrl)
-          let imageLoc = await getOrDownloadFile(`ocr/${md5}.png`, imageUrl)
+        let outputLoc = imageLoc.replace(`${md5}.png`, `${md5}_512.png`)
-          let outputLoc = imageLoc.replace(`${md5}.png`, `${md5}_512.png`)
+        await resizeAndCropImage(imageLoc, outputLoc, 512)
-          await resizeAndCropImage(imageLoc, outputLoc, 512)
+        let buffer = fs.readFileSync(outputLoc)
-          let buffer = fs.readFileSync(outputLoc)
+        option.image = buffer.toString('base64')
-          option.image = buffer.toString('base64')
-        }
       }
       if (Config.smartMode) {
         /**

utils/config.js

@@ -175,10 +175,13 @@ const defaultConfig = {
   qwenTemperature: 1,
   qwenEnableSearch: true,
   geminiKey: '',
-  geminiModel: 'gemini-pro',
+  geminiModel: 'gemini-1.5-flash',
   geminiPrompt: 'You are Gemini. Your answer shouldn\'t be too verbose. Prefer to answer in Chinese.',
   // origin: https://generativelanguage.googleapis.com
   geminiBaseUrl: 'https://gemini.ikechan8370.com',
+  geminiTemperature: 0.9,
+  geminiMaxOutputTokens: 2000,
+
   chatglmRefreshToken: '',
   sunoSessToken: '',
   sunoClientToken: '',

utils/tools/SendMessageToSpecificGroupOrUserTool.js

@@ -15,7 +15,7 @@ export class SendMessageToSpecificGroupOrUserTool extends AbstractTool {
         description: 'target qq or group number'
       }
     },
-    required: ['msg', 'target']
+    required: ['msg', 'targetGroupIdOrQQNumber']
   }
 
   func = async function (opt, e) {

utils/tools/SendMusicTool.js

@@ -14,7 +14,7 @@ export class SendMusicTool extends AbstractTool {
         description: 'Fill in the target user_id or groupId when you need to send music to specific group or user, otherwise leave blank'
       }
     },
-    required: ['keyword']
+    required: ['id']
   }
 
   func = async function (opts, e) {