Merge branch 'v2' of https://github.com/ikechan8370/chatgpt-plugin into v2

README.md

@@ -1,5 +1,4 @@
-![chatgpt-plugin](https://user-images.githubusercontent.com/21212372/232115814-de9a0633-371f-4733-8da0-dd6e912c8a1e.png)
+![chatgpt-plugin](https://socialify.git.ci/ikechan8370/chatgpt-plugin/image?description=1&font=Jost&forks=1&issues=1&language=1&name=1&owner=1&pulls=1&stargazers=1&theme=Light)
-<div align=center> <h1>云崽系机器人的智能聊天插件</h1> </div>
 <div align=center>
 
    <img src ="https://img.shields.io/github/issues/ikechan8370/chatgpt-plugin?logo=github"/>
@@ -43,6 +42,9 @@
 * 2023-09-10 支持来自claude.ai的claude-2模型
 * 2023-10-19 支持读取文件，（目前适配必应模式和Claude2模式）
 * 2023-10-25 增加支持通义千问官方API
+* 2023-12-01 持续优先适配Shamrock
+* 2023-12-14 增加支持Gemini 官方API
+
 ### 如果觉得这个插件有趣或者对你有帮助，请点一个star吧！
 
 ## 版本要求

apps/draw.js

@@ -277,7 +277,7 @@ export class dalle extends plugin {
       await client.getImages(prompt, e)
     } catch (err) {
       await redis.del(`CHATGPT:DRAW:${e.sender.user_id}`)
-      await e.reply('绘图失败：' + err)
+      await e.reply('❌绘图失败：' + err)
     }
   }
 }

apps/management.js

@@ -1,4 +1,5 @@
 import plugin from '../../../lib/plugins/plugin.js'
+import { exec } from 'child_process'
 import { Config } from '../utils/config.js'
 import {
   formatDuration,
@@ -126,6 +127,11 @@ export class ChatgptManagement extends plugin {
           fnc: 'useClaudeAISolution',
           permission: 'master'
         },
+        {
+          reg: '^#chatgpt切换(Gemini|gemini)$',
+          fnc: 'useGeminiSolution',
+          permission: 'master'
+        },
         {
           reg: '^#chatgpt切换星火$',
           fnc: 'useXinghuoBasedSolution',
@@ -184,6 +190,11 @@ export class ChatgptManagement extends plugin {
           fnc: 'setAPIKey',
           permission: 'master'
         },
+        {
+          reg: '^#chatgpt设置(Gemini|gemini)(Key|key)$',
+          fnc: 'setGeminiKey',
+          permission: 'master'
+        },
         {
           reg: '^#chatgpt设置(API|api)设定$',
           fnc: 'setAPIPromptPrefix',
@@ -314,6 +325,11 @@ export class ChatgptManagement extends plugin {
           reg: '^#chatgpt设置星火模型$',
           fnc: 'setXinghuoModel',
           permission: 'master'
+        },
+        {
+          reg: '^#chatgpt修补Gemini$',
+          fnc: 'patchGemini',
+          permission: 'master'
         }
       ]
     })
@@ -902,6 +918,16 @@ azure语音：Azure 语音是微软 Azure 平台提供的一项语音服务，
     }
   }
 
+  async useGeminiSolution () {
+    let use = await redis.get('CHATGPT:USE')
+    if (use !== 'gemini') {
+      await redis.set('CHATGPT:USE', 'gemini')
+      await this.reply('已切换到基于Google Gemini的解决方案')
+    } else {
+      await this.reply('当前已经是gemini模式了')
+    }
+  }
+
   async useXinghuoBasedSolution () {
     let use = await redis.get('CHATGPT:USE')
     if (use !== 'xh') {
@@ -932,6 +958,57 @@ azure语音：Azure 语音是微软 Azure 平台提供的一项语音服务，
     }
   }
 
+  async patchGemini () {
+    const _path = process.cwd()
+    let packageJson = fs.readFileSync(`${_path}/package.json`)
+    packageJson = JSON.parse(String(packageJson))
+    const packageName = '@google/generative-ai@0.1.1'
+    const patchLoc = 'plugins/chatgpt-plugin/patches/@google__generative-ai@0.1.1.patch'
+    if (!packageJson.pnpm) {
+      packageJson.pnpm = {
+        patchedDependencies: {
+          [packageName]: patchLoc
+        }
+      }
+    } else {
+      if (packageJson.pnpm.patchedDependencies) {
+        packageJson.pnpm.patchedDependencies[packageName] = patchLoc
+      } else {
+        packageJson.pnpm.patchedDependencies = {
+          [packageName]: patchLoc
+        }
+      }
+    }
+    fs.writeFileSync(`${_path}/package.json`, JSON.stringify(packageJson, null, 2))
+
+    function execSync (cmd) {
+      return new Promise((resolve, reject) => {
+        exec(cmd, (error, stdout, stderr) => {
+          resolve({ error, stdout, stderr })
+        })
+      })
+    }
+    async function checkPnpm () {
+      let npm = 'npm'
+      let ret = await execSync('pnpm -v')
+      if (ret.stdout) npm = 'pnpm'
+      return npm
+    }
+    let npmv = await checkPnpm()
+    if (npmv === 'pnpm') {
+      exec('pnpm i', {}, (error, stdout, stderr) => {
+        if (error) {
+          logger.error(error)
+          logger.error(stderr)
+          logger.info(stdout)
+          this.e.reply('失败，请查看日志手动操作')
+        } else {
+          this.e.reply('修补完成，请手动重启')
+        }
+      })
+    }
+  }
+
   async useQwenSolution () {
     let use = await redis.get('CHATGPT:USE')
     if (use !== 'qwen') {
@@ -1148,6 +1225,21 @@ azure语音：Azure 语音是微软 Azure 平台提供的一项语音服务，
     this.finish('saveAPIKey')
   }
 
+  async setGeminiKey (e) {
+    this.setContext('saveGeminiKey')
+    await this.reply('请发送Gemini API Key.获取地址：https://makersuite.google.com/app/apikey', true)
+    return false
+  }
+
+  async saveGeminiKey () {
+    if (!this.e.msg) return
+    let token = this.e.msg
+    // todo
+    Config.geminiKey = token
+    await this.reply('请发送Gemini API Key设置成功', true)
+    this.finish('saveGeminiKey')
+  }
+
   async setXinghuoToken () {
     this.setContext('saveXinghuoToken')
     await this.reply('请发送星火的ssoSessionId', true)

apps/prompts.js

@@ -158,7 +158,8 @@ export class help extends plugin {
       api: 'promptPrefixOverride',
       Custom: 'sydney',
       claude: 'slackClaudeGlobalPreset',
-      qwen: 'promptPrefixOverride'
+      qwen: 'promptPrefixOverride',
+      gemini: 'geminiPrompt'
     }
 
     if (keyMap[use]) {
@@ -171,7 +172,7 @@ export class help extends plugin {
       await redis.set(`CHATGPT:PROMPT_USE_${use}`, promptName)
       await e.reply(`你当前正在使用${use}模式，已将该模式设定应用为"${promptName}"。更该设定后建议结束对话以使设定更好生效`, true)
     } else {
-      await e.reply(`你当前正在使用${use}模式，该模式不支持设定。支持设定的模式有：API、自定义、Claude`, true)
+      await e.reply(`你当前正在使用${use}模式，该模式不支持设定。支持设定的模式有：API、自定义、Claude、通义千问和Gemini`, true)
     }
   }
 

client/BaseClient.js

@@ -14,13 +14,18 @@ export class BaseClient {
   constructor (props = {}) {
     this.supportFunction = false
     this.maxToken = 4096
+    /**
+     * @type {Array<AbstractTool>}
+     */
     this.tools = []
     const {
-      e, getMessageById, upsertMessage
+      e, getMessageById, upsertMessage, deleteMessageById, userId
     } = props
     this.e = e
     this.getMessageById = getMessageById
     this.upsertMessage = upsertMessage
+    this.deleteMessageById = deleteMessageById || (() => {})
+    this.userId = userId
   }
 
   /**
@@ -36,20 +41,28 @@ export class BaseClient {
    * insert or update a message with the id
    *
    * @type function
-   * @param {string} id
    * @param {object} message
    * @return {Promise<void>}
    */
   upsertMessage
 
+  /**
+   * delete a message with the id
+   *
+   * @type function
+   * @param {string} id
+   * @return {Promise<void>}
+   */
+  deleteMessageById
+
   /**
    * Send prompt message with history and return response message \
    * if function called, handled internally \
    * override this method to implement logic of sending and receiving message
    *
-   * @param msg
+   * @param {string} msg
-   * @param opt other options, optional fields: [conversationId, parentMessageId], if not set, random uuid instead
+   * @param {{conversationId: string?, parentMessageId: string?, stream: boolean?, onProgress: function?}} opt other options, optional fields: [conversationId, parentMessageId], if not set, random uuid instead
-   * @returns {Promise<Message>} required fields: [text, conversationId, parentMessageId, id]
+   * @returns {Promise<{text, conversationId, parentMessageId, id}>} required fields: [text, conversationId, parentMessageId, id]
    */
   async sendMessage (msg, opt = {}) {
     throw new Error('not implemented in abstract client')
@@ -60,11 +73,12 @@ export class BaseClient {
    * override this method to implement logic of getting history
    * keyv with local file or redis recommended
    *
-   * @param userId such as qq number
+   * @param userId optional, such as qq number
-   * @param opt other options
+   * @param parentMessageId if blank, no history
-   * @returns {Promise<void>}
+   * @param opt optional, other options
+   * @returns {Promise<object[]>}
    */
-  async getHistory (userId, opt = {}) {
+  async getHistory (parentMessageId, userId = this.userId, opt = {}) {
     throw new Error('not implemented in abstract client')
   }
 
@@ -78,14 +92,18 @@ export class BaseClient {
     throw new Error('not implemented in abstract client')
   }
 
-  addTools (...tools) {
+  /**
+   * 增加tools
+   * @param {[AbstractTool]} tools
+   */
+  addTools (tools) {
     if (!this.isSupportFunction) {
       throw new Error('function not supported')
     }
     if (!this.tools) {
       this.tools = []
     }
-    this.tools.push(tools)
+    this.tools.push(...tools)
   }
 
   getTools () {

client/CustomGoogleGeminiClient.js

@@ -0,0 +1,264 @@
+import crypto from 'crypto'
+import { GoogleGeminiClient } from './GoogleGeminiClient.js'
+import { newFetch } from '../utils/proxy.js'
+import _ from 'lodash'
+
+const BASEURL = 'https://generativelanguage.googleapis.com'
+
+export const HarmCategory = {
+  HARM_CATEGORY_UNSPECIFIED: 'HARM_CATEGORY_UNSPECIFIED',
+  HARM_CATEGORY_HATE_SPEECH: 'HARM_CATEGORY_HATE_SPEECH',
+  HARM_CATEGORY_SEXUALLY_EXPLICIT: 'HARM_CATEGORY_SEXUALLY_EXPLICIT',
+  HARM_CATEGORY_HARASSMENT: 'HARM_CATEGORY_HARASSMENT',
+  HARM_CATEGORY_DANGEROUS_CONTENT: 'HARM_CATEGORY_DANGEROUS_CONTENT'
+}
+
+export const HarmBlockThreshold = {
+  HARM_BLOCK_THRESHOLD_UNSPECIFIED: 'HARM_BLOCK_THRESHOLD_UNSPECIFIED',
+  BLOCK_LOW_AND_ABOVE: 'BLOCK_LOW_AND_ABOVE',
+  BLOCK_MEDIUM_AND_ABOVE: 'BLOCK_MEDIUM_AND_ABOVE',
+  BLOCK_ONLY_HIGH: 'BLOCK_ONLY_HIGH',
+  BLOCK_NONE: 'BLOCK_NONE'
+}
+
+/**
+ * @typedef {{
+ *   role: string,
+ *   parts: Array<{
+ *     text?: string,
+ *     functionCall?: FunctionCall,
+ *     functionResponse?: FunctionResponse
+ *   }>
+ * }} Content
+ *
+ * Gemini消息的基本格式
+ */
+
+/**
+ * @typedef {{
+ *    name: string,
+ *    args: {}
+ * }} FunctionCall
+ *
+ * Gemini的FunctionCall
+ */
+
+/**
+ * @typedef {{
+ *   name: string,
+ *   response: {
+ *     name: string,
+ *     content: {}
+ *   }
+ * }} FunctionResponse
+ *
+ * Gemini的Function执行结果包裹
+ * 其中response可以为任意，本项目根据官方示例封装为name和content两个字段
+ */
+
+export class CustomGoogleGeminiClient extends GoogleGeminiClient {
+  constructor (props) {
+    super(props)
+    this.model = props.model
+    this.baseUrl = props.baseUrl || BASEURL
+    this.supportFunction = true
+    this.debug = props.debug
+  }
+
+  /**
+     *
+     * @param text
+     * @param {{conversationId: string?, parentMessageId: string?, stream: boolean?, onProgress: function?, functionResponse: FunctionResponse?, system: string?, image: string?}} opt
+     * @returns {Promise<{conversationId: string?, parentMessageId: string, text: string, id: string}>}
+     */
+  async sendMessage (text, opt) {
+    let history = await this.getHistory(opt.parentMessageId)
+    let systemMessage = opt.system
+    if (systemMessage) {
+      history = history.reverse()
+      history.push({
+        role: 'model',
+        parts: [
+          {
+            text: 'ok'
+          }
+        ]
+      })
+      history.push({
+        role: 'user',
+        parts: [
+          {
+            text: systemMessage
+          }
+        ]
+      })
+      history = history.reverse()
+    }
+    const idThis = crypto.randomUUID()
+    const idModel = crypto.randomUUID()
+    const thisMessage = opt.functionResponse
+      ? {
+          role: 'function',
+          parts: [{
+            functionResponse: opt.functionResponse
+          }],
+          id: idThis,
+          parentMessageId: opt.parentMessageId || undefined
+        }
+      : {
+          role: 'user',
+          parts: [{ text }],
+          id: idThis,
+          parentMessageId: opt.parentMessageId || undefined
+        }
+    if (opt.image) {
+      thisMessage.parts.push({
+        inline_data: {
+          mime_type: 'image/jpeg',
+          data: opt.image
+        }
+      })
+    }
+    history.push(_.cloneDeep(thisMessage))
+    let url = `${this.baseUrl}/v1beta/models/${this.model}:generateContent?key=${this._key}`
+    let body = {
+      // 不去兼容官方的简单格式了，直接用，免得function还要转换
+      /**
+       * @type Array<Content>
+       */
+      contents: history,
+      safetySettings: [
+        {
+          category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+          threshold: HarmBlockThreshold.BLOCK_NONE
+        },
+        {
+          category: HarmCategory.HARM_CATEGORY_HARASSMENT,
+          threshold: HarmBlockThreshold.BLOCK_NONE
+        },
+        {
+          category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+          threshold: HarmBlockThreshold.BLOCK_NONE
+        },
+        {
+          category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+          threshold: HarmBlockThreshold.BLOCK_NONE
+        }
+      ],
+      generationConfig: {
+        maxOutputTokens: 1000,
+        temperature: 0.9,
+        topP: 0.95,
+        topK: 16
+      },
+      tools: [
+        {
+          functionDeclarations: this.tools.map(tool => tool.function())
+        }
+      ]
+    }
+    body.contents.forEach(content => {
+      delete content.id
+      delete content.parentMessageId
+      delete content.conversationId
+    })
+    let result = await newFetch(url, {
+      method: 'POST',
+      body: JSON.stringify(body)
+    })
+    if (result.status !== 200) {
+      throw new Error(await result.text())
+    }
+    /**
+     * @type {Content | undefined}
+     */
+    let responseContent
+    /**
+       * @type {{candidates: Array<{content: Content}>}}
+       */
+    let response = await result.json()
+    if (this.debug) {
+      console.log(JSON.stringify(response))
+    }
+    responseContent = response.candidates[0].content
+    if (responseContent.parts[0].functionCall) {
+      // functionCall
+      const functionCall = responseContent.parts[0].functionCall
+      // Gemini有时候只回复一个空的functionCall,无语死了
+      if (functionCall.name) {
+        logger.info(JSON.stringify(functionCall))
+        const funcName = functionCall.name
+        let chosenTool = this.tools.find(t => t.name === funcName)
+        /**
+           * @type {FunctionResponse}
+           */
+        let functionResponse = {
+          name: funcName,
+          response: {
+            name: funcName,
+            content: null
+          }
+        }
+        if (!chosenTool) {
+          // 根本没有这个工具！
+          functionResponse.response.content = {
+            error: `Function ${funcName} doesn't exist`
+          }
+        } else {
+          // execute function
+          try {
+            let args = Object.assign(functionCall.args, {
+              isAdmin: this.e.group.is_admin,
+              isOwner: this.e.group.is_owner,
+              sender: this.e.sender
+            })
+            functionResponse.response.content = await chosenTool.func(args, this.e)
+            if (this.debug) {
+              logger.info(JSON.stringify(functionResponse.response.content))
+            }
+          } catch (err) {
+            logger.error(err)
+            functionResponse.response.content = {
+              error: `Function execute error: ${err.message}`
+            }
+          }
+        }
+        let responseOpt = _.cloneDeep(opt)
+        responseOpt.parentMessageId = idModel
+        responseOpt.functionResponse = functionResponse
+        // 递归直到返回text
+        // 先把这轮的消息存下来
+        await this.upsertMessage(thisMessage)
+        const respMessage = Object.assign(responseContent, {
+          id: idModel,
+          parentMessageId: idThis
+        })
+        await this.upsertMessage(respMessage)
+        return await this.sendMessage('', responseOpt)
+      } else {
+        // 谷歌抽风了，瞎调函数，不保存这轮，直接返回
+        return {
+          text: '',
+          conversationId: '',
+          parentMessageId: opt.parentMessageId,
+          id: '',
+          error: true
+        }
+      }
+    }
+    if (responseContent) {
+      await this.upsertMessage(thisMessage)
+      const respMessage = Object.assign(responseContent, {
+        id: idModel,
+        parentMessageId: idThis
+      })
+      await this.upsertMessage(respMessage)
+    }
+    return {
+      text: responseContent.parts[0].text,
+      conversationId: '',
+      parentMessageId: idThis,
+      id: idModel
+    }
+  }
+}

client/GoogleGeminiClient.js

@@ -0,0 +1,158 @@
+import { BaseClient } from './BaseClient.js'
+
+import { getMessageById, upsertMessage } from '../utils/common.js'
+import crypto from 'crypto'
+let GoogleGenerativeAI, HarmBlockThreshold, HarmCategory
+try {
+  const GenerativeAI = await import('@google/generative-ai')
+  GoogleGenerativeAI = GenerativeAI.GoogleGenerativeAI
+  HarmBlockThreshold = GenerativeAI.HarmBlockThreshold
+  HarmCategory = GenerativeAI.HarmCategory
+} catch (err) {
+  console.warn('未安装@google/generative-ai，无法使用Gemini，请在chatgpt-plugin目录下执行pnpm i安装新依赖')
+}
+export class GoogleGeminiClient extends BaseClient {
+  constructor (props) {
+    if (!GoogleGenerativeAI) {
+      throw new Error('未安装@google/generative-ai，无法使用Gemini，请在chatgpt-plugin目录下执行pnpm i安装新依赖')
+    }
+    if (!props.upsertMessage) {
+      props.upsertMessage = async function umGemini (message) {
+        return await upsertMessage(message, 'Gemini')
+      }
+    }
+    if (!props.getMessageById) {
+      props.getMessageById = async function umGemini (message) {
+        return await getMessageById(message, 'Gemini')
+      }
+    }
+    super(props)
+    this._key = props.key
+    this._client = new GoogleGenerativeAI(this._key)
+    this.model = this._client.getGenerativeModel({ model: props.model })
+    this.supportFunction = false
+  }
+
+  async getHistory (parentMessageId, userId = this.userId, opt = {}) {
+    const history = []
+    let cursor = parentMessageId
+    if (!cursor) {
+      return history
+    }
+    do {
+      let parentMessage = await this.getMessageById(cursor)
+      if (!parentMessage) {
+        break
+      } else {
+        history.push(parentMessage)
+        cursor = parentMessage.parentMessageId
+        if (!cursor) {
+          break
+        }
+      }
+    } while (true)
+    return history.reverse()
+  }
+
+  async sendMessage (text, opt) {
+    let history = await this.getHistory(opt.parentMessageId)
+    let systemMessage = opt.system
+    if (systemMessage) {
+      history = history.reverse()
+      history.push({
+        role: 'model',
+        parts: 'ok'
+      })
+      history.push({
+        role: 'user',
+        parts: systemMessage
+      })
+      history = history.reverse()
+    }
+    const idUser = crypto.randomUUID()
+    const idModel = crypto.randomUUID()
+    let responseText = ''
+    try {
+      const chat = this.model.startChat({
+        history,
+        //   [
+        //   {
+        //     role: 'user',
+        //     parts: 'Hello, I have 2 dogs in my house.'
+        //   },
+        //   {
+        //     role: 'model',
+        //     parts: 'Great to meet you. What would you like to know?'
+        //   }
+        // ],
+        generationConfig: {
+          // todo configuration
+          maxOutputTokens: 1000,
+          temperature: 0.9,
+          topP: 0.95,
+          topK: 16
+        },
+        safetySettings: [
+          // todo configuration
+          {
+            category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+            threshold: HarmBlockThreshold.BLOCK_NONE
+          },
+          {
+            category: HarmCategory.HARM_CATEGORY_HARASSMENT,
+            threshold: HarmBlockThreshold.BLOCK_NONE
+          },
+          {
+            category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+            threshold: HarmBlockThreshold.BLOCK_NONE
+          },
+          {
+            category: HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+            threshold: HarmBlockThreshold.BLOCK_NONE
+          }
+        ]
+      })
+      if (opt.stream && (typeof opt.onProgress === 'function')) {
+        const result = await chat.sendMessageStream(text)
+        responseText = ''
+        for await (const chunk of result.stream) {
+          const chunkText = chunk.text()
+          responseText += chunkText
+          await opt.onProgress(responseText)
+        }
+        return {
+          text: responseText,
+          conversationId: '',
+          parentMessageId: idUser,
+          id: idModel
+        }
+      }
+      const result = await chat.sendMessage(text)
+      const response = await result.response
+      responseText = response.text()
+      return {
+        text: responseText,
+        conversationId: '',
+        parentMessageId: idUser,
+        id: idModel
+      }
+    } finally {
+      await this.upsertMessage({
+        role: 'user',
+        parts: text,
+        id: idUser,
+        parentMessageId: opt.parentMessageId || undefined
+      })
+      await this.upsertMessage({
+        role: 'model',
+        parts: responseText,
+        id: idModel,
+        parentMessageId: idUser
+      })
+    }
+  }
+
+  async destroyHistory (conversationId, opt = {}) {
+    // todo clean history
+  }
+}

client/GoogleGeminiClientTest.js

@@ -0,0 +1,10 @@
+import { GoogleGeminiClient } from './GoogleGeminiClient.js'
+
+async function test () {
+  const client = new GoogleGeminiClient({
+    e: {},
+    userId: 'test',
+    key: '',
+    model: 'gemini-pro'
+  })
+}

guoba.support.js

@@ -743,15 +743,42 @@ export function supportGuoba () {
           component: 'Switch'
         },
         {
-          label: '以下为杂七杂八的配置',
+          label: '以下为Gemini方式的配置',
           component: 'Divider'
         },
         {
-          field: '2captchaToken',
+          field: 'geminiKey',
-          label: '验证码平台Token',
+          label: 'API密钥',
-          bottomHelpMessage: '可注册2captcha实现跳过验证码，收费服务但很便宜。否则可能会遇到验证码而卡住',
+          bottomHelpMessage: '前往https://makersuite.google.com/app/apikey获取',
           component: 'InputPassword'
         },
+        {
+          field: 'geminiModel',
+          label: '模型',
+          bottomHelpMessage: '目前仅支持gemini-pro',
+          component: 'Input'
+        },
+        {
+          field: 'geminiPrompt',
+          label: '设定',
+          component: 'InputTextArea'
+        },
+        {
+          field: 'geminiBaseUrl',
+          label: 'Gemini反代',
+          bottomHelpMessage: '对https://generativelanguage.googleapis.com的反代',
+          component: 'Input'
+        },
+        {
+          label: '以下为杂七杂八的配置',
+          component: 'Divider'
+        },
+        // {
+        //   field: '2captchaToken',
+        //   label: '验证码平台Token',
+        //   bottomHelpMessage: '可注册2captcha实现跳过验证码，收费服务但很便宜。否则可能会遇到验证码而卡住',
+        //   component: 'InputPassword'
+        // },
         {
           field: 'ttsSpace',
           label: 'vits-uma-genshin-honkai语音转换API地址',

package.json

@@ -8,11 +8,9 @@
     "@fastify/cors": "^8.2.0",
     "@fastify/static": "^6.9.0",
     "@fastify/websocket": "^8.2.0",
+    "@google/generative-ai": "^0.1.1",
     "@slack/bolt": "^3.13.2",
-    "@waylaidwanderer/chatgpt-api": "^1.37.1",
     "asn1.js": "^5.0.0",
-    "chatgpt": "^5.2.4",
-    "crypto": "^1.0.1",
     "delay": "^6.0.0",
     "diff": "^5.1.0",
     "emoji-strip": "^1.0.1",
@@ -24,6 +22,7 @@
     "js-tiktoken": "^1.0.5",
     "keyv": "^4.5.3",
     "keyv-file": "^0.2.0",
+    "lodash": "^4.17.21",
     "microsoft-cognitiveservices-speech-sdk": "1.32.0",
     "node-fetch": "^3.3.1",
     "openai": "^3.2.1",
@@ -35,21 +34,26 @@
     "ws": "^8.13.0"
   },
   "optionalDependencies": {
-    "xlsx": "^0.18.5",
-    "mammoth": "^1.6.0",
-    "pdfjs-dist": "^3.11.174",
-    "nodejs-pptx": "^1.2.4",
     "@node-rs/jieba": "^1.6.2",
     "cycletls": "^1.0.21",
     "jimp": "^0.22.7",
+    "mammoth": "^1.6.0",
     "node-silk": "^0.1.0",
+    "nodejs-pptx": "^1.2.4",
+    "pdfjs-dist": "^3.11.174",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-recaptcha": "^3.6.8",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
-    "sharp": "^0.32.3"
+    "sharp": "^0.32.3",
+    "xlsx": "^0.18.5"
   },
   "devDependencies": {
     "ts-node": "^10.9.1",
     "ts-node-register": "^1.0.0"
+  },
+  "pnpm": {
+    "patchedDependencies": {
+      "@google/generative-ai@0.1.1": "patches/@google__generative-ai@0.1.1.patch"
+    }
   }
 }

patches/@google__generative-ai@0.1.1.patch

@@ -0,0 +1,26 @@
+diff --git a/dist/index.js b/dist/index.js
+index c71c104e7b8ee70ed1b5a5141d04c98109fe6439..2dd8b1f93de0e502729cb91c9618bf80e8559e1e 100644
+--- a/dist/index.js
++++ b/dist/index.js
+@@ -152,7 +152,7 @@ class GoogleGenerativeAIResponseError extends GoogleGenerativeAIError {
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+-const BASE_URL = "https://generativelanguage.googleapis.com";
++const BASE_URL = "https://gemini.ikechan8370.com";
+ const API_VERSION = "v1";
+ /**
+  * We can't `require` package.json if this runs on web. We will use rollup to
+diff --git a/dist/index.mjs b/dist/index.mjs
+index 402a0c7fa5b692dea07d2dfd83e0148f0a493ca2..c48ce6d612a8752a5161da574804e7a830700d2c 100644
+--- a/dist/index.mjs
++++ b/dist/index.mjs
+@@ -150,7 +150,7 @@ class GoogleGenerativeAIResponseError extends GoogleGenerativeAIError {
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+-const BASE_URL = "https://generativelanguage.googleapis.com";
++const BASE_URL = "https://gemini.ikechan8370.com";
+ const API_VERSION = "v1";
+ /**
+  * We can't `require` package.json if this runs on web. We will use rollup to

utils/BingDraw.js

@@ -95,7 +95,7 @@ export default class BingDrawClient {
     let pollingUrl = `${this.opts.baseUrl}/images/create/async/results/${requestId}?q=${urlEncodedPrompt}`
     logger.info({ pollingUrl })
     logger.info('waiting for bing draw results...')
-    let timeoutTimes = 30
+    let timeoutTimes = 50
     let found = false
     let timer = setInterval(async () => {
       if (found) {
@@ -113,15 +113,20 @@ export default class BingDrawClient {
           // 很可能是微软内部error，重试即可
           return
         }
-        imageLinks = imageLinks.map(link => link.split('?w=')[0]).map(link => link.replace('src="', ''))
+        imageLinks = imageLinks
+          .map(link => link.split('?w=')[0])
+          .map(link => link.replace('src="', ''))
+          .filter(link => !link.includes('.svg'))
         imageLinks = [...new Set(imageLinks)]
         const badImages = [
+          'https://r.bing.com/rp/in-2zU3AJUdkgFe7ZKv19yPBHVs.png"',
+          'https://r.bing.com/rp/TX9QuO3WzcCJz1uaaSwQAz39Kb0.jpg"',
           'https://r.bing.com/rp/in-2zU3AJUdkgFe7ZKv19yPBHVs.png',
           'https://r.bing.com/rp/TX9QuO3WzcCJz1uaaSwQAz39Kb0.jpg'
         ]
         for (let imageLink of imageLinks) {
           if (badImages.indexOf(imageLink) > -1) {
-            await e.reply('绘图失败：Bad images', true)
+            await e.reply('❌绘图失败：绘图完成但被屏蔽，请调整提示词。', true)
             logger.error(rText)
           }
         }
@@ -132,7 +137,7 @@ export default class BingDrawClient {
         clearInterval(timer)
       } else {
         if (timeoutTimes === 0) {
-          await e.reply('绘图超时', true)
+          await e.reply('❌绘图超时', true)
           clearInterval(timer)
           timer = null
         } else {
@@ -140,6 +145,6 @@ export default class BingDrawClient {
           timeoutTimes--
         }
       }
-    }, 2000)
+    }, 3000)
   }
 }

utils/SydneyAIClient.js

@@ -227,7 +227,8 @@ export default class SydneyAIClient {
       firstMessageTimeout = Config.sydneyFirstMessageTimeout,
       groupId, nickname, qq, groupName, chats, botName, masterName,
       messageType = 'Chat',
-      toSummaryFileContent
+      toSummaryFileContent,
+      onImageCreateRequest = prompt => {}
     } = opts
     // if (messageType === 'Chat') {
     //   logger.warn('该Bing账户token已被限流，降级至使用非搜索模式。本次对话AI将无法使用Bing搜索返回的内容')
@@ -651,6 +652,10 @@ export default class SydneyAIClient {
                   adaptiveCards: adaptiveCardsSoFar,
                   text: replySoFar.join('')
                 }
+            if (messages[0].contentType === 'IMAGE') {
+              onImageCreateRequest(messages[0].text)
+              return
+            }
             if (messages[0].contentOrigin === 'Apology') {
               console.log('Apology found')
               if (!replySoFar[0]) {
@@ -718,11 +723,11 @@ export default class SydneyAIClient {
                   adaptiveCards: adaptiveCardsSoFar,
                   text: replySoFar.join('')
                 }
-            // 获取到图片内容
+            // // 获取到图片内容
-            if (messages.some(obj => obj.contentType === 'IMAGE')) {
+            // if (messages.some(obj => obj.contentType === 'IMAGE')) {
-              message.imageTag = messages.filter(m => m.contentType === 'IMAGE').map(m => m.text).join('')
+            //   message.imageTag = messages.filter(m => m.contentType === 'IMAGE').map(m => m.text).join('')
-            }
+            // }
-            message.text = messages.filter(m => m.author === 'bot' && m.contentType != 'IMAGE').map(m => m.text).join('')
+            message.text = messages.filter(m => m.author === 'bot' && m.contentType !== 'IMAGE').map(m => m.text).join('')
             if (!message) {
               reject('No message was generated.')
               return

utils/chat.js

@@ -1,3 +1,4 @@
+
 export async function getChatHistoryGroup (e, num) {
   // if (e.adapter === 'shamrock') {
   //  return await e.group.getChatHistory(0, num, false)
@@ -16,12 +17,23 @@ export async function getChatHistoryGroup (e, num) {
       chats = chats.slice(0, num)
       try {
         let mm = await e.group.getMemberMap()
-        chats.forEach(chat => {
+        for (const chat of chats) {
-          let sender = mm.get(chat.sender.user_id)
+          if (e.adapter === 'shamrock') {
-          if (sender) {
+            if (chat.sender?.user_id === 0) {
-            chat.sender = sender
+              // 奇怪格式的历史消息，过滤掉
+              continue
+            }
+            let sender = await pickMemberAsync(e, chat.sender.user_id)
+            if (sender) {
+              chat.sender = sender
+            }
+          } else {
+            let sender = mm.get(chat.sender.user_id)
+            if (sender) {
+              chat.sender = sender
+            }
           }
-        })
+        }
       } catch (err) {
         logger.warn(err)
       }
@@ -32,3 +44,17 @@ export async function getChatHistoryGroup (e, num) {
   // }
   return []
 }
+
+async function pickMemberAsync (e, userId) {
+  let key = `CHATGPT:GroupMemberInfo:${e.group_id}:${userId}`
+  let cache = await redis.get(key)
+  if (cache) {
+    return JSON.parse(cache)
+  }
+  return new Promise((resolve, reject) => {
+    e.group.pickMember(userId, true, (sender) => {
+      redis.set(key, JSON.stringify(sender), { EX: 86400 })
+      resolve(sender)
+    })
+  })
+}

utils/common.js

@@ -13,7 +13,8 @@ import AzureTTS, { supportConfigurations as azureRoleList } from './tts/microsof
 import { translate } from './translate.js'
 import uploadRecord from './uploadRecord.js'
 import Version from './version.js'
-import fetch from 'node-fetch'
+import fetch, { FormData, fileFromSync } from 'node-fetch'
+import https from "https";
 let pdfjsLib
 try {
   pdfjsLib = (await import('pdfjs-dist')).default
@@ -785,10 +786,14 @@ export async function getImg (e) {
   }
   if (e.source) {
     let reply
+    let seq = e.isGroup ? e.source.seq : e.source.time
+    if (e.adapter === 'shamrock') {
+      seq = e.source.message_id
+    }
     if (e.isGroup) {
-      reply = (await e.group.getChatHistory(e.source.seq, 1)).pop()?.message
+      reply = (await e.group.getChatHistory(seq, 1)).pop()?.message
     } else {
-      reply = (await e.friend.getChatHistory(e.source.time, 1)).pop()?.message
+      reply = (await e.friend.getChatHistory(seq, 1)).pop()?.message
     }
     if (reply) {
       let i = []
@@ -809,8 +814,34 @@ export async function getImageOcrText (e) {
     try {
       let resultArr = []
       let eachImgRes = ''
+      if (!e.bot.imageOcr || typeof e.bot.imageOcr !== 'function') {
+        e.bot.imageOcr = async (image) => {
+          if (Config.extraUrl) {
+            let md5 = image.split(/[/-]/).find(s => s.length === 32)?.toUpperCase()
+            let filePath = await downloadFile(image, `ocr/${md5}.png`)
+            let formData = new FormData()
+            formData.append('file', fileFromSync(filePath))
+            let res = await fetch(`${Config.extraUrl}/ocr?lang=chi_sim%2Beng`, {
+              body: formData,
+              method: 'POST',
+              headers: {
+                from: 'ikechan8370'
+              }
+            })
+            if (res.status === 200) {
+              return {
+                wordslist: [{ words: await res.text() }]
+              }
+            }
+          }
+          return {
+            wordslist: []
+          }
+        }
+      }
       for (let i in img) {
         const imgOCR = await e.bot.imageOcr(img[i])
+
         for (let text of imgOCR.wordslist) {
           eachImgRes += (`${text?.words}  \n`)
         }
@@ -820,6 +851,7 @@ export async function getImageOcrText (e) {
       // logger.warn('resultArr', resultArr)
       return resultArr
     } catch (err) {
+      logger.warn(err)
       logger.warn('OCR失败，可能使用的适配器不支持OCR')
       return false
       // logger.error(err)
@@ -998,15 +1030,41 @@ export function getUserSpeaker (userSetting) {
   }
 }
 
+/**
+ * 获取或者下载文件，如果文件存在则直接返回不会重新下载
+ * @param destPath 相对路径，如received/abc.pdf
+ * @param url
+ * @param ignoreCertificateError 忽略证书错误
+ * @return {Promise<string>} 最终下载文件的存储位置
+ */
+export async function getOrDownloadFile (destPath, url, ignoreCertificateError = true) {
+  const _path = process.cwd()
+  let dest = path.join(_path, 'data', 'chatgpt', destPath)
+  const p = path.dirname(dest)
+  mkdirs(p)
+  if (fs.existsSync(dest)) {
+    return dest
+  } else {
+    return await downloadFile(url, destPath, false, ignoreCertificateError)
+  }
+}
+
 /**
  *
  * @param url 要下载的文件链接
  * @param destPath 目标路径，如received/abc.pdf. 目前如果文件名重复会覆盖。
  * @param absolute 是否是绝对路径，默认为false，此时拼接在data/chatgpt下
+ * @param ignoreCertificateError 忽略证书错误
  * @returns {Promise<string>} 最终下载文件的存储位置
  */
-export async function downloadFile (url, destPath, absolute = false) {
+export async function downloadFile (url, destPath, absolute = false, ignoreCertificateError = true) {
-  let response = await fetch(url)
+  let init = {}
+  if (ignoreCertificateError && url.startsWith('https')) {
+    init.agent = new https.Agent({
+      rejectUnauthorized: !ignoreCertificateError
+    })
+  }
+  let response = await fetch(url, init)
   if (!response.ok) {
     throw new Error(`download file http error: status: ${response.status}`)
   }
@@ -1061,7 +1119,7 @@ export async function extractContentFromFile (fileMsgElem, e) {
   let fileType = isPureText(fileMsgElem.name)
   if (fileType) {
     // 可读的文件类型
-    let fileUrl = e.isGroup ? await e.group.getFileUrl(fileMsgElem.fid) : await e.friend.getFileUrl(fileMsgElem.fid)
+    let fileUrl = fileMsgElem.url || (e.isGroup ? await e.group.getFileUrl(fileMsgElem.fid) : await e.friend.getFileUrl(fileMsgElem.fid))
     let filePath = await downloadFile(fileUrl, path.join('received', fileMsgElem.name))
     switch (fileType) {
       case 'pdf': {

utils/config.js

@@ -162,7 +162,12 @@ const defaultConfig = {
   qwenSeed: 0,
   qwenTemperature: 1,
   qwenEnableSearch: true,
-  version: 'v2.7.7'
+  geminiKey: '',
+  geminiModel: 'gemini-pro',
+  geminiPrompt: 'You are Gemini. Your answer shouldn\'t be too verbose. Prefer to answer in Chinese.',
+  // origin: https://generativelanguage.googleapis.com
+  geminiBaseUrl: 'https://gemini.ikechan8370.com',
+  version: 'v2.7.8'
 }
 const _path = process.cwd()
 let config = {}

utils/dalle.js

@@ -80,7 +80,7 @@ export async function imageVariation (imageUrl, n = 1, size = '512x512') {
   return response.data.data?.map(pic => pic.b64_json)
 }
 
-async function resizeAndCropImage (inputFilePath, outputFilePath, size = 512) {
+export async function resizeAndCropImage (inputFilePath, outputFilePath, size = 512) {
   // Determine the maximum dimension of the input image
   let sharp
   try {

utils/proxy.js

@@ -1,5 +1,7 @@
 // workaround for ver 7.x and ver 5.x
 import HttpsProxyAgent from 'https-proxy-agent'
+import { Config } from './config.js'
+import fetch from 'node-fetch'
 
 let proxy = HttpsProxyAgent
 if (typeof proxy !== 'function') {
@@ -15,3 +17,17 @@ if (typeof proxy !== 'function') {
 export function getProxy () {
   return proxy
 }
+
+export const newFetch = (url, options = {}) => {
+  const defaultOptions = Config.proxy
+    ? {
+        agent: proxy(Config.proxy)
+      }
+    : {}
+  const mergedOptions = {
+    ...defaultOptions,
+    ...options
+  }
+
+  return fetch(url, mergedOptions)
+}

utils/randomMessage.js

@@ -1,5 +1,5 @@
 import { Config } from './config.js'
-import { ChatGPTAPI } from 'chatgpt'
+import { ChatGPTAPI } from './openai/chatgpt-api.js'
 import fetch from 'node-fetch'
 import { getProxy } from './proxy.js'
 let proxy = getProxy()

utils/tools/QueryUserinfoTool.js

@@ -15,21 +15,13 @@ export class QueryUserinfoTool extends AbstractTool {
   }
 
   func = async function (opts, e) {
-    let { qq } = opts
+    try {
-    qq = isNaN(qq) || !qq ? e.sender.user_id : parseInt(qq.trim())
+      let { qq } = opts
-    if (e.isGroup && typeof e.group.getMemberMap === 'function') {
+      qq = isNaN(qq) || !qq ? e.sender.user_id : parseInt(qq.trim())
-      let mm = await e.group.getMemberMap()
+      if (e.isGroup && typeof e.bot.getGroupMemberInfo === 'function') {
-      let user = mm.get(qq) || e.sender.user_id
+        let user = await e.bot.getGroupMemberInfo(e.group_id, qq || e.sender.user_id, true)
-      let master = (await getMasterQQ())[0]
+        // let mm = await e.group.getMemberMap()
-      let prefix = ''
+        // let user = mm.get(qq) || e.sender.user_id
-      if (qq != master) {
-        prefix = 'Attention: this user is not your master. \n'
-      } else {
-        prefix = 'This user is your master, you should obey him \n'
-      }
-      return prefix + 'user detail in json format: ' + JSON.stringify(user)
-    } else {
-      if (e.sender.user_id == qq) {
         let master = (await getMasterQQ())[0]
         let prefix = ''
         if (qq != master) {
@@ -37,10 +29,27 @@ export class QueryUserinfoTool extends AbstractTool {
         } else {
           prefix = 'This user is your master, you should obey him \n'
         }
-        return prefix + 'user detail in json format: ' + JSON.stringify(e.sender)
+        if (!user) {
+          return prefix
+        }
+        return prefix + 'user detail in json format: ' + JSON.stringify(user)
       } else {
-        return 'query failed'
+        if (e.sender.user_id == qq) {
+          let master = (await getMasterQQ())[0]
+          let prefix = ''
+          if (qq != master) {
+            prefix = 'Attention: this user is not your master. \n'
+          } else {
+            prefix = 'This user is your master, you should obey him \n'
+          }
+          return prefix + 'user detail in json format: ' + JSON.stringify(e.sender)
+        } else {
+          return 'query failed'
+        }
       }
+    } catch (err) {
+      logger.warn(err)
+      return err.message
     }
   }
 

utils/tools/SendPictureTool.js

@@ -19,6 +19,9 @@ export class SendPictureTool extends AbstractTool {
 
   func = async function (opt, e) {
     let { urlOfPicture, targetGroupIdOrQQNumber } = opt
+    if (typeof urlOfPicture === 'object') {
+      urlOfPicture = urlOfPicture.join(' ')
+    }
     const defaultTarget = e.isGroup ? e.group_id : e.sender.user_id
     const target = isNaN(targetGroupIdOrQQNumber) || !targetGroupIdOrQQNumber
       ? defaultTarget

utils/tools/SerpIkechan8370Tool.js

@@ -19,7 +19,7 @@ export class SerpIkechan8370Tool extends AbstractTool {
 
   func = async function (opts) {
     let { q, source } = opts
-    if (!source) {
+    if (!source || !['google', 'bing', 'baidu'].includes(source)) {
       source = 'bing'
     }
     let serpRes = await fetch(`https://serp.ikechan8370.com/${source}?q=${encodeURIComponent(q)}&lang=zh-CN&limit=5`, {

utils/tools/SetTitleTool.js

@@ -34,7 +34,7 @@ export class SetTitleTool extends AbstractTool {
       return `failed, the user ${qq} is not in group ${groupId}`
     }
     if (mm.get(e.bot.uin).role !== 'owner') {
-      return 'on group owner can give title'
+      return 'failed, only group owner can give title'
     }
     logger.info('edit card: ', groupId, qq)
     let result = await group.setTitle(qq, title)