diff --git a/apps/chat.js b/apps/chat.js index 813f61b..87a0dd0 100644 --- a/apps/chat.js +++ b/apps/chat.js @@ -10,27 +10,28 @@ import AzureTTS from '../utils/tts/microsoft-azure.js' import VoiceVoxTTS from '../utils/tts/voicevox.js' import Version from '../utils/version.js' import { - completeJSON, - extractContentFromFile, - formatDate, - formatDate2, - generateAudio, - getDefaultReplySetting, - getImageOcrText, - getImg, - getMasterQQ, - getMaxModelTokens, - getMessageById, - getUin, - getUserData, - getUserReplySetting, - isCN, - isImage, - makeForwardMsg, - randomString, - render, - renderUrl, - upsertMessage + completeJSON, + extractContentFromFile, + formatDate, + formatDate2, + generateAudio, + getDefaultReplySetting, + getImageOcrText, + getImg, + getMasterQQ, + getMaxModelTokens, + getMessageById, + getOrDownloadFile, + getUin, + getUserData, + getUserReplySetting, + isCN, + isImage, + makeForwardMsg, + randomString, + render, + renderUrl, + upsertMessage } from '../utils/common.js' import { ChatGPTPuppeteer } from '../utils/browser.js' import { KeyvFile } from 'keyv-file' @@ -78,6 +79,8 @@ import { getProxy } from '../utils/proxy.js' import { QwenApi } from '../utils/alibaba/qwen-api.js' import { getChatHistoryGroup } from '../utils/chat.js' import { CustomGoogleGeminiClient } from '../client/CustomGoogleGeminiClient.js' +import { resizeAndCropImage } from '../utils/dalle.js' +import fs from 'fs' const roleMap = { owner: 'group owner', @@ -1522,13 +1525,17 @@ export class chatgpt extends plugin { } const userData = await getUserData(e.user_id) const useCast = userData.cast || {} - switch (use) { - case 'browser': { + if (use === 'browser') { + { return await this.chatgptBrowserBased(prompt, conversation) } - case 'bing': { + } else if (use === 'bing') { + { let throttledTokens = [] - let { bingToken, allThrottled } = await getAvailableBingToken(conversation, throttledTokens) + let { + bingToken, + allThrottled + } = await getAvailableBingToken(conversation, throttledTokens) let cookies if (bingToken?.indexOf('=') > -1) { cookies = bingToken @@ -1712,45 +1719,44 @@ export class chatgpt extends plugin { } else { retry = 0 } - } else - if (message && typeof message === 'string' && message.indexOf('限流') > -1) { - throttledTokens.push(bingToken) - let bingTokens = JSON.parse(await redis.get('CHATGPT:BING_TOKENS')) - const badBingToken = bingTokens.findIndex(element => element.Token === bingToken) - const now = new Date() - const hours = now.getHours() - now.setHours(hours + 6) - bingTokens[badBingToken].State = '受限' - bingTokens[badBingToken].DisactivationTime = now - await redis.set('CHATGPT:BING_TOKENS', JSON.stringify(bingTokens)) + } else if (message && typeof message === 'string' && message.indexOf('限流') > -1) { + throttledTokens.push(bingToken) + let bingTokens = JSON.parse(await redis.get('CHATGPT:BING_TOKENS')) + const badBingToken = bingTokens.findIndex(element => element.Token === bingToken) + const now = new Date() + const hours = now.getHours() + now.setHours(hours + 6) + bingTokens[badBingToken].State = '受限' + bingTokens[badBingToken].DisactivationTime = now + await redis.set('CHATGPT:BING_TOKENS', JSON.stringify(bingTokens)) // 不减次数 - } else if (message && typeof message === 'string' && message.indexOf('UnauthorizedRequest') > -1) { + } else if (message && typeof message === 'string' && message.indexOf('UnauthorizedRequest') > -1) { // token过期了 - let bingTokens = JSON.parse(await redis.get('CHATGPT:BING_TOKENS')) - const badBingToken = bingTokens.findIndex(element => element.Token === bingToken) - if (badBingToken > 0) { - // 可能是微软抽风，给三次机会 - if (bingTokens[badBingToken]?.exception) { - if (bingTokens[badBingToken].exception <= 3) { - bingTokens[badBingToken].exception += 1 - } else { - bingTokens[badBingToken].exception = 0 - bingTokens[badBingToken].State = '过期' - } + let bingTokens = JSON.parse(await redis.get('CHATGPT:BING_TOKENS')) + const badBingToken = bingTokens.findIndex(element => element.Token === bingToken) + if (badBingToken > 0) { + // 可能是微软抽风，给三次机会 + if (bingTokens[badBingToken]?.exception) { + if (bingTokens[badBingToken].exception <= 3) { + bingTokens[badBingToken].exception += 1 } else { - bingTokens[badBingToken].exception = 1 + bingTokens[badBingToken].exception = 0 + bingTokens[badBingToken].State = '过期' } - await redis.set('CHATGPT:BING_TOKENS', JSON.stringify(bingTokens)) } else { - retry = retry - 1 + bingTokens[badBingToken].exception = 1 } - errorMessage = 'UnauthorizedRequest：必应token不正确或已过期' + await redis.set('CHATGPT:BING_TOKENS', JSON.stringify(bingTokens)) + } else { + retry = retry - 1 + } + errorMessage = 'UnauthorizedRequest：必应token不正确或已过期' // logger.warn(`token${bingToken}疑似不存在或已过期，再试试`) // retry = retry - 1 - } else { - retry-- - errorMessage = message === 'Timed out waiting for response. Try enabling debug mode to see more information.' ? (reply ? `${reply}\n不行了，我的大脑过载了，处理不过来了!` : '必应的小脑瓜不好使了，不知道怎么回答！') : message - } + } else { + retry-- + errorMessage = message === 'Timed out waiting for response. Try enabling debug mode to see more information.' ? (reply ? `${reply}\n不行了，我的大脑过载了，处理不过来了!` : '必应的小脑瓜不好使了，不知道怎么回答！') : message + } } } while (retry > 0) if (errorMessage) { @@ -1785,7 +1791,8 @@ export class chatgpt extends plugin { } } } - case 'api3': { + } else if (use === 'api3') { + { // official without cloudflare let accessToken = await redis.get('CHATGPT:TOKEN') if (!accessToken) { @@ -1809,7 +1816,8 @@ export class chatgpt extends plugin { } return sendMessageResult } - case 'chatglm': { + } else if (use === 'chatglm') { + { const cacheOptions = { namespace: 'chatglm_6b', store: new KeyvFile({ filename: 'cache.json' }) @@ -1821,7 +1829,8 @@ export class chatgpt extends plugin { let sendMessageResult = await this.chatGPTApi.sendMessage(prompt, conversation) return sendMessageResult } - case 'poe': { + } else if (use === 'poe') { + { const cookie = await redis.get('CHATGPT:POE_TOKEN') if (!cookie) { throw new Error('未绑定Poe Cookie，请使用#chatgpt设置Poe token命令绑定cookie') @@ -1839,7 +1848,8 @@ export class chatgpt extends plugin { text: response.data } } - case 'claude': { + } else if (use === 'claude') { + { let client = new SlackClaudeClient({ slackUserToken: Config.slackUserToken, slackChannelId: Config.slackChannelId @@ -1863,7 +1873,8 @@ export class chatgpt extends plugin { text } } - case 'claude2': { + } else if (use === 'claude2') { + { let { conversationId } = conversation let client = new ClaudeAIClient({ organizationId: Config.claudeAIOrganizationId, @@ -1903,7 +1914,8 @@ export class chatgpt extends plugin { return await client.sendMessage(prompt, conv.uuid, attachments) } } - case 'xh': { + } else if (use === 'xh') { + { const cacheOptions = { namespace: 'xh', store: new KeyvFile({ filename: 'cache.json' }) @@ -1926,7 +1938,8 @@ export class chatgpt extends plugin { }) return response } - case 'azure': { + } else if (use === 'azure') { + { let azureModel try { azureModel = await import('@azure/openai') @@ -1936,15 +1949,22 @@ export class chatgpt extends plugin { let OpenAIClient = azureModel.OpenAIClient let AzureKeyCredential = azureModel.AzureKeyCredential let msg = conversation.messages - let content = { role: 'user', content: prompt } + let content = { + role: 'user', + content: prompt + } msg.push(content) const client = new OpenAIClient(Config.azureUrl, new AzureKeyCredential(Config.azApiKey)) const deploymentName = Config.azureDeploymentName const { choices } = await client.getChatCompletions(deploymentName, msg) let completion = choices[0].message - return { text: completion.content, message: completion } + return { + text: completion.content, + message: completion + } } - case 'qwen': { + } else if (use === 'qwen') { + { let completionParams = { parameters: { top_p: Config.qwenTopP || 0.5, @@ -1958,12 +1978,15 @@ export class chatgpt extends plugin { completionParams.model = Config.qwenModel } const currentDate = new Date().toISOString().split('T')[0] - async function um (message) { + + async function um(message) { return await upsertMessage(message, 'QWEN') } - async function gm (id) { + + async function gm(id) { return await getMessageById(id, 'QWEN') } + let opts = { apiKey: Config.qwenApiKey, debug: false, @@ -1995,7 +2018,8 @@ export class chatgpt extends plugin { } return msg } - case 'bard': { + } else if (use === 'bard') { + { // 处理cookie const matchesPSID = /__Secure-1PSID=([^;]+)/.exec(Config.bardPsid) const matchesPSIDTS = /__Secure-1PSIDTS=([^;]+)/.exec(Config.bardPsid) @@ -2025,13 +2049,13 @@ export class chatgpt extends plugin { bardURL: Config.bardForceUseReverse ? Config.bardReverseProxy : 'https://bard.google.com' }) let chat = await bot.createChat(conversation?.conversationId - ? { + ? { conversationID: conversation.conversationId, responseID: conversation.parentMessageId, choiceID: conversation.clientId, _reqID: conversation.invocationId } - : {}) + : {}) let response = await chat.ask(prompt, { image: imageBuff, format: Bard.JSON @@ -2045,7 +2069,8 @@ export class chatgpt extends plugin { images: response.images } } - case 'gemini': { + } else if (use === 'gemini') { + { let client = new CustomGoogleGeminiClient({ e, userId: e.sender.user_id, @@ -2054,6 +2079,28 @@ export class chatgpt extends plugin { baseUrl: Config.geminiBaseUrl, debug: Config.debug }) + let option = { + stream: false, + onProgress: (data) => { + if (Config.debug) { + logger.info(data) + } + }, + parentMessageId: conversation.parentMessageId, + conversationId: conversation.conversationId + } + if (Config.geminiModel.includes('vision')) { + const image = await getImg(e) + let imageUrl = image ? image[0] : undefined + if (imageUrl) { + let md5 = imageUrl.split(/[/-]/).find(s => s.length === 32)?.toUpperCase() + let imageLoc = await getOrDownloadFile(`ocr/${md5}.png`, imageUrl) + let outputLoc = imageLoc.replace(`${md5}.png`, `${md5}_512.png`) + await resizeAndCropImage(imageLoc, outputLoc, 512) + let buffer = fs.readFileSync(outputLoc) + option.image = buffer.toString('base64') + } + } if (Config.smartMode) { /** * @type {AbstractTool[]} @@ -2063,20 +2110,20 @@ export class chatgpt extends plugin { new WebsiteTool(), new SendPictureTool(), new SendVideoTool(), - new ImageCaptionTool(), + // new ImageCaptionTool(), new SearchVideoTool(), new SendAvatarTool(), new SerpImageTool(), new SearchMusicTool(), new SendMusicTool(), - new SerpIkechan8370Tool(), - new SerpTool(), + // new SerpIkechan8370Tool(), + // new SerpTool(), new SendAudioMessageTool(), - new ProcessPictureTool(), + // new ProcessPictureTool(), new APTool(), - new HandleMessageMsgTool(), + // new HandleMessageMsgTool(), new SendMessageToSpecificGroupOrUserTool(), - new SendDiceTool(), + // new SendDiceTool(), new QueryGenshinTool() ] if (Config.amapKey) { @@ -2129,27 +2176,18 @@ export class chatgpt extends plugin { if (chats) { system += 'There is the conversation history in the group, you must chat according to the conversation history context"' system += chats - .map(chat => { - let sender = chat.sender || {} - return `【${sender.card || sender.nickname}】(qq：${sender.user_id}, ${roleMap[sender.role] || 'normal user'}，${sender.area ? 'from ' + sender.area + ', ' : ''} ${sender.age} years old, 群头衔：${sender.title}, gender: ${sender.sex}, time：${formatDate(new Date(chat.time * 1000))}, messageId: ${chat.message_id}) 说：${chat.raw_message}` - }) - .join('\n') + .map(chat => { + let sender = chat.sender || {} + return `【${sender.card || sender.nickname}】(qq：${sender.user_id}, ${roleMap[sender.role] || 'normal user'}，${sender.area ? 'from ' + sender.area + ', ' : ''} ${sender.age} years old, 群头衔：${sender.title}, gender: ${sender.sex}, time：${formatDate(new Date(chat.time * 1000))}, messageId: ${chat.message_id}) 说：${chat.raw_message}` + }) + .join('\n') } } - let option = { - stream: false, - onProgress: (data) => { - if (Config.debug) { - logger.info(data) - } - }, - parentMessageId: conversation.parentMessageId, - conversationId: conversation.conversationId, - system - } + option.system = system return await client.sendMessage(prompt, option) } - default: { + } else { + { // openai api let completionParams = {} if (Config.model) { @@ -2181,7 +2219,7 @@ export class chatgpt extends plugin { const defaultBotName = 'ChatGPT' const groupContextTip = Config.groupContextTip system = system.replaceAll(namePlaceholder, opt.botName || defaultBotName) + - ((Config.enableGroupContext && opt.groupId) ? groupContextTip : '') + ((Config.enableGroupContext && opt.groupId) ? groupContextTip : '') system += 'Attention, you are currently chatting in a qq group, then one who asks you now is' + `${opt.nickname}(${opt.qq})。` system += `the group name is ${opt.groupName}, group id is ${opt.groupId}。` if (opt.botName) { @@ -2190,16 +2228,16 @@ export class chatgpt extends plugin { if (chats) { system += 'There is the conversation history in the group, you must chat according to the conversation history context"' system += chats - .map(chat => { - let sender = chat.sender || {} - // if (sender.user_id === e.bot.uin && chat.raw_message.startsWith('建议的回复')) { - if (chat.raw_message.startsWith('建议的回复')) { - // 建议的回复太容易污染设定导致对话太固定跑偏了 - return '' - } - return `【${sender.card || sender.nickname}】(qq：${sender.user_id}, ${roleMap[sender.role] || 'normal user'}，${sender.area ? 'from ' + sender.area + ', ' : ''} ${sender.age} years old, 群头衔：${sender.title}, gender: ${sender.sex}, time：${formatDate(new Date(chat.time * 1000))}, messageId: ${chat.message_id}) 说：${chat.raw_message}` - }) - .join('\n') + .map(chat => { + let sender = chat.sender || {} + // if (sender.user_id === e.bot.uin && chat.raw_message.startsWith('建议的回复')) { + if (chat.raw_message.startsWith('建议的回复')) { + // 建议的回复太容易污染设定导致对话太固定跑偏了 + return '' + } + return `【${sender.card || sender.nickname}】(qq：${sender.user_id}, ${roleMap[sender.role] || 'normal user'}，${sender.area ? 'from ' + sender.area + ', ' : ''} ${sender.age} years old, 群头衔：${sender.title}, gender: ${sender.sex}, time：${formatDate(new Date(chat.time * 1000))}, messageId: ${chat.message_id}) 说：${chat.raw_message}` + }) + .join('\n') } } catch (err) { if (e.isGroup) { @@ -2372,7 +2410,10 @@ export class chatgpt extends plugin { if (msg.text) { await e.reply(msg.text.replace('\n\n\n', '\n')) } - let { name, arguments: args } = msg.functionCall + let { + name, + arguments: args + } = msg.functionCall args = JSON.parse(args) // 感觉换成targetGroupIdOrUserQQNumber这种表意比较清楚的变量名，效果会好一丢丢 if (!args.groupId) { @@ -2383,7 +2424,10 @@ export class chatgpt extends plugin { } catch (err) { args.groupId = e.group_id + '' || e.sender.user_id + '' } - let functionResult = await fullFuncMap[name.trim()].exec(Object.assign({ isAdmin, sender }, args), e) + let functionResult = await fullFuncMap[name.trim()].exec(Object.assign({ + isAdmin, + sender + }, args), e) logger.mark(`function ${name} execution result: ${functionResult}`) option.parentMessageId = msg.id option.name = name diff --git a/client/CustomGoogleGeminiClient.js b/client/CustomGoogleGeminiClient.js index 4ac4aed..7953764 100644 --- a/client/CustomGoogleGeminiClient.js +++ b/client/CustomGoogleGeminiClient.js @@ -68,7 +68,7 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient { /** * * @param text - * @param {{conversationId: string?, parentMessageId: string?, stream: boolean?, onProgress: function?, functionResponse: FunctionResponse?, system: string?}} opt + * @param {{conversationId: string?, parentMessageId: string?, stream: boolean?, onProgress: function?, functionResponse: FunctionResponse?, system: string?, image: string?}} opt * @returns {Promise<{conversationId: string?, parentMessageId: string, text: string, id: string}>} */ async sendMessage (text, opt) { @@ -111,8 +111,16 @@ export class CustomGoogleGeminiClient extends GoogleGeminiClient { id: idThis, parentMessageId: opt.parentMessageId || undefined } + if (opt.image) { + thisMessage.parts.push({ + inline_data: { + mime_type: 'image/jpeg', + data: opt.image + } + }) + } history.push(_.cloneDeep(thisMessage)) - let url = `${this.baseUrl}/v1beta/models/gemini-pro:generateContent?key=${this._key}` + let url = `${this.baseUrl}/v1beta/models/${this.model}:generateContent?key=${this._key}` let body = { // 不去兼容官方的简单格式了，直接用，免得function还要转换 /** diff --git a/utils/common.js b/utils/common.js index cfe04f3..3a00052 100644 --- a/utils/common.js +++ b/utils/common.js @@ -1030,6 +1030,25 @@ export function getUserSpeaker (userSetting) { } } +/** + * 获取或者下载文件，如果文件存在则直接返回不会重新下载 + * @param destPath 相对路径，如received/abc.pdf + * @param url + * @param ignoreCertificateError 忽略证书错误 + * @return {Promise} 最终下载文件的存储位置 + */ +export async function getOrDownloadFile (destPath, url, ignoreCertificateError = true) { + const _path = process.cwd() + let dest = path.join(_path, 'data', 'chatgpt', destPath) + const p = path.dirname(dest) + mkdirs(p) + if (fs.existsSync(dest)) { + return dest + } else { + return await downloadFile(url, destPath, false, ignoreCertificateError) + } +} + /** * * @param url 要下载的文件链接 diff --git a/utils/dalle.js b/utils/dalle.js index 4c17371..11bd368 100644 --- a/utils/dalle.js +++ b/utils/dalle.js @@ -80,7 +80,7 @@ export async function imageVariation (imageUrl, n = 1, size = '512x512') { return response.data.data?.map(pic => pic.b64_json) } -async function resizeAndCropImage (inputFilePath, outputFilePath, size = 512) { +export async function resizeAndCropImage (inputFilePath, outputFilePath, size = 512) { // Determine the maximum dimension of the input image let sharp try { diff --git a/utils/tools/SendPictureTool.js b/utils/tools/SendPictureTool.js index d3d81cc..96eb2c3 100644 --- a/utils/tools/SendPictureTool.js +++ b/utils/tools/SendPictureTool.js @@ -19,6 +19,9 @@ export class SendPictureTool extends AbstractTool { func = async function (opt, e) { let { urlOfPicture, targetGroupIdOrQQNumber } = opt + if (typeof urlOfPicture === 'object') { + urlOfPicture = urlOfPicture.join(' ') + } const defaultTarget = e.isGroup ? e.group_id : e.sender.user_id const target = isNaN(targetGroupIdOrQQNumber) || !targetGroupIdOrQQNumber ? defaultTarget