From 34bba041e5ba3ef5ffbf170df195f2061c6a46c5 Mon Sep 17 00:00:00 2001
From: ikechan8370 <geyinchi@buaa.edu.cn>
Date: Sat, 24 Jun 2023 23:24:14 +0800
Subject: [PATCH] fix: add a image caption tool

---
 apps/chat.js                    | 29 ++++++++++++++++++-
 utils/config.js                 |  1 +
 utils/tools/ImageCaptionTool.js | 51 +++++++++++++++++++++++++++++++++
 3 files changed, 80 insertions(+), 1 deletion(-)
 create mode 100644 utils/tools/ImageCaptionTool.js

diff --git a/apps/chat.js b/apps/chat.js
index fa316e6..38fa4c3 100644
--- a/apps/chat.js
+++ b/apps/chat.js
@@ -60,6 +60,7 @@ import { SerpTool } from '../utils/tools/SerpTool.js'
 import { SerpIkechan8370Tool } from '../utils/tools/SerpIkechan8370Tool.js'
 import {SendPictureTool} from "../utils/tools/SendPictureTool.js";
 import {SerpImageTool} from "../utils/tools/SearchImageTool.js";
+import {ImageCaptionTool} from "../utils/tools/ImageCaptionTool.js";
 try {
   await import('emoji-strip')
 } catch (err) {
@@ -1957,7 +1958,7 @@ export class chatgpt extends plugin {
           new SendVideoTool(),
           new SearchMusicTool(),
           new SendMusicTool(),
-          new SendAvatarTool(),
+          // new SendAvatarTool(),
           // new SendDiceTool(),
           new EditCardTool(),
           new QueryStarRailTool(),
@@ -1967,8 +1968,33 @@ export class chatgpt extends plugin {
           new WeatherTool(),
           new SendPictureTool(),
           new SerpImageTool(),
+          new ImageCaptionTool(),
           serpTool
         ]
+        let img = []
+        if (e.source) {
+          // 优先从回复找图
+          let reply
+          if (e.isGroup) {
+            reply = (await e.group.getChatHistory(e.source.seq, 1)).pop()?.message
+          } else {
+            reply = (await e.friend.getChatHistory(e.source.time, 1)).pop()?.message
+          }
+          if (reply) {
+            for (let val of reply) {
+              if (val.type === 'image') {
+                console.log(val)
+                img.push(val.url)
+              }
+            }
+          }
+        }
+        if (e.img) {
+          img.push(...e.img)
+        }
+        if (img.length > 0) {
+          prompt += `\nthe url of the picture(s) above: ${img.join(', ')}`
+        }
         // if (e.sender.role === 'admin' || e.sender.role === 'owner') {
         //   tools.push(...[new JinyanTool(), new KickOutTool()])
         // }
@@ -2004,6 +2030,7 @@ export class chatgpt extends plugin {
             await e.reply('字数超限啦，将为您自动结束本次对话。')
             return null
           } else {
+            logger.error(err)
             throw new Error(err)
           }
         }
diff --git a/utils/config.js b/utils/config.js
index 455abfb..92044fa 100644
--- a/utils/config.js
+++ b/utils/config.js
@@ -128,6 +128,7 @@ const defaultConfig = {
   amapKey: '',
   azSerpKey: '',
   serpSource: 'ikechan8370',
+  extraUrl: '',
   version: 'v2.7.0'
 }
 const _path = process.cwd()
diff --git a/utils/tools/ImageCaptionTool.js b/utils/tools/ImageCaptionTool.js
new file mode 100644
index 0000000..85a8637
--- /dev/null
+++ b/utils/tools/ImageCaptionTool.js
@@ -0,0 +1,51 @@
+import { AbstractTool } from './AbstractTool.js'
+import fetch, { File, FormData } from 'node-fetch'
+import { Config } from '../config.js'
+import fs from 'fs'
+import crypto from 'crypto'
+export class ImageCaptionTool extends AbstractTool {
+  name = 'imageCaption'
+
+  parameters = {
+    properties: {
+      imgUrl: {
+        type: 'string',
+        description: 'the url of the image.'
+      },
+      qq: {
+        type: 'string',
+        description: 'if the picture is an avatar of a user, just give his qq number'
+      }
+    },
+    required: []
+  }
+
+  description = 'useful when you want to know what is inside a picture, such as user\'s avatar'
+
+  func = async function (opts) {
+    let { imgUrl, qq } = opts
+    if (qq) {
+      imgUrl = `https://q1.qlogo.cn/g?b=qq&s=160&nk=${qq}`
+    }
+    if (!imgUrl) {
+      return 'you must give at least one parameter of imgUrl and qq'
+    }
+    const imageResponse = await fetch(imgUrl)
+    const blob = await imageResponse.blob()
+    const arrayBuffer = await blob.arrayBuffer()
+    const buffer = Buffer.from(arrayBuffer)
+    // await fs.writeFileSync(`data/chatgpt/${crypto.randomUUID()}`, buffer)
+    let formData = new FormData()
+    formData.append('file', new File([buffer], 'file.png', { type: 'image/png' }))
+    let captionRes = await fetch(`${Config.extraUrl}/image-captioning`, {
+      method: 'POST',
+      body: formData
+    })
+    if (captionRes.status === 200) {
+      let result = await captionRes.text()
+      return `the content of this picture is: ${result}`
+    } else {
+      return 'error happened'
+    }
+  }
+}