feat: 尝试添加silk转码支持以避免缺少amr导致无法使用语音

This commit is contained in:
ikechan8370 2023-04-08 21:39:34 +08:00
parent 6b2cd446f9
commit 93459ed94e
5 changed files with 425 additions and 57 deletions

View file

@ -7,7 +7,7 @@ import { ChatGPTAPI } from 'chatgpt'
import { BingAIClient } from '@waylaidwanderer/chatgpt-api'
import SydneyAIClient from '../utils/SydneyAIClient.js'
import {
render,renderUrl,
render, renderUrl,
getMessageById,
makeForwardMsg,
upsertMessage,
@ -24,6 +24,7 @@ import { deleteConversation, getConversations, getLatestMessageIdByConversationI
import { convertSpeaker, generateAudio, speakers } from '../utils/tts.js'
import ChatGLMClient from '../utils/chatglm.js'
import { convertFaces } from '../utils/face.js'
import uploadRecord from '../utils/uploadRecord.js'
try {
await import('keyv')
} catch (err) {
@ -38,7 +39,13 @@ if (Config.proxy) {
console.warn('未安装https-proxy-agent请在插件目录下执行pnpm add https-proxy-agent')
}
}
let useSilk = false
try {
await import('node-silk')
useSilk = true
} catch (e) {
useSilk = false
}
/**
* 每个对话保留的时长单个对话内ai是保留上下文的超时后销毁对话再次对话创建新的对话
* 单位
@ -827,7 +834,17 @@ export class chatgpt extends plugin {
if (Config.ttsSpace && ttsResponse.length <= Config.ttsAutoFallbackThreshold) {
try {
let wav = await generateAudio(ttsResponse, speaker, '中日混合(中文用[ZH][ZH]包裹起来,日文用[JA][JA]包裹起来)')
await e.reply(segment.record(wav))
if (useSilk) {
try {
let sendable = await uploadRecord(wav)
await e.reply(sendable)
} catch (err) {
logger.error(err)
await e.reply(segment.record(wav))
}
} else {
await e.reply(segment.record(wav))
}
} catch (err) {
await this.reply('合成语音发生错误~')
}
@ -873,7 +890,7 @@ export class chatgpt extends plugin {
await this.reply(`出现错误:${err}`, true, { recallMsg: e.isGroup ? 10 : 0 })
} else {
// 这里是否还需要上传到缓存服务器呐?多半是代理服务器的问题,本地也修不了,应该不用吧。
await this.renderImage(e, use !== 'bing' ? 'content/ChatGPT/index' : 'content/Bing/index', `通信异常,错误信息如下 ${err?.message || err?.data?.message || (typeof(err) === 'object' ? JSON.stringify(err) : err) || '未能确认错误类型!'}`, prompt)
await this.renderImage(e, use !== 'bing' ? 'content/ChatGPT/index' : 'content/Bing/index', `通信异常,错误信息如下 ${err?.message || err?.data?.message || (typeof (err) === 'object' ? JSON.stringify(err) : err) || '未能确认错误类型!'}`, prompt)
}
}
}
@ -971,10 +988,10 @@ export class chatgpt extends plugin {
prompt: new Buffer.from(prompt).toString('base64'),
senderName: e.sender.nickname,
style: Config.toneStyle,
mood: mood,
quote: quote,
mood,
quote,
group: e.isGroup ? e.group.name : '',
suggest: suggest ? suggest.split("\n").filter(Boolean) : [],
suggest: suggest ? suggest.split('\n').filter(Boolean) : [],
images: imgUrls
},
bing: use === 'bing',
@ -989,51 +1006,47 @@ export class chatgpt extends plugin {
if (cacheres.ok) {
cacheData = Object.assign({}, cacheData, await cacheres.json())
}
if (cacheData.error)
await this.reply(`出现错误:${cacheData.error}`, true)
else
await e.reply(await renderUrl(e, viewHost + `page/${cacheData.file}?qr=${Config.showQRCode ? 'true' : 'false'}`, { retType: Config.quoteReply ? 'base64' : '', Viewport: {width: Config.chatViewWidth, height: parseInt(Config.chatViewWidth * 0.56)} }), e.isGroup && Config.quoteReply)
if (cacheData.error) { await this.reply(`出现错误:${cacheData.error}`, true) } else { await e.reply(await renderUrl(e, viewHost + `page/${cacheData.file}?qr=${Config.showQRCode ? 'true' : 'false'}`, { retType: Config.quoteReply ? 'base64' : '', Viewport: { width: Config.chatViewWidth, height: parseInt(Config.chatViewWidth * 0.56) } }), e.isGroup && Config.quoteReply) }
} else {
if (Config.cacheEntry) cacheData.file = randomString()
const cacheresOption = {
method: 'POST',
headers: {
'Content-Type': 'application/json'
if (Config.cacheEntry) cacheData.file = randomString()
const cacheresOption = {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
content: {
content: new Buffer.from(content).toString('base64'),
prompt: new Buffer.from(prompt).toString('base64'),
senderName: e.sender.nickname,
style: Config.toneStyle,
mood,
quote
},
body: JSON.stringify({
content: {
content: new Buffer.from(content).toString('base64'),
prompt: new Buffer.from(prompt).toString('base64'),
senderName: e.sender.nickname,
style: Config.toneStyle,
mood,
quote
},
bing: use === 'bing',
entry: Config.cacheEntry ? cacheData.file : ''
})
}
if (Config.cacheEntry) {
fetch(`${Config.cacheUrl}/cache`, cacheresOption)
} else {
const cacheres = await fetch(`${Config.cacheUrl}/cache`, cacheresOption)
if (cacheres.ok) {
cacheData = Object.assign({}, cacheData, await cacheres.json())
}
}
await e.reply(await render(e, 'chatgpt-plugin', template, {
content: new Buffer.from(content).toString('base64'),
prompt: new Buffer.from(prompt).toString('base64'),
senderName: e.sender.nickname,
quote: quote.length > 0,
quotes: quote,
cache: cacheData,
style: Config.toneStyle,
mood,
version
}, { retType: Config.quoteReply ? 'base64' : '' }), e.isGroup && Config.quoteReply)
bing: use === 'bing',
entry: Config.cacheEntry ? cacheData.file : ''
})
}
if (Config.cacheEntry) {
fetch(`${Config.cacheUrl}/cache`, cacheresOption)
} else {
const cacheres = await fetch(`${Config.cacheUrl}/cache`, cacheresOption)
if (cacheres.ok) {
cacheData = Object.assign({}, cacheData, await cacheres.json())
}
}
await e.reply(await render(e, 'chatgpt-plugin', template, {
content: new Buffer.from(content).toString('base64'),
prompt: new Buffer.from(prompt).toString('base64'),
senderName: e.sender.nickname,
quote: quote.length > 0,
quotes: quote,
cache: cacheData,
style: Config.toneStyle,
mood,
version
}, { retType: Config.quoteReply ? 'base64' : '' }), e.isGroup && Config.quoteReply)
}
}
async sendMessage (prompt, conversation = {}, use, e) {

View file

@ -6,6 +6,15 @@ import fs from 'fs'
import { emojiRegex, googleRequestUrl } from '../utils/emoj/index.js'
import fetch from 'node-fetch'
import { mkdirs } from '../utils/common.js'
import uploadRecord from "../utils/uploadRecord.js";
let useSilk = false
try {
await import('node-silk')
useSilk = true
} catch (e) {
useSilk = false
}
export class Entertainment extends plugin {
constructor (e) {
super({
@ -123,7 +132,11 @@ export class Entertainment extends plugin {
logger.info(`打招呼给群聊${groupId}` + message)
if (Config.defaultUseTTS) {
let audio = await generateAudio(message, Config.defaultTTSRole)
await Bot.sendGroupMsg(groupId, segment.record(audio))
if (useSilk) {
await Bot.sendGroupMsg(groupId, await uploadRecord(audio))
} else {
await Bot.sendGroupMsg(groupId, segment.record(audio))
}
} else {
await Bot.sendGroupMsg(groupId, message)
}

View file

@ -3,23 +3,24 @@
"type": "module",
"author": "ikechan8370",
"dependencies": {
"@fastify/cors": "^8.2.0",
"@fastify/static": "^6.9.0",
"@waylaidwanderer/chatgpt-api": "^1.33.2",
"chatgpt": "^5.1.1",
"delay": "^5.0.0",
"eventsource": "^2.0.2",
"eventsource-parser": "^1.0.0",
"fastify": "^4.13.0",
"https-proxy-agent": "5.0.1",
"keyv": "^4.5.2",
"keyv-file": "^0.2.0",
"node-fetch": "^3.3.1",
"node-silk": "^0.1.0",
"openai": "^3.2.1",
"random": "^4.1.0",
"undici": "^5.21.0",
"uuid": "^9.0.0",
"ws": "^8.13.0",
"@fastify/cors": "^8.2.0",
"@fastify/static": "^6.9.0",
"fastify": "^4.13.0",
"eventsource-parser": "^1.0.0"
"ws": "^8.13.0"
},
"optionalDependencies": {
"jimp": "^0.22.7",

View file

@ -17,7 +17,7 @@ const defaultConfig = {
alsoSendText: false,
autoUsePicture: true,
autoUsePictureThreshold: 1200,
ttsAutoFallbackThreshold: 99,
ttsAutoFallbackThreshold: 299,
conversationPreserveTime: 0,
toggleMode: 'at',
quoteReply: true,
@ -85,7 +85,7 @@ const defaultConfig = {
viewHost: '',
chatViewWidth: 1280,
chatViewBotName: '',
version: 'v2.4.12'
version: 'v2.4.13'
}
const _path = process.cwd()
let config = {}

341
utils/uploadRecord.js Normal file
View file

@ -0,0 +1,341 @@
import Contactable, { core } from 'oicq'
import querystring from 'querystring'
import fetch from 'node-fetch'
import fs from 'fs'
import os from 'os'
import util from 'util'
import stream from 'stream'
import crypto from 'crypto'
import child_process from 'child_process'
// import { pcm2slk } from 'node-silk'
let errors = {}
let pcm2slk
try {
pcm2slk = (await import('node-silk')).pcm2slk
} catch (e) {
logger.warn('未安装node-silk如ffmpeg不支持amr编码请安装node-silk以支持语音模式')
}
async function uploadRecord (recordUrl) {
const result = await getPttBuffer(recordUrl, Bot.config.ffmpeg_path)
if (!result.buffer) {
return false
}
let buf = result.buffer
const hash = md5(buf)
const codec = String(buf.slice(0, 7)).includes('SILK') ? 1 : 0
const body = core.pb.encode({
1: 3,
2: 3,
5: {
1: Contactable.target,
2: Bot.uin,
3: 0,
4: hash,
5: buf.length,
6: hash,
7: 5,
8: 9,
9: 4,
11: 0,
10: Bot.apk.version,
12: 1,
13: 1,
14: 0,
15: 1
}
})
const payload = await Bot.sendUni('PttStore.GroupPttUp', body)
const rsp = core.pb.decode(payload)[5]
rsp[2] && (0, errors.drop)(rsp[2], rsp[3])
const ip = rsp[5]?.[0] || rsp[5]; const port = rsp[6]?.[0] || rsp[6]
const ukey = rsp[7].toHex(); const filekey = rsp[11].toHex()
const params = {
ver: 4679,
ukey,
filekey,
filesize: buf.length,
bmd5: hash.toString('hex'),
mType: 'pttDu',
voice_encodec: codec
}
const url = `http://${int32ip2str(ip)}:${port}/?` + querystring.stringify(params)
const headers = {
'User-Agent': `QQ/${Bot.apk.version} CFNetwork/1126`,
'Net-Type': 'Wifi'
}
await fetch(url, {
method: 'POST', // post请求
headers,
body: buf
})
// await axios.post(url, buf, { headers });
const fid = rsp[11].toBuffer()
const b = core.pb.encode({
1: 4,
2: Bot.uin,
3: fid,
4: hash,
5: hash.toString('hex') + '.amr',
6: buf.length,
11: 1,
18: fid,
30: Buffer.from([8, 0, 40, 0, 56, 0])
})
return {
type: 'record', file: 'protobuf://' + Buffer.from(b).toString('base64')
}
}
export default uploadRecord
async function getPttBuffer (file, ffmpeg = 'ffmpeg') {
let buffer
let time
if (file instanceof Buffer || file.startsWith('base64://')) {
// Buffer或base64
const buf = file instanceof Buffer ? file : Buffer.from(file.slice(9), 'base64')
const head = buf.slice(0, 7).toString()
if (head.includes('SILK') || head.includes('AMR')) {
return buf
} else {
const tmpfile = TMP_DIR + '/' + (0, uuid)()
await fs.promises.writeFile(tmpfile, buf)
return audioTrans(tmpfile, ffmpeg)
}
} else if (file.startsWith('http://') || file.startsWith('https://')) {
// 网络文件
// const readable = (await axios.get(file, { responseType: "stream" })).data;
try {
const headers = {
'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 12; MI 9 Build/SKQ1.211230.001)'
}
let response = await fetch(file, {
method: 'GET', // post请求
headers
})
const buf = Buffer.from(await response.arrayBuffer())
const tmpfile = TMP_DIR + '/' + (0, uuid)()
await fs.promises.writeFile(tmpfile, buf)
// await (0, pipeline)(readable.pipe(new DownloadTransform), fs.createWriteStream(tmpfile));
const head = await read7Bytes(tmpfile)
if (head.includes('SILK') || head.includes('AMR')) {
fs.unlink(tmpfile, NOOP)
buffer = buf
} else {
buffer = await audioTrans(tmpfile, ffmpeg)
}
} catch (err) {}
} else {
// 本地文件
file = String(file).replace(/^file:\/{2}/, '')
IS_WIN && file.startsWith('/') && (file = file.slice(1))
const head = await read7Bytes(file)
if (head.includes('SILK') || head.includes('AMR')) {
buffer = await fs.promises.readFile(file)
} else {
buffer = await audioTrans(file, ffmpeg)
}
}
return { buffer, time }
}
async function audioTrans (file, ffmpeg = 'ffmpeg') {
return new Promise((resolve, reject) => {
const tmpfile = TMP_DIR + '/' + (0, uuid)();
(0, child_process.exec)(`${ffmpeg} -i "${file}" -f s16le -ac 1 -ar 24000 "${tmpfile}"`, async (error, stdout, stderr) => {
try {
resolve(pcm2slk(fs.readFileSync(tmpfile)))
} catch {
reject(new core.ApiRejection(ErrorCode.FFmpegPttTransError, '音频转码到pcm失败请确认你的ffmpeg可以处理此转换'))
} finally {
fs.unlink(tmpfile, NOOP)
}
})
})
}
async function read7Bytes (file) {
const fd = await fs.promises.open(file, 'r')
const buf = (await fd.read(Buffer.alloc(7), 0, 7, 0)).buffer
fd.close()
return buf
}
function uuid () {
let hex = crypto.randomBytes(16).toString('hex')
return hex.substr(0, 8) + '-' + hex.substr(8, 4) + '-' + hex.substr(12, 4) + '-' + hex.substr(16, 4) + '-' + hex.substr(20)
}
/** 计算流的md5 */
function md5Stream (readable) {
return new Promise((resolve, reject) => {
readable.on('error', reject)
readable.pipe(crypto.createHash('md5')
.on('error', reject)
.on('data', resolve))
})
}
/** 计算文件的md5和sha */
function fileHash (filepath) {
const readable = fs.createReadStream(filepath)
const sha = new Promise((resolve, reject) => {
readable.on('error', reject)
readable.pipe(crypto.createHash('sha1')
.on('error', reject)
.on('data', resolve))
})
return Promise.all([md5Stream(readable), sha])
}
/** 群号转uin */
function code2uin (code) {
let left = Math.floor(code / 1000000)
if (left >= 0 && left <= 10) { left += 202 } else if (left >= 11 && left <= 19) { left += 469 } else if (left >= 20 && left <= 66) { left += 2080 } else if (left >= 67 && left <= 156) { left += 1943 } else if (left >= 157 && left <= 209) { left += 1990 } else if (left >= 210 && left <= 309) { left += 3890 } else if (left >= 310 && left <= 335) { left += 3490 } else if (left >= 336 && left <= 386) { left += 2265 } else if (left >= 387 && left <= 499) { left += 3490 }
return left * 1000000 + code % 1000000
}
/** uin转群号 */
function uin2code (uin) {
let left = Math.floor(uin / 1000000)
if (left >= 202 && left <= 212) { left -= 202 } else if (left >= 480 && left <= 488) { left -= 469 } else if (left >= 2100 && left <= 2146) { left -= 2080 } else if (left >= 2010 && left <= 2099) { left -= 1943 } else if (left >= 2147 && left <= 2199) { left -= 1990 } else if (left >= 2600 && left <= 2651) { left -= 2265 } else if (left >= 3800 && left <= 3989) { left -= 3490 } else if (left >= 4100 && left <= 4199) { left -= 3890 }
return left * 1000000 + uin % 1000000
}
function int32ip2str (ip) {
if (typeof ip === 'string') { return ip }
ip = ip & 0xffffffff
return [
ip & 0xff,
(ip & 0xff00) >> 8,
(ip & 0xff0000) >> 16,
(ip & 0xff000000) >> 24 & 0xff
].join('.')
}
/** 解析彩色群名片 */
function parseFunString (buf) {
if (buf[0] === 0xA) {
let res = ''
try {
let arr = core.pb.decode(buf)[1]
if (!Array.isArray(arr)) { arr = [arr] }
for (let v of arr) {
if (v[2]) { res += String(v[2]) }
}
} catch { }
return res
} else {
return String(buf)
}
}
/** xml转义 */
function escapeXml (str) {
return str.replace(/[&"><]/g, function (s) {
if (s === '&') { return '&amp;' }
if (s === '<') { return '&lt;' }
if (s === '>') { return '&gt;' }
if (s === '"') { return '&quot;' }
return ''
})
}
/** 用于下载限量 */
class DownloadTransform extends stream.Transform {
constructor () {
super(...arguments)
this._size = 0
}
_transform (data, encoding, callback) {
this._size += data.length
let error = null
if (this._size <= MAX_UPLOAD_SIZE) { this.push(data) } else { error = new Error('downloading over 30MB is refused') }
callback(error)
}
}
const IS_WIN = os.platform() === 'win32'
/** 系统临时目录,用于临时存放下载的图片等内容 */
const TMP_DIR = os.tmpdir()
/** 最大上传和下载大小以图片上传限制为准30MB */
const MAX_UPLOAD_SIZE = 31457280
/** no operation */
const NOOP = () => { }
/** promisified pipeline */
const pipeline = (0, util.promisify)(stream.pipeline)
/** md5 hash */
const md5 = (data) => (0, crypto.createHash)('md5').update(data).digest()
errors.LoginErrorCode = errors.drop = errors.ErrorCode = void 0
let ErrorCode;
(function (ErrorCode) {
/** 客户端离线 */
ErrorCode[ErrorCode.ClientNotOnline = -1] = 'ClientNotOnline'
/** 发包超时未收到服务器回应 */
ErrorCode[ErrorCode.PacketTimeout = -2] = 'PacketTimeout'
/** 用户不存在 */
ErrorCode[ErrorCode.UserNotExists = -10] = 'UserNotExists'
/** 群不存在(未加入) */
ErrorCode[ErrorCode.GroupNotJoined = -20] = 'GroupNotJoined'
/** 群员不存在 */
ErrorCode[ErrorCode.MemberNotExists = -30] = 'MemberNotExists'
/** 发消息时传入的参数不正确 */
ErrorCode[ErrorCode.MessageBuilderError = -60] = 'MessageBuilderError'
/** 群消息被风控发送失败 */
ErrorCode[ErrorCode.RiskMessageError = -70] = 'RiskMessageError'
/** 群消息有敏感词发送失败 */
ErrorCode[ErrorCode.SensitiveWordsError = -80] = 'SensitiveWordsError'
/** 上传图片/文件/视频等数据超时 */
ErrorCode[ErrorCode.HighwayTimeout = -110] = 'HighwayTimeout'
/** 上传图片/文件/视频等数据遇到网络错误 */
ErrorCode[ErrorCode.HighwayNetworkError = -120] = 'HighwayNetworkError'
/** 没有上传通道 */
ErrorCode[ErrorCode.NoUploadChannel = -130] = 'NoUploadChannel'
/** 不支持的file类型(没有流) */
ErrorCode[ErrorCode.HighwayFileTypeError = -140] = 'HighwayFileTypeError'
/** 文件安全校验未通过不存在 */
ErrorCode[ErrorCode.UnsafeFile = -150] = 'UnsafeFile'
/** 离线(私聊)文件不存在 */
ErrorCode[ErrorCode.OfflineFileNotExists = -160] = 'OfflineFileNotExists'
/** 群文件不存在(无法转发) */
ErrorCode[ErrorCode.GroupFileNotExists = -170] = 'GroupFileNotExists'
/** 获取视频中的图片失败 */
ErrorCode[ErrorCode.FFmpegVideoThumbError = -210] = 'FFmpegVideoThumbError'
/** 音频转换失败 */
ErrorCode[ErrorCode.FFmpegPttTransError = -220] = 'FFmpegPttTransError'
})(ErrorCode = errors.ErrorCode || (errors.ErrorCode = {}))
const ErrorMessage = {
[ErrorCode.UserNotExists]: '查无此人',
[ErrorCode.GroupNotJoined]: '未加入的群',
[ErrorCode.MemberNotExists]: '幽灵群员',
[ErrorCode.RiskMessageError]: '群消息发送失败,可能被风控',
[ErrorCode.SensitiveWordsError]: '群消息发送失败,请检查消息内容',
10: '消息过长',
34: '消息过长',
120: '在该群被禁言',
121: 'AT全体剩余次数不足'
}
function drop (code, message) {
if (!message || !message.length) { message = ErrorMessage[code] }
throw new core.ApiRejection(code, message)
}
errors.drop = drop
/** 登录时可能出现的错误,不在列的都属于未知错误,暂时无法解决 */
let LoginErrorCode;
(function (LoginErrorCode) {
/** 密码错误 */
LoginErrorCode[LoginErrorCode.WrongPassword = 1] = 'WrongPassword'
/** 账号被冻结 */
LoginErrorCode[LoginErrorCode.AccountFrozen = 40] = 'AccountFrozen'
/** 发短信太频繁 */
LoginErrorCode[LoginErrorCode.TooManySms = 162] = 'TooManySms'
/** 短信验证码错误 */
LoginErrorCode[LoginErrorCode.WrongSmsCode = 163] = 'WrongSmsCode'
/** 滑块ticket错误 */
LoginErrorCode[LoginErrorCode.WrongTicket = 237] = 'WrongTicket'
})(LoginErrorCode = errors.LoginErrorCode || (errors.LoginErrorCode = {}))