在新的多TTS源模式下添加了VoiceVox源 (#381)

* feat: 添加基于voicevox语音支持

* fix: 配置里添加voicevox语音源

* fix: 切换语音模式对各种语言源提示错误

* fix: 修正voicevox角色风格设置问题

* fix: 修改角色性格切换问题

* fix: 解决角色性格切换问题

* fix 换源分支少打个break(java17-user是这样的)

* fix: 支持云转码的一些调整

---------

Co-authored-by: 葛胤池 <geyinchi@buaa.edu.cn>

apps/chat.js

@@ -8,6 +8,7 @@ import { BingAIClient } from '@waylaidwanderer/chatgpt-api'
 import SydneyAIClient from '../utils/SydneyAIClient.js'
 import { PoeClient } from '../utils/poe/index.js'
 import AzureTTS from '../utils/tts/microsoft-azure.js'
+import VoiceVoxTTS from '../utils/tts/voicevox.js'
 import fs from 'fs'
 import {
   render, renderUrl,
@@ -57,10 +58,10 @@ if (Config.proxy) {
 const defaultPropmtPrefix = ', a large language model trained by OpenAI. You answer as concisely as possible for each response (e.g. don’t be verbose). It is very important that you answer as concisely as possible, so please remember this. If you are generating a list, do not have too many items. Keep the number of items short.'
 const newFetch = (url, options = {}) => {
   const defaultOptions = Config.proxy
-    ? {
+      ? {
         agent: proxy(Config.proxy)
       }
-    : {}
+      : {}
   const mergedOptions = {
     ...defaultOptions,
     ...options
@@ -510,9 +511,25 @@ export class chatgpt extends plugin {
   }
 
   async switch2Audio (e) {
-    if (!Config.ttsSpace) {
+    switch (Config.ttsMode) {
-      await this.reply('您没有配置VITS API，请前往锅巴面板进行配置')
+      case 'vits-uma-genshin-honkai':
-      return
+        if (!Config.ttsSpace) {
+          await this.reply('您没有配置VITS API，请前往锅巴面板进行配置')
+          return
+        }
+        break
+      case 'azure':
+        if (!Config.azureKey) {
+          await this.reply('您没有配置Azure Key，请前往锅巴面板进行配置')
+          return
+        }
+        break
+      case 'voicevox':
+        if (!Config.voicevoxSpace) {
+          await this.reply('您没有配置VoiceVox API，请前往锅巴面板进行配置')
+          return
+        }
+        break
     }
     let userSetting = await redis.get(`CHATGPT:USER:${e.sender.user_id}`)
     if (!userSetting) {
@@ -536,8 +553,12 @@ export class chatgpt extends plugin {
         Config.ttsMode = 'azure'
         break
       }
+      case '3': {
+        Config.ttsMode = 'voicevox'
+        break
+      }
       default: {
-        await e.reply('请使用#chatgpt语音换源+数字进行换源。1为vits-uma-genshin-honkai，2为微软Azure')
+        await e.reply('请使用#chatgpt语音换源+数字进行换源。1为vits-uma-genshin-honkai，2为微软Azure，3为voicevox')
         return
       }
     }
@@ -553,6 +574,10 @@ export class chatgpt extends plugin {
       await this.reply('您没有配置azure 密钥，请前往后台管理或锅巴面板进行配置')
       return
     }
+    if (Config.ttsMode === 'voicevox' && !Config.voicevoxSpace) {
+      await this.reply('您没有配置voicevox API，请前往后台管理或锅巴面板进行配置')
+      return
+    }
     const regex = /^#chatgpt设置(语音角色|角色语音|角色)/
     let speaker = e.msg.replace(regex, '').trim() || '随机'
     switch (Config.ttsMode) {
@@ -590,6 +615,34 @@ export class chatgpt extends plugin {
         }
         break
       }
+      case 'voicevox': {
+        let regex = /^(.*?)-(.*)$/
+        let match = regex.exec(speaker)
+        let style = null
+        if (match) {
+          speaker = match[1]
+          style = match[2]
+        }
+        let chosen = VoiceVoxTTS.supportConfigurations.filter(s => s.name === speaker)
+        if (chosen.length === 0) {
+          await this.reply(`抱歉，没有"${speaker}"这个角色，目前voicevox模式下支持的角色有${VoiceVoxTTS.supportConfigurations.map(item => item.name).join('、')}`)
+          break
+        }
+        if (style && !chosen[0].styles.find(item => item.name === style)) {
+          await this.reply(`抱歉，"${speaker}"这个角色没有"${style}"这个风格，目前支持的风格有${chosen[0].styles.map(item => item.name).join('、')}`)
+          break
+        }
+        let userSetting = await redis.get(`CHATGPT:USER:${e.sender.user_id}`)
+        if (!userSetting) {
+          userSetting = getDefaultReplySetting()
+        } else {
+          userSetting = JSON.parse(userSetting)
+        }
+        userSetting.ttsRoleVoiceVox = chosen[0].name + (style ? `-${style}` : '')
+        await redis.set(`CHATGPT:USER:${e.sender.user_id}`, JSON.stringify(userSetting))
+        await this.reply(`您的默认语音角色已被设置为”${userSetting.ttsRoleVoiceVox}“`)
+        break
+      }
     }
   }
 
@@ -695,6 +748,8 @@ export class chatgpt extends plugin {
       speaker = convertSpeaker(userSetting.ttsRole || Config.defaultTTSRole)
     } else if (Config.ttsMode === 'azure') {
       speaker = userSetting.ttsRoleAzure || Config.azureTTSSpeaker
+    } else if (Config.ttsMode === 'voicevox') {
+      speaker = userSetting.ttsRoleVoiceVox || Config.voicevoxTTSSpeaker
     }
     // 每个回答可以指定
     let trySplit = prompt.split('回答：')
@@ -981,14 +1036,18 @@ export class chatgpt extends plugin {
           }
         } else if (Config.ttsMode === 'azure' && Config.azureTTSKey) {
           wav = await AzureTTS.generateAudio(ttsResponse, {
-            speaker: speaker
+            speaker
+          })
+        } else if (Config.ttsMode === 'voicevox' && Config.voicevoxSpace) {
+          wav = await VoiceVoxTTS.generateAudio(ttsResponse, {
+            speaker
           })
         } else {
           await this.reply('你没有配置转语音API哦')
         }
         try {
           try {
-            let sendable = await uploadRecord(wav, Config.ttsMode === 'azure')
+            let sendable = await uploadRecord(wav, Config.ttsMode)
             if (sendable) {
               await e.reply(sendable)
             } else {
@@ -1706,19 +1765,19 @@ export class chatgpt extends plugin {
         Authorization: 'Bearer ' + Config.apiKey
       }
     })
-      .then(response => response.json())
+        .then(response => response.json())
-      .then(data => {
+        .then(data => {
-        if (data.error) {
+          if (data.error) {
-          this.reply('获取失败：' + data.error.code)
+            this.reply('获取失败：' + data.error.code)
-          return false
+            return false
-        } else {
+          } else {
-          let total_granted = data.total_granted.toFixed(2)
+            let total_granted = data.total_granted.toFixed(2)
-          let total_used = data.total_used.toFixed(2)
+            let total_used = data.total_used.toFixed(2)
-          let total_available = data.total_available.toFixed(2)
+            let total_available = data.total_available.toFixed(2)
-          let expires_at = new Date(data.grants.data[0].expires_at * 1000).toLocaleDateString().replace(/\//g, '-')
+            let expires_at = new Date(data.grants.data[0].expires_at * 1000).toLocaleDateString().replace(/\//g, '-')
-          this.reply('总额度：$' + total_granted + '\n已经使用额度：$' + total_used + '\n当前剩余额度：$' + total_available + '\n到期日期(UTC)：' + expires_at)
+            this.reply('总额度：$' + total_granted + '\n已经使用额度：$' + total_used + '\n当前剩余额度：$' + total_available + '\n到期日期(UTC)：' + expires_at)
-        }
+          }
-      })
+        })
   }
 
   /**

guoba.support.js

@@ -1,6 +1,7 @@
 import { Config } from './utils/config.js'
 import { speakers } from './utils/tts.js'
 import AzureTTS from './utils/tts/microsoft-azure.js'
+import VoiceVoxTTS from './utils/tts/voicevox.js'
 // 支持锅巴
 export function supportGuoba () {
   return {
@@ -87,6 +88,10 @@ export function supportGuoba () {
               {
                 label: '微软Azure',
                 value: 'azure'
+              },
+              {
+                label: 'VoiceVox',
+                value: 'voicevox'
               }
             ]
           }
@@ -100,6 +105,19 @@ export function supportGuoba () {
             options: speakers.concat('随机').map(s => { return { label: s, value: s } })
           }
         },
+        {
+          field: 'voicevoxTTSSpeaker',
+          label: '语音模式默认角色（VoiceVox）',
+          bottomHelpMessage: 'VoiceVox语音模式下，未指定角色时使用的角色。若留空，将使用随机角色回复。若用户通过指令指定了角色，将忽略本设定',
+          component: 'Select',
+          componentProps: {
+            options: VoiceVoxTTS.supportConfigurations.map(item => {
+              return item.styles.map(style => {
+                return `${item.name}-${style.name}`
+              }).concat(item.name)
+            }).flat().concat('随机').map(s => { return { label: s, value: s } })
+          }
+        },
         {
           field: 'azureTTSSpeaker',
           label: '语音模式默认角色（微软Azure）',
@@ -545,6 +563,12 @@ export function supportGuoba () {
           bottomHelpMessage: '前往duplicate空间https://huggingface.co/spaces/ikechan8370/vits-uma-genshin-honkai后查看api地址',
           component: 'Input'
         },
+        {
+          field: 'voicevoxSpace',
+          label: 'voicevox语音转换API地址',
+          bottomHelpMessage: '可使用https://2ndelement-voicevox.hf.space, 也可github搜索voicevox-engine自建',
+          component: 'Input'
+        },
         {
           field: 'azureTTSKey',
           label: 'Azure语音服务密钥',
@@ -710,7 +734,7 @@ export function supportGuoba () {
           label: 'Live2D模型',
           bottomHelpMessage: '选择Live2D使用的模型',
           component: 'Input'
-        },
+        }
       ],
       // 获取配置数据方法（用于前端填充显示数据）
       getConfigData () {

utils/config.js

@@ -109,7 +109,10 @@ const defaultConfig = {
   azureTTSKey: '',
   azureTTSRegion: '',
   azureTTSSpeaker: 'zh-CN-XiaochenNeural',
-  version: 'v2.5.7'
+  voicevoxSpace: '',
+  voicevoxTTSSpeaker: '护士机器子T',
+  version: 'v2.5.7',
+
 }
 const _path = process.cwd()
 let config = {}

utils/tts/voicevox.js

@@ -0,0 +1,225 @@
+import { Config } from '../config.js'
+
+let proxy
+if (Config.proxy) {
+    try {
+        proxy = (await import('https-proxy-agent')).default
+    } catch (e) {
+        console.warn('未安装https-proxy-agent，请在插件目录下执行pnpm add https-proxy-agent')
+    }
+}
+
+const newFetch = (url, options = {}) => {
+    const defaultOptions = Config.proxy
+        ? {
+            agent: proxy(Config.proxy)
+        }
+        : {}
+
+    const mergedOptions = {
+        ...defaultOptions,
+        ...options
+    }
+
+    return fetch(url, mergedOptions)
+}
+
+async function generateAudio (text, options = {}) {
+    let host = Config.voicevoxSpace
+    let speaker = options.speaker || '随机'
+    if (speaker === '随机') {
+        speaker = supportConfigurations[Math.floor(Math.random() * supportConfigurations.length)].name
+    }
+    let regex = /^(.*?)-(.*)$/
+    let match = regex.exec(speaker)
+    let style = null
+    if (match) {
+        speaker = match[1]
+        style = match[2]
+    }
+    speaker = supportConfigurations.find(s => s.name === speaker)
+    let speakerId
+    if (style) {
+        speakerId = speaker.styles.find(s => s.name === style).id
+    } else {
+        speakerId = speaker.styles[Math.floor(Math.random() * speaker?.styles.length)].id
+    }
+    logger.info(`使用${speaker.name}的${speaker.styles.find(s => s.id === speakerId).name}风格基于文本${text}生成语音。`)
+    const accentPhrasesResponse = await newFetch(`${host}/accent_phrases?text=${encodeURIComponent(text)}&speaker=${speakerId}`, {
+        method: 'POST'
+    })
+
+    const accentPhrases = await accentPhrasesResponse.json()
+
+    const synthesisResponse = await newFetch(`${host}/synthesis?speaker=${speakerId}&enable_interrogative_upspeak=false`, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json'
+        },
+        body: JSON.stringify({
+            accent_phrases: accentPhrases,
+            speedScale: 1,
+            pitchScale: 0,
+            intonationScale: 1,
+            volumeScale: 1,
+            prePhonemeLength: 0.1,
+            postPhonemeLength: 0.1,
+            outputSamplingRate: 24000,
+            outputStereo: false
+        })
+    })
+
+    const synthesisResponseData = await synthesisResponse.arrayBuffer()
+    return Buffer.from(synthesisResponseData)
+}
+
+const supportConfigurations = [
+    {
+        supported_features: { permitted_synthesis_morphing: 'SELF_ONLY' },
+        name: '四国めたん',
+        speaker_uuid: '7ffcb7ce-00ec-4bdc-82cd-45a8889e43ff',
+        styles: [{ name: 'ノーマル', id: 2 }, { name: 'あまあま', id: 0 }, { name: 'ツンツン', id: 6 }, {
+            name: 'セクシー', id: 4
+        }, { name: 'ささやき', id: 36 }, { name: 'ヒソヒソ', id: 37 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'SELF_ONLY' },
+        name: 'ずんだもん',
+        speaker_uuid: '388f246b-8c41-4ac1-8e2d-5d79f3ff56d9',
+        styles: [{ name: 'ノーマル', id: 3 }, { name: 'あまあま', id: 1 }, { name: 'ツンツン', id: 7 }, {
+            name: 'セクシー', id: 5
+        }, { name: 'ささやき', id: 22 }, { name: 'ヒソヒソ', id: 38 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '春日部つむぎ',
+        speaker_uuid: '35b2c544-660e-401e-b503-0e14c635303a',
+        styles: [{ name: 'ノーマル', id: 8 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '雨晴はう',
+        speaker_uuid: '3474ee95-c274-47f9-aa1a-8322163d96f1',
+        styles: [{ name: 'ノーマル', id: 10 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '波音リツ',
+        speaker_uuid: 'b1a81618-b27b-40d2-b0ea-27a9ad408c4b',
+        styles: [{ name: 'ノーマル', id: 9 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '玄野武宏',
+        speaker_uuid: 'c30dc15a-0992-4f8d-8bb8-ad3b314e6a6f',
+        styles: [{ name: 'ノーマル', id: 11 }, { name: '喜び', id: 39 }, { name: 'ツンギレ', id: 40 }, {
+            name: '悲しみ', id: 41
+        }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '白上虎太郎',
+        speaker_uuid: 'e5020595-5c5d-4e87-b849-270a518d0dcf',
+        styles: [{ name: 'ふつう', id: 12 }, { name: 'わーい', id: 32 }, { name: 'びくびく', id: 33 }, {
+            name: 'おこ', id: 34
+        }, { name: 'びえーん', id: 35 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '青山龍星',
+        speaker_uuid: '4f51116a-d9ee-4516-925d-21f183e2afad',
+        styles: [{ name: 'ノーマル', id: 13 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '冥鳴ひまり',
+        speaker_uuid: '8eaad775-3119-417e-8cf4-2a10bfd592c8',
+        styles: [{ name: 'ノーマル', id: 14 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'SELF_ONLY' },
+        name: '九州そら',
+        speaker_uuid: '481fb609-6446-4870-9f46-90c4dd623403',
+        styles: [{ name: 'ノーマル', id: 16 }, { name: 'あまあま', id: 15 }, { name: 'ツンツン', id: 18 }, {
+            name: 'セクシー', id: 17
+        }, { name: 'ささやき', id: 19 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'SELF_ONLY' },
+        name: 'もち子さん',
+        speaker_uuid: '9f3ee141-26ad-437e-97bd-d22298d02ad2',
+        styles: [{ name: 'ノーマル', id: 20 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '剣崎雌雄',
+        speaker_uuid: '1a17ca16-7ee5-4ea5-b191-2f02ace24d21',
+        styles: [{ name: 'ノーマル', id: 21 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: 'WhiteCUL',
+        speaker_uuid: '67d5d8da-acd7-4207-bb10-b5542d3a663b',
+        styles: [{ name: 'ノーマル', id: 23 }, { name: 'たのしい', id: 24 }, { name: 'かなしい', id: 25 }, {
+            name: 'びえーん', id: 26
+        }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '後鬼',
+        speaker_uuid: '0f56c2f2-644c-49c9-8989-94e11f7129d0',
+        styles: [{ name: '人間ver.', id: 27 }, { name: 'ぬいぐるみver.', id: 28 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: 'No.7',
+        speaker_uuid: '044830d2-f23b-44d6-ac0d-b5d733caa900',
+        styles: [{ name: 'ノーマル', id: 29 }, { name: 'アナウンス', id: 30 }, { name: '読み聞かせ', id: 31 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: 'ちび式じい',
+        speaker_uuid: '468b8e94-9da4-4f7a-8715-a22a48844f9e',
+        styles: [{ name: 'ノーマル', id: 42 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '櫻歌ミコ',
+        speaker_uuid: '0693554c-338e-4790-8982-b9c6d476dc69',
+        styles: [{ name: 'ノーマル', id: 43 }, { name: '第二形態', id: 44 }, { name: 'ロリ', id: 45 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '小夜/SAYO',
+        speaker_uuid: 'a8cc6d22-aad0-4ab8-bf1e-2f843924164a',
+        styles: [{ name: 'ノーマル', id: 46 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '护士机器子T',
+        speaker_uuid: '882a636f-3bac-431a-966d-c5e6bba9f949',
+        styles: [{ name: 'ノーマル', id: 47 }, { name: '楽々', id: 48 }, { name: '恐怖', id: 49 }, {
+            name: '内緒話', id: 50
+        }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '†聖騎士 紅桜†',
+        speaker_uuid: '471e39d2-fb11-4c8c-8d89-4b322d2498e0',
+        styles: [{ name: 'ノーマル', id: 51 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '雀松朱司',
+        speaker_uuid: '0acebdee-a4a5-4e12-a695-e19609728e30',
+        styles: [{ name: 'ノーマル', id: 52 }],
+        version: '0.14.2'
+    }, {
+        supported_features: { permitted_synthesis_morphing: 'ALL' },
+        name: '麒ヶ島宗麟',
+        speaker_uuid: '7d1e7ba7-f957-40e5-a3fc-da49f769ab65',
+        styles: [{ name: 'ノーマル', id: 53 }],
+        version: '0.14.2'
+    }]
+
+export default { generateAudio, supportConfigurations }

utils/uploadRecord.js

@@ -8,6 +8,7 @@ import stream from 'stream'
 import crypto from 'crypto'
 import child_process from 'child_process'
 import { Config } from './config.js'
+import {mkdirs} from "./common.js";
 let module
 try {
   module = await import('oicq')
@@ -37,13 +38,29 @@ if (module) {
 // import { pcm2slk } from 'node-silk'
 let errors = {}
 
-async function uploadRecord (recordUrl, forceFile) {
+async function uploadRecord (recordUrl, ttsMode = 'vits-uma-genshin-honkai') {
+  let recordType = 'url'
+  let tmpFile = ''
+  if (ttsMode === 'azure') {
+    recordType = 'file'
+  } else if (ttsMode === 'voicevox') {
+    recordType = 'buffer'
+    tmpFile = `data/chatgpt/tts/tmp/${crypto.randomUUID()}.wav`
+  }
   let result
   if (pcm2slk) {
     result = await getPttBuffer(recordUrl, Bot.config.ffmpeg_path)
   } else if (Config.cloudTranscode) {
+    logger.mark('使用云转码silk进行高清语音生成:"')
     try {
-      if (forceFile || Config.cloudMode === 'file') {
+      if (recordType === 'buffer') {
+        // save it as a file
+        mkdirs('data/chatgpt/tts/tmp')
+        fs.writeFileSync(tmpFile, recordUrl)
+        recordType = 'file'
+        recordUrl = tmpFile
+      }
+      if (recordType === 'file' || Config.cloudMode === 'file') {
         const formData = new FormData()
         let buffer
         if (!recordUrl.startsWith('http')) {
@@ -103,7 +120,7 @@ async function uploadRecord (recordUrl, forceFile) {
   if (!result.buffer) {
     return false
   }
-  let buf = result.buffer
+  let buf = Buffer.from(result.buffer)
   const hash = md5(buf)
   const codec = String(buf.slice(0, 7)).includes('SILK') ? 1 : 0
   const body = core.pb.encode({
@@ -165,6 +182,13 @@ async function uploadRecord (recordUrl, forceFile) {
     18: fid,
     30: Buffer.from([8, 0, 40, 0, 56, 0])
   })
+  if (tmpFile) {
+    try {
+      fs.unlinkSync(tmpFile)
+    } catch (err) {
+      logger.warn('fail to delete temp audio file')
+    }
+  }
   return {
     type: 'record', file: 'protobuf://' + Buffer.from(b).toString('base64')
   }