Merge branch 'temp' into v2

This commit is contained in:
葛胤池 2023-04-29 18:35:47 +08:00
commit d3bd7be722
5 changed files with 346 additions and 11 deletions

View file

@ -8,6 +8,7 @@ import { BingAIClient } from '@waylaidwanderer/chatgpt-api'
import SydneyAIClient from '../utils/SydneyAIClient.js'
import { PoeClient } from '../utils/poe/index.js'
import AzureTTS from '../utils/tts/microsoft-azure.js'
import VoiceVoxTTS from '../utils/tts/voicevox.js'
import fs from 'fs'
import {
render, renderUrl,
@ -510,9 +511,25 @@ export class chatgpt extends plugin {
}
async switch2Audio (e) {
if (!Config.ttsSpace) {
await this.reply('您没有配置VITS API请前往锅巴面板进行配置')
return
switch (Config.ttsMode) {
case 'vits-uma-genshin-honkai':
if (!Config.ttsSpace) {
await this.reply('您没有配置VITS API请前往锅巴面板进行配置')
return
}
break
case 'azure':
if (!Config.azureKey) {
await this.reply('您没有配置Azure Key请前往锅巴面板进行配置')
return
}
break
case 'voicevox':
if (!Config.voicevoxSpace) {
await this.reply('您没有配置VoiceVox API请前往锅巴面板进行配置')
return
}
break
}
let userSetting = await redis.get(`CHATGPT:USER:${e.sender.user_id}`)
if (!userSetting) {
@ -536,8 +553,12 @@ export class chatgpt extends plugin {
Config.ttsMode = 'azure'
break
}
case '3': {
Config.ttsMode = 'voicevox'
break
}
default: {
await e.reply('请使用#chatgpt语音换源+数字进行换源。1为vits-uma-genshin-honkai2为微软Azure')
await e.reply('请使用#chatgpt语音换源+数字进行换源。1为vits-uma-genshin-honkai2为微软Azure3为voicevox')
return
}
}
@ -553,6 +574,10 @@ export class chatgpt extends plugin {
await this.reply('您没有配置azure 密钥,请前往后台管理或锅巴面板进行配置')
return
}
if (Config.ttsMode === 'voicevox' && !Config.voicevoxSpace) {
await this.reply('您没有配置voicevox API请前往后台管理或锅巴面板进行配置')
return
}
const regex = /^#chatgpt设置(语音角色|角色语音|角色)/
let speaker = e.msg.replace(regex, '').trim() || '随机'
switch (Config.ttsMode) {
@ -590,6 +615,34 @@ export class chatgpt extends plugin {
}
break
}
case 'voicevox': {
let regex = /^(.*?)-(.*)$/
let match = regex.exec(speaker)
let style = null
if (match) {
speaker = match[1]
style = match[2]
}
let chosen = VoiceVoxTTS.supportConfigurations.filter(s => s.name === speaker)
if (chosen.length === 0) {
await this.reply(`抱歉,没有"${speaker}"这个角色目前voicevox模式下支持的角色有${VoiceVoxTTS.supportConfigurations.map(item => item.name).join('、')}`)
break
}
if (style && !chosen[0].styles.find(item => item.name === style)) {
await this.reply(`抱歉,"${speaker}"这个角色没有"${style}"这个风格,目前支持的风格有${chosen[0].styles.map(item => item.name).join('、')}`)
break
}
let userSetting = await redis.get(`CHATGPT:USER:${e.sender.user_id}`)
if (!userSetting) {
userSetting = getDefaultReplySetting()
} else {
userSetting = JSON.parse(userSetting)
}
userSetting.ttsRoleVoiceVox = chosen[0].name + (style ? `-${style}` : '')
await redis.set(`CHATGPT:USER:${e.sender.user_id}`, JSON.stringify(userSetting))
await this.reply(`您的默认语音角色已被设置为”${userSetting.ttsRoleVoiceVox}`)
break
}
}
}
@ -695,6 +748,8 @@ export class chatgpt extends plugin {
speaker = convertSpeaker(userSetting.ttsRole || Config.defaultTTSRole)
} else if (Config.ttsMode === 'azure') {
speaker = userSetting.ttsRoleAzure || Config.azureTTSSpeaker
} else if (Config.ttsMode === 'voicevox') {
speaker = userSetting.ttsRoleVoiceVox || Config.voicevoxTTSSpeaker
}
// 每个回答可以指定
let trySplit = prompt.split('回答:')
@ -981,14 +1036,18 @@ export class chatgpt extends plugin {
}
} else if (Config.ttsMode === 'azure' && Config.azureTTSKey) {
wav = await AzureTTS.generateAudio(ttsResponse, {
speaker: speaker
speaker
})
} else if (Config.ttsMode === 'voicevox' && Config.voicevoxSpace) {
wav = await VoiceVoxTTS.generateAudio(ttsResponse, {
speaker
})
} else {
await this.reply('你没有配置转语音API哦')
}
try {
try {
let sendable = await uploadRecord(wav, Config.ttsMode === 'azure')
let sendable = await uploadRecord(wav, Config.ttsMode)
if (sendable) {
await e.reply(sendable)
} else {

View file

@ -1,6 +1,7 @@
import { Config } from './utils/config.js'
import { speakers } from './utils/tts.js'
import AzureTTS from './utils/tts/microsoft-azure.js'
import VoiceVoxTTS from './utils/tts/voicevox.js'
// 支持锅巴
export function supportGuoba () {
return {
@ -87,6 +88,10 @@ export function supportGuoba () {
{
label: '微软Azure',
value: 'azure'
},
{
label: 'VoiceVox',
value: 'voicevox'
}
]
}
@ -100,6 +105,19 @@ export function supportGuoba () {
options: speakers.concat('随机').map(s => { return { label: s, value: s } })
}
},
{
field: 'voicevoxTTSSpeaker',
label: '语音模式默认角色VoiceVox',
bottomHelpMessage: 'VoiceVox语音模式下未指定角色时使用的角色。若留空将使用随机角色回复。若用户通过指令指定了角色将忽略本设定',
component: 'Select',
componentProps: {
options: VoiceVoxTTS.supportConfigurations.map(item => {
return item.styles.map(style => {
return `${item.name}-${style.name}`
}).concat(item.name)
}).flat().concat('随机').map(s => { return { label: s, value: s } })
}
},
{
field: 'azureTTSSpeaker',
label: '语音模式默认角色微软Azure',
@ -545,6 +563,12 @@ export function supportGuoba () {
bottomHelpMessage: '前往duplicate空间https://huggingface.co/spaces/ikechan8370/vits-uma-genshin-honkai后查看api地址',
component: 'Input'
},
{
field: 'voicevoxSpace',
label: 'voicevox语音转换API地址',
bottomHelpMessage: '可使用https://2ndelement-voicevox.hf.space, 也可github搜索voicevox-engine自建',
component: 'Input'
},
{
field: 'azureTTSKey',
label: 'Azure语音服务密钥',
@ -704,7 +728,7 @@ export function supportGuoba () {
label: 'Live2D模型',
bottomHelpMessage: '选择Live2D使用的模型',
component: 'Input'
},
}
],
// 获取配置数据方法(用于前端填充显示数据)
getConfigData () {

View file

@ -103,12 +103,15 @@ const defaultConfig = {
slackClaudeGlobalPreset: '',
slackClaudeSpecifiedChannel: '',
cloudTranscode: 'https://silk.201666.xyz',
cloudMode: 'url',
cloudMode: 'file',
ttsMode: 'vits-uma-genshin-honkai', // or azure
azureTTSKey: '',
azureTTSRegion: '',
azureTTSSpeaker: 'zh-CN-XiaochenNeural',
voicevoxSpace: '',
voicevoxTTSSpeaker: '护士机器子T',
version: 'v2.5.7'
}
const _path = process.cwd()
let config = {}

225
utils/tts/voicevox.js Normal file
View file

@ -0,0 +1,225 @@
import { Config } from '../config.js'
let proxy
if (Config.proxy) {
try {
proxy = (await import('https-proxy-agent')).default
} catch (e) {
console.warn('未安装https-proxy-agent请在插件目录下执行pnpm add https-proxy-agent')
}
}
const newFetch = (url, options = {}) => {
const defaultOptions = Config.proxy
? {
agent: proxy(Config.proxy)
}
: {}
const mergedOptions = {
...defaultOptions,
...options
}
return fetch(url, mergedOptions)
}
async function generateAudio (text, options = {}) {
let host = Config.voicevoxSpace
let speaker = options.speaker || '随机'
if (speaker === '随机') {
speaker = supportConfigurations[Math.floor(Math.random() * supportConfigurations.length)].name
}
let regex = /^(.*?)-(.*)$/
let match = regex.exec(speaker)
let style = null
if (match) {
speaker = match[1]
style = match[2]
}
speaker = supportConfigurations.find(s => s.name === speaker)
let speakerId
if (style) {
speakerId = speaker.styles.find(s => s.name === style).id
} else {
speakerId = speaker.styles[Math.floor(Math.random() * speaker?.styles.length)].id
}
logger.info(`使用${speaker.name}${speaker.styles.find(s => s.id === speakerId).name}风格基于文本${text}生成语音。`)
const accentPhrasesResponse = await newFetch(`${host}/accent_phrases?text=${encodeURIComponent(text)}&speaker=${speakerId}`, {
method: 'POST'
})
const accentPhrases = await accentPhrasesResponse.json()
const synthesisResponse = await newFetch(`${host}/synthesis?speaker=${speakerId}&enable_interrogative_upspeak=false`, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
accent_phrases: accentPhrases,
speedScale: 1,
pitchScale: 0,
intonationScale: 1,
volumeScale: 1,
prePhonemeLength: 0.1,
postPhonemeLength: 0.1,
outputSamplingRate: 24000,
outputStereo: false
})
})
const synthesisResponseData = await synthesisResponse.arrayBuffer()
return Buffer.from(synthesisResponseData)
}
const supportConfigurations = [
{
supported_features: { permitted_synthesis_morphing: 'SELF_ONLY' },
name: '四国めたん',
speaker_uuid: '7ffcb7ce-00ec-4bdc-82cd-45a8889e43ff',
styles: [{ name: 'ノーマル', id: 2 }, { name: 'あまあま', id: 0 }, { name: 'ツンツン', id: 6 }, {
name: 'セクシー', id: 4
}, { name: 'ささやき', id: 36 }, { name: 'ヒソヒソ', id: 37 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'SELF_ONLY' },
name: 'ずんだもん',
speaker_uuid: '388f246b-8c41-4ac1-8e2d-5d79f3ff56d9',
styles: [{ name: 'ノーマル', id: 3 }, { name: 'あまあま', id: 1 }, { name: 'ツンツン', id: 7 }, {
name: 'セクシー', id: 5
}, { name: 'ささやき', id: 22 }, { name: 'ヒソヒソ', id: 38 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '春日部つむぎ',
speaker_uuid: '35b2c544-660e-401e-b503-0e14c635303a',
styles: [{ name: 'ノーマル', id: 8 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '雨晴はう',
speaker_uuid: '3474ee95-c274-47f9-aa1a-8322163d96f1',
styles: [{ name: 'ノーマル', id: 10 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '波音リツ',
speaker_uuid: 'b1a81618-b27b-40d2-b0ea-27a9ad408c4b',
styles: [{ name: 'ノーマル', id: 9 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '玄野武宏',
speaker_uuid: 'c30dc15a-0992-4f8d-8bb8-ad3b314e6a6f',
styles: [{ name: 'ノーマル', id: 11 }, { name: '喜び', id: 39 }, { name: 'ツンギレ', id: 40 }, {
name: '悲しみ', id: 41
}],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '白上虎太郎',
speaker_uuid: 'e5020595-5c5d-4e87-b849-270a518d0dcf',
styles: [{ name: 'ふつう', id: 12 }, { name: 'わーい', id: 32 }, { name: 'びくびく', id: 33 }, {
name: 'おこ', id: 34
}, { name: 'びえーん', id: 35 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '青山龍星',
speaker_uuid: '4f51116a-d9ee-4516-925d-21f183e2afad',
styles: [{ name: 'ノーマル', id: 13 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '冥鳴ひまり',
speaker_uuid: '8eaad775-3119-417e-8cf4-2a10bfd592c8',
styles: [{ name: 'ノーマル', id: 14 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'SELF_ONLY' },
name: '九州そら',
speaker_uuid: '481fb609-6446-4870-9f46-90c4dd623403',
styles: [{ name: 'ノーマル', id: 16 }, { name: 'あまあま', id: 15 }, { name: 'ツンツン', id: 18 }, {
name: 'セクシー', id: 17
}, { name: 'ささやき', id: 19 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'SELF_ONLY' },
name: 'もち子さん',
speaker_uuid: '9f3ee141-26ad-437e-97bd-d22298d02ad2',
styles: [{ name: 'ノーマル', id: 20 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '剣崎雌雄',
speaker_uuid: '1a17ca16-7ee5-4ea5-b191-2f02ace24d21',
styles: [{ name: 'ノーマル', id: 21 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: 'WhiteCUL',
speaker_uuid: '67d5d8da-acd7-4207-bb10-b5542d3a663b',
styles: [{ name: 'ノーマル', id: 23 }, { name: 'たのしい', id: 24 }, { name: 'かなしい', id: 25 }, {
name: 'びえーん', id: 26
}],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '後鬼',
speaker_uuid: '0f56c2f2-644c-49c9-8989-94e11f7129d0',
styles: [{ name: '人間ver.', id: 27 }, { name: 'ぬいぐるみver.', id: 28 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: 'No.7',
speaker_uuid: '044830d2-f23b-44d6-ac0d-b5d733caa900',
styles: [{ name: 'ノーマル', id: 29 }, { name: 'アナウンス', id: 30 }, { name: '読み聞かせ', id: 31 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: 'ちび式じい',
speaker_uuid: '468b8e94-9da4-4f7a-8715-a22a48844f9e',
styles: [{ name: 'ノーマル', id: 42 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '櫻歌ミコ',
speaker_uuid: '0693554c-338e-4790-8982-b9c6d476dc69',
styles: [{ name: 'ノーマル', id: 43 }, { name: '第二形態', id: 44 }, { name: 'ロリ', id: 45 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '小夜/SAYO',
speaker_uuid: 'a8cc6d22-aad0-4ab8-bf1e-2f843924164a',
styles: [{ name: 'ノーマル', id: 46 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '护士机器子T',
speaker_uuid: '882a636f-3bac-431a-966d-c5e6bba9f949',
styles: [{ name: 'ノーマル', id: 47 }, { name: '楽々', id: 48 }, { name: '恐怖', id: 49 }, {
name: '内緒話', id: 50
}],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '†聖騎士 紅桜†',
speaker_uuid: '471e39d2-fb11-4c8c-8d89-4b322d2498e0',
styles: [{ name: 'ノーマル', id: 51 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '雀松朱司',
speaker_uuid: '0acebdee-a4a5-4e12-a695-e19609728e30',
styles: [{ name: 'ノーマル', id: 52 }],
version: '0.14.2'
}, {
supported_features: { permitted_synthesis_morphing: 'ALL' },
name: '麒ヶ島宗麟',
speaker_uuid: '7d1e7ba7-f957-40e5-a3fc-da49f769ab65',
styles: [{ name: 'ノーマル', id: 53 }],
version: '0.14.2'
}]
export default { generateAudio, supportConfigurations }

View file

@ -8,6 +8,7 @@ import stream from 'stream'
import crypto from 'crypto'
import child_process from 'child_process'
import { Config } from './config.js'
import {mkdirs} from "./common.js";
let module
try {
module = await import('oicq')
@ -37,13 +38,29 @@ if (module) {
// import { pcm2slk } from 'node-silk'
let errors = {}
async function uploadRecord (recordUrl, forceFile) {
async function uploadRecord (recordUrl, ttsMode = 'vits-uma-genshin-honkai') {
let recordType = 'url'
let tmpFile = ''
if (ttsMode === 'azure') {
recordType = 'file'
} else if (ttsMode === 'voicevox') {
recordType = 'buffer'
tmpFile = `data/chatgpt/tts/tmp/${crypto.randomUUID()}.wav`
}
let result
if (pcm2slk) {
result = await getPttBuffer(recordUrl, Bot.config.ffmpeg_path)
} else if (Config.cloudTranscode) {
logger.mark('使用云转码silk进行高清语音生成:"')
try {
if (forceFile || Config.cloudMode === 'file') {
if (recordType === 'buffer') {
// save it as a file
mkdirs('data/chatgpt/tts/tmp')
fs.writeFileSync(tmpFile, recordUrl)
recordType = 'file'
recordUrl = tmpFile
}
if (recordType === 'file' || Config.cloudMode === 'file') {
const formData = new FormData()
let buffer
if (!recordUrl.startsWith('http')) {
@ -103,7 +120,7 @@ async function uploadRecord (recordUrl, forceFile) {
if (!result.buffer) {
return false
}
let buf = result.buffer
let buf = Buffer.from(result.buffer)
const hash = md5(buf)
const codec = String(buf.slice(0, 7)).includes('SILK') ? 1 : 0
const body = core.pb.encode({
@ -165,6 +182,13 @@ async function uploadRecord (recordUrl, forceFile) {
18: fid,
30: Buffer.from([8, 0, 40, 0, 56, 0])
})
if (tmpFile) {
try {
fs.unlinkSync(tmpFile)
} catch (err) {
logger.warn('fail to delete temp audio file')
}
}
return {
type: 'record', file: 'protobuf://' + Buffer.from(b).toString('base64')
}