MurasameNoa/chatgpt-plugin
1
0
Fork 0
mirror of https://github.com/ikechan8370/chatgpt-plugin.git synced 2025-12-17 05:47:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: 增加通义千问支持

Browse source
This commit is contained in:
ikechan8370 2023-10-25 21:50:44 +08:00
parent d528840297
commit c0936e6e2a
13 changed files with 1380 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

8
utils/alibaba/tokenizer.ts Normal file
View file

@ -0,0 +1,8 @@
import { getEncoding } from 'js-tiktoken'
// TODO: make this configurable
const tokenizer = getEncoding('cl100k_base')
export function encode(input: string): Uint32Array {
return new Uint32Array(tokenizer.encode(input))
}