MurasameNoa/chatgpt-plugin
1
0
Fork 0
mirror of https://github.com/ikechan8370/chatgpt-plugin.git synced 2025-12-17 22:07:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: 增加通义千问支持

Browse source
This commit is contained in:
ikechan8370 2023-10-25 21:50:44 +08:00
parent d528840297
commit c0936e6e2a
13 changed files with 1380 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

6
utils/alibaba/tokenizer.js Normal file
View file

@ -0,0 +1,6 @@
import { getEncoding } from 'js-tiktoken';
// TODO: make this configurable
var tokenizer = getEncoding('cl100k_base');
export function encode(input) {
return new Uint32Array(tokenizer.encode(input));
}