MurasameNoa/chatgpt-plugin
1
0
Fork 0
mirror of https://github.com/ikechan8370/chatgpt-plugin.git synced 2025-12-17 13:57:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: 2.7 dev start

Browse source
This commit is contained in:
ikechan8370 2023-06-22 18:30:08 +08:00
parent fbe8953667
commit 4a4dceec18
7 changed files with 5459 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

8
utils/openai/tokenizer.ts Normal file
View file

@ -0,0 +1,8 @@
import { getEncoding } from 'js-tiktoken'
// TODO: make this configurable
const tokenizer = getEncoding('cl100k_base')
export function encode(input: string): Uint32Array {
return new Uint32Array(tokenizer.encode(input))
}