chatgpt-plugin/utils/alibaba/tokenizer.ts
2025-02-03 00:07:26 +08:00

9 lines
242 B
TypeScript

// @ts-ignore
import { getEncoding } from 'js-tiktoken'
// TODO: make this configurable
const tokenizer = getEncoding('cl100k_base')
export function encode(input: string): Uint32Array {
return new Uint32Array(tokenizer.encode(input))
}