mirror of
https://github.com/ikechan8370/chatgpt-plugin.git
synced 2025-12-16 13:27:08 +00:00
9 lines
242 B
TypeScript
9 lines
242 B
TypeScript
// @ts-ignore
|
|
import { getEncoding } from 'js-tiktoken'
|
|
|
|
// TODO: make this configurable
|
|
const tokenizer = getEncoding('cl100k_base')
|
|
|
|
export function encode(input: string): Uint32Array {
|
|
return new Uint32Array(tokenizer.encode(input))
|
|
}
|