mirror of
https://github.com/ikechan8370/chatgpt-plugin.git
synced 2025-12-17 13:57:10 +00:00
8 lines
No EOL
227 B
TypeScript
8 lines
No EOL
227 B
TypeScript
import { getEncoding } from 'js-tiktoken'
|
|
|
|
// TODO: make this configurable
|
|
const tokenizer = getEncoding('cl100k_base')
|
|
|
|
export function encode(input: string): Uint32Array {
|
|
return new Uint32Array(tokenizer.encode(input))
|
|
} |