mirror of
https://github.com/ikechan8370/chatgpt-plugin.git
synced 2025-12-17 13:57:10 +00:00
6 lines
206 B
JavaScript
6 lines
206 B
JavaScript
import { getEncoding } from 'js-tiktoken';
|
|
// TODO: make this configurable
|
|
var tokenizer = getEncoding('cl100k_base');
|
|
export function encode(input) {
|
|
return new Uint32Array(tokenizer.encode(input));
|
|
}
|