heimoshuiyu/chatgpt-api-web
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases 1 Wiki Activity

calculate token and forget some message

Browse Source
This commit is contained in:
黑墨水鱼
2023-03-28 21:12:34 +08:00
parent 26a66d112b
commit 7c34379ecb
3 changed files with 19 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/chatgpt.ts
View File

@@ -25,6 +25,14 @@ export interface FetchResponse {
index: number | undefined;
}[];
}
// https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them
export function calculate_token_length(content: string): number {
const totalCount = content.length;
const chineseCount = content.match(/[\u00ff-\uffff]|\S+/g)?.length ?? 0;
const englishCount = totalCount - chineseCount;
const tokenLength = englishCount / 4 + (chineseCount * 4) / 3;
return ~~tokenLength;
}
class Chat {
OPENAI_API_KEY: string;
@@ -114,13 +122,8 @@ class Chat {
return this._fetch(true);
}
// https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them
calculate_token_length(content: string): number {
const totalCount = content.length;
const chineseCount = content.match(/[\u00ff-\uffff]|\S+/g)?.length ?? 0;
const englishCount = totalCount - chineseCount;
const tokenLength = englishCount / 4 + (chineseCount * 4) / 3;
return ~~tokenLength;
return calculate_token_length(content);
}
user(...messages: string[]) {