Skip to content

fix: CJK-aware token estimation in estimateTokens #47

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lishixiang0705 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: CJK-aware token estimation in estimateTokens #47

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion src/assembler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,17 @@ export interface AssembleContextResult {

/** Simple token estimate: ~4 chars per token, same as VoltCode's Token.estimate */
function estimateTokens(text: string): number {
return Math.ceil(text.length / 4);
let count = 0;
for (const ch of text) {
const code = ch.codePointAt(0)!;
// CJK Unified Ideographs + Extension A: ~1.5 tokens per char
if ((code >= 0x4E00 && code <= 0x9FFF) || (code >= 0x3400 && code <= 0x4DBF)) {
count += 1.5;
} else {
count += 0.25;
}
}
return Math.ceil(count);
}

type SummaryPromptSignal = Pick<SummaryRecord, "kind" | "depth" | "descendantCount">;
Expand Down
12 changes: 11 additions & 1 deletion src/compaction.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,17 @@ type CondensedPhaseCandidate = {

/** Estimate token count from character length (~4 chars per token). */
function estimateTokens(content: string): number {
return Math.ceil(content.length / 4);
let count = 0;
for (const ch of content) {
const code = ch.codePointAt(0)!;
// CJK Unified Ideographs + Extension A: ~1.5 tokens per char
if ((code >= 0x4E00 && code <= 0x9FFF) || (code >= 0x3400 && code <= 0x4DBF)) {
count += 1.5;
} else {
count += 0.25;
}
}
return Math.ceil(count);
}

/** Format a timestamp as `YYYY-MM-DD HH:mm TZ` for prompt source text. */
Expand Down