feat: add HuggingFace tokenizer integration for precise token counting

.changeset/green-steaks-juggle.md

@@ -0,0 +1,5 @@
+---
+"@martian-engineering/lossless-claw": minor
+---
+
+Add optional precise HuggingFace-based token counting for supported models and ensure tokenizer warmup works on first use. Proxy configuration is now applied only to tokenizer downloads and proxy credentials are redacted from logs.

.gitignore

@@ -8,3 +8,4 @@ dist/
 tui/lcm-tui
 dist/
 tui/tui
+pnpm-lock.yaml

AGENTS.md

@@ -0,0 +1,4 @@
+# Repository Notes
+
+- When adding or changing plugin config, config-related UI hints, or any manifest-facing capability, update `openclaw.plugin.json` in the same change.
+- Keep manifest updates manual unless a real shared schema source is introduced; do not add manifest-generation scripts without explicit approval.

index.ts

@@ -13,7 +13,8 @@ import { createLcmDescribeTool } from "./src/tools/lcm-describe-tool.js";
 import { createLcmExpandQueryTool } from "./src/tools/lcm-expand-query-tool.js";
 import { createLcmExpandTool } from "./src/tools/lcm-expand-tool.js";
 import { createLcmGrepTool } from "./src/tools/lcm-grep-tool.js";
-import type { LcmDependencies } from "./src/types.js";
+import type { LcmDependencies, TokenizerService } from "./src/types.js";
+import { HuggingFaceTokenizer, redactUrlCredentials } from "./src/tokenizers/huggingface.js";
 
 /** Parse `agent:<agentId>:<suffix...>` session keys. */
 function parseAgentSessionKey(sessionKey: string): { agentId: string; suffix: string } | null {
@@ -855,6 +856,8 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
     api.logger.warn(buildLegacyAuthFallbackWarning());
   }
 
+  const redactedProxy = redactUrlCredentials(config.proxy) ?? "none";
+
   return {
     config,
     complete: async ({
@@ -1262,6 +1265,20 @@ function createLcmDependencies(api: OpenClawPluginApi): LcmDependencies {
       error: (msg) => api.logger.error(msg),
       debug: (msg) => api.logger.debug?.(msg),
     },
+    tokenizer: config.useTokenizer
+      ? (() => {
+          const t = new HuggingFaceTokenizer(envSnapshot.openclawDefaultModel || "glm-5", config.proxy);
+          void t.initialize().catch((error) => {
+            api.logger.warn(
+              `[lcm] Tokenizer warmup failed (model=${envSnapshot.openclawDefaultModel || "glm-5"}): ${error instanceof Error ? error.message : String(error)}`,
+            );
+          });
+          api.logger.info(
+            `[lcm] Tokenizer created (model=${envSnapshot.openclawDefaultModel || "glm-5"}, proxy=${redactedProxy})`,
+          );
+          return t;
+        })()
+      : undefined,
   };
 }
 
@@ -1317,7 +1334,7 @@ const lcmPlugin = {
     );
 
     api.logger.info(
-      `[lcm] Plugin loaded (enabled=${deps.config.enabled}, db=${deps.config.databasePath}, threshold=${deps.config.contextThreshold})`,
+      `[lcm] Plugin loaded (enabled=${deps.config.enabled}, db=${deps.config.databasePath}, threshold=${deps.config.contextThreshold}, useTokenizer=${deps.config.useTokenizer}${deps.config.proxy ? `, proxy=${redactUrlCredentials(deps.config.proxy)}` : ""})`,
     );
   },
 };

openclaw.plugin.json

@@ -24,27 +24,39 @@
     "summaryProvider": {
       "label": "Summary Provider",
       "help": "Provider override for LCM summarization (e.g., 'openai-resp')"
+    },
+    "useTokenizer": {
+      "label": "Use Precise Tokenizer",
+      "help": "Use HuggingFace tokenizer service instead of chars/4 heuristic"
+    },
+    "proxy": {
+      "label": "Proxy URL",
+      "help": "HTTP(S) proxy for tokenizer downloads from HuggingFace"
     }
   },
   "configSchema": {
     "type": "object",
     "additionalProperties": false,
     "properties": {
       "enabled": {
-        "type": "boolean"
+        "type": "boolean",
+        "description": "Enable or disable the plugin"
       },
       "contextThreshold": {
         "type": "number",
         "minimum": 0,
-        "maximum": 1
+        "maximum": 1,
+        "description": "Fraction of context window that triggers compaction (0.0–1.0)"
       },
       "incrementalMaxDepth": {
         "type": "integer",
-        "minimum": -1
+        "minimum": -1,
+        "description": "How deep incremental compaction goes (0 = leaf only, -1 = unlimited)"
       },
       "freshTailCount": {
         "type": "integer",
-        "minimum": 1
+        "minimum": 1,
+        "description": "Number of recent messages protected from compaction"
       },
       "leafMinFanout": {
         "type": "integer",
@@ -59,17 +71,47 @@
         "minimum": 2
       },
       "dbPath": {
-        "type": "string"
+        "type": "string",
+        "description": "Path to LCM SQLite database (default: ~/.openclaw/lcm.db)"
       },
       "largeFileThresholdTokens": {
         "type": "integer",
-        "minimum": 1000
+        "minimum": 1000,
+        "description": "Token threshold for treating files as 'large'"
       },
       "summaryModel": {
         "type": "string"
       },
       "summaryProvider": {
         "type": "string"
+      },
+      "useTokenizer": {
+        "type": "boolean",
+        "description": "Use precise tokenizer service instead of chars/4 heuristic"
+      },
+      "proxy": {
+        "type": "string",
+        "description": "HTTP(S) proxy URL for tokenizer downloads from HuggingFace"
+      },
+      "timezone": {
+        "type": "string",
+        "description": "IANA timezone for timestamps in summaries"
+      },
+      "pruneHeartbeatOk": {
+        "type": "boolean",
+        "description": "Delete HEARTBEAT_OK turn cycles from LCM storage"
+      },
+      "autocompactDisabled": {
+        "type": "boolean",
+        "description": "Disable automatic compaction"
+      },
+      "largeFileSummaryProvider": {
+        "type": "string",
+        "description": "Provider override for large-file summarization"
+      },
+      "largeFileSummaryModel": {
+        "type": "string",
+        "description": "Model override for large-file summarization"
       }
     }
   }

package.json

@@ -30,9 +30,11 @@
     "version-packages": "changeset version"
   },
   "dependencies": {
+    "@huggingface/tokenizers": "^0.1.2",
     "@mariozechner/pi-agent-core": "*",
     "@mariozechner/pi-ai": "*",
-    "@sinclair/typebox": "0.34.48"
+    "@sinclair/typebox": "0.34.48",
+    "undici": "^7.22.0"
   },
   "devDependencies": {
     "@changesets/cli": "^2.30.0",

src/assembler.ts

@@ -6,6 +6,8 @@ import type {
   MessageRole,
 } from "./store/conversation-store.js";
 import type { SummaryStore, ContextItemRecord, SummaryRecord } from "./store/summary-store.js";
+import type { TokenizerService } from "./types.js";
+import { calculateTokens } from "./token-utils.js";
 
 type AgentMessage = Parameters<ContextEngine["ingest"]>[0]["message"];
 
@@ -16,6 +18,10 @@ export interface AssembleContextInput {
   tokenBudget: number;
   /** Number of most recent raw turns to always include (default: 8) */
   freshTailCount?: number;
+  /** Whether to use tokenizer for token counting (optional) */
+  useTokenizer?: boolean;
+  /** Tokenizer service for precise token counting (optional) */
+  tokenizer?: TokenizerService;
 }
 
 export interface AssembleContextResult {
@@ -33,13 +39,6 @@ export interface AssembleContextResult {
   };
 }
 
-// ── Helpers ──────────────────────────────────────────────────────────────────
-
-/** Simple token estimate: ~4 chars per token, same as VoltCode's Token.estimate */
-function estimateTokens(text: string): number {
-  return Math.ceil(text.length / 4);
-}
-
 type SummaryPromptSignal = Pick<SummaryRecord, "kind" | "depth" | "descendantCount">;
 
 /**
@@ -563,10 +562,19 @@ export class ContextAssembler {
    * 5. Return the final ordered messages in chronological order.
    */
   async assemble(input: AssembleContextInput): Promise<AssembleContextResult> {
+    if (input.useTokenizer && input.tokenizer?.initialize) {
+      try {
+        await input.tokenizer.initialize();
+      } catch {
+        // Fall back to heuristic counting when tokenizer warmup fails.
+      }
+    }
+
     const { conversationId, tokenBudget } = input;
     const freshTailCount = input.freshTailCount ?? 8;
+    const useTokenizer = input.useTokenizer;
+    const tokenizer = input.tokenizer;
 
-    // Step 1: Get all context items ordered by ordinal
     const contextItems = await this.summaryStore.getContextItems(conversationId);
 
     if (contextItems.length === 0) {
@@ -577,8 +585,7 @@ export class ContextAssembler {
       };
     }
 
-    // Step 2: Resolve each context item into a ResolvedItem
-    const resolved = await this.resolveItems(contextItems);
+    const resolved = await this.resolveItems(contextItems, useTokenizer, tokenizer);
 
     // Count stats from the full (pre-truncation) set
     let rawMessageCount = 0;
@@ -685,11 +692,15 @@ export class ContextAssembler {
    *
    * Items that cannot be resolved (e.g. deleted message) are silently skipped.
    */
-  private async resolveItems(contextItems: ContextItemRecord[]): Promise<ResolvedItem[]> {
+  private async resolveItems(
+    contextItems: ContextItemRecord[],
+    useTokenizer?: boolean,
+    tokenizer?: TokenizerService
+  ): Promise<ResolvedItem[]> {
     const resolved: ResolvedItem[] = [];
 
     for (const item of contextItems) {
-      const result = await this.resolveItem(item);
+      const result = await this.resolveItem(item, useTokenizer, tokenizer);
       if (result) {
         resolved.push(result);
       }
@@ -701,23 +712,27 @@ export class ContextAssembler {
   /**
    * Resolve a single context item.
    */
-  private async resolveItem(item: ContextItemRecord): Promise<ResolvedItem | null> {
+  private async resolveItem(
+    item: ContextItemRecord,
+    useTokenizer?: boolean,
+    tokenizer?: TokenizerService
+  ): Promise<ResolvedItem | null> {
     if (item.itemType === "message" && item.messageId != null) {
-      return this.resolveMessageItem(item);
+      return this.resolveMessageItem(item, useTokenizer, tokenizer);
     }
 
     if (item.itemType === "summary" && item.summaryId != null) {
-      return this.resolveSummaryItem(item);
+      return this.resolveSummaryItem(item, useTokenizer, tokenizer);
     }
 
-    // Malformed item — skip
     return null;
   }
 
-  /**
-   * Resolve a context item that references a raw message.
-   */
-  private async resolveMessageItem(item: ContextItemRecord): Promise<ResolvedItem | null> {
+  private async resolveMessageItem(
+    item: ContextItemRecord,
+    useTokenizer?: boolean,
+    tokenizer?: TokenizerService
+  ): Promise<ResolvedItem | null> {
     const msg = await this.conversationStore.getMessageById(item.messageId!);
     if (!msg) {
       return null;
@@ -737,7 +752,10 @@ export class ContextAssembler {
     const content = contentFromParts(parts, role, msg.content);
     const contentText =
       typeof content === "string" ? content : (JSON.stringify(content) ?? msg.content);
-    const tokenCount = msg.tokenCount > 0 ? msg.tokenCount : estimateTokens(contentText);
+    // Preserve short-circuit optimization: use stored tokenCount if available
+    const tokenCount = msg.tokenCount > 0
+      ? msg.tokenCount
+      : calculateTokens(contentText, useTokenizer, tokenizer);
 
     // Cast: these are reconstructed from DB storage, not live agent messages,
     // so they won't carry the full AgentMessage metadata (timestamp, usage, etc.)
@@ -775,18 +793,18 @@ export class ContextAssembler {
     };
   }
 
-  /**
-   * Resolve a context item that references a summary.
-   * Summaries are presented as user messages with a structured XML wrapper.
-   */
-  private async resolveSummaryItem(item: ContextItemRecord): Promise<ResolvedItem | null> {
+  private async resolveSummaryItem(
+    item: ContextItemRecord,
+    useTokenizer?: boolean,
+    tokenizer?: TokenizerService
+  ): Promise<ResolvedItem | null> {
     const summary = await this.summaryStore.getSummary(item.summaryId!);
     if (!summary) {
       return null;
     }
 
     const content = await formatSummaryContent(summary, this.summaryStore, this.timezone);
-    const tokens = estimateTokens(content);
+    const tokens = calculateTokens(content, useTokenizer, tokenizer);
 
     // Cast: summaries are synthetic user messages without full AgentMessage metadata
     return {