telemetry: capture tlon tool call summaries

index.ts

@@ -11,6 +11,12 @@ import { resolveBridgeForCommand } from "./src/monitor/command-auth.js";
 import { setTlonRuntime } from "./src/runtime.js";
 import { getSessionRole } from "./src/session-roles.js";
 import { recordToolCall } from "./src/telemetry.js";
+import {
+  ALLOWED_TLON_COMMANDS,
+  findTlonSubcommandIndex,
+  shellSplitCommand,
+  summarizeTlonCommand,
+} from "./src/tlon-tool-command.js";
 import { resolveTlonBinary } from "./src/tlon-binary.js";
 import { checkBlockedSendOperation } from "./src/tlon-tool-guard.js";
 import {
@@ -40,97 +46,6 @@ function readPluginVersion(): string {
   }
 }
 
-// Whitelist of allowed tlon subcommands
-const ALLOWED_TLON_COMMANDS = new Set([
-  "activity",
-  "channels",
-  "contacts",
-  "dms",
-  "expose",
-  "groups",
-  "hooks",
-  "messages",
-  "notebook",
-  "posts",
-  "settings",
-  "upload",
-  "help",
-  "version",
-]);
-
-/** Credential flags that the tlon skill binary accepts before the subcommand. */
-const CREDENTIAL_FLAGS_WITH_VALUE = new Set(["--config", "--url", "--ship", "--code", "--cookie"]);
-
-/**
- * Find the first positional argument (subcommand) by skipping credential flags
- * and their values. Returns the index into `args`, or -1 if none found.
- */
-function findSubcommandIndex(args: string[]): number {
-  let i = 0;
-  while (i < args.length) {
-    const arg = args[i];
-    // --flag=value form: skip one token
-    if (arg.startsWith("--") && arg.includes("=")) {
-      const flag = arg.slice(0, arg.indexOf("="));
-      if (CREDENTIAL_FLAGS_WITH_VALUE.has(flag)) {
-        i += 1;
-        continue;
-      }
-    }
-    // --flag value form: skip two tokens
-    if (CREDENTIAL_FLAGS_WITH_VALUE.has(arg)) {
-      i += 2;
-      continue;
-    }
-    // Not a credential flag — this is the subcommand
-    return i;
-  }
-  return -1;
-}
-
-/**
- * Shell-like argument splitter that respects quotes
- */
-function shellSplit(str: string): string[] {
-  const args: string[] = [];
-  let cur = "";
-  let inDouble = false;
-  let inSingle = false;
-  let escape = false;
-
-  for (const ch of str) {
-    if (escape) {
-      cur += ch;
-      escape = false;
-      continue;
-    }
-    if (ch === "\\" && !inSingle) {
-      escape = true;
-      continue;
-    }
-    if (ch === '"' && !inSingle) {
-      inDouble = !inDouble;
-      continue;
-    }
-    if (ch === "'" && !inDouble) {
-      inSingle = !inSingle;
-      continue;
-    }
-    if (/\s/.test(ch) && !inDouble && !inSingle) {
-      if (cur) {
-        args.push(cur);
-        cur = "";
-      }
-      continue;
-    }
-    cur += ch;
-  }
-  if (cur) {
-    args.push(cur);
-  }
-  return args;
-}
-
 /**
  * Run the tlon command and return the result
  */
@@ -290,9 +205,11 @@ export default defineChannelPluginEntry({
       },
       async execute(_id: string, params: { command: string }) {
         try {
-          const args = shellSplit(params.command);
+          const args = shellSplitCommand(params.command);
 
-          const subIdx = findSubcommandIndex(args);
+          // Skip credential flags (--config, --url, --ship, --code, --cookie)
+          // to find the actual subcommand, matching what the skill binary does.
+          const subIdx = findTlonSubcommandIndex(args);
           const subcommand = subIdx >= 0 ? args[subIdx] : undefined;
           if (!subcommand || !ALLOWED_TLON_COMMANDS.has(subcommand)) {
             return {
@@ -400,6 +317,10 @@ export default defineChannelPluginEntry({
         toolName: event.toolName,
         durationMs: event.durationMs,
         error: event.error,
+        context:
+          event.toolName === "tlon" && typeof event.params.command === "string"
+            ? summarizeTlonCommand(event.params.command)
+            : undefined,
       });
     });
 

src/telemetry.test.ts

@@ -18,6 +18,7 @@ vi.mock("posthog-node", () => ({
 }));
 
 import { _testing, createTlonTelemetry, recordToolCall } from "./telemetry.js";
+import { summarizeTlonCommand } from "./tlon-tool-command.js";
 
 describe("telemetry tool tracking", () => {
   beforeEach(() => {
@@ -129,13 +130,19 @@ describe("telemetry tool tracking", () => {
             toolName: "web_search",
             durationMs: 125,
             error: null,
+            summaryKey: null,
           },
           {
             toolName: "read",
             durationMs: null,
             error: "tool failed",
+            summaryKey: null,
           },
         ],
+        tlonToolCallCount: 0,
+        tlonToolSummaryKeys: [],
+        tlonToolChannelKinds: [],
+        tlonToolUpdateFields: [],
       }),
     });
 
@@ -156,6 +163,10 @@ describe("telemetry tool tracking", () => {
     expect(capturedEvent?.properties.toolCount).toBe(0);
     expect(capturedEvent?.properties.toolTotalDurationMs).toBe(0);
     expect(capturedEvent?.properties.toolErrorCount).toBe(0);
+    expect(capturedEvent?.properties.tlonToolCallCount).toBe(0);
+    expect(capturedEvent?.properties.tlonToolSummaryKeys).toEqual([]);
+    expect(capturedEvent?.properties.tlonToolChannelKinds).toEqual([]);
+    expect(capturedEvent?.properties.tlonToolUpdateFields).toEqual([]);
   });
 
   it("classifies reply outcomes", async () => {
@@ -246,8 +257,13 @@ describe("telemetry tool tracking", () => {
             toolName: "web_search",
             durationMs: 125,
             error: null,
+            summaryKey: null,
           },
         ],
+        tlonToolCallCount: 0,
+        tlonToolSummaryKeys: [],
+        tlonToolChannelKinds: [],
+        tlonToolUpdateFields: [],
       },
     });
 
@@ -262,6 +278,107 @@ describe("telemetry tool tracking", () => {
     expect(postHogMocks.shutdown).toHaveBeenCalledTimes(1);
   });
 
+  it("captures privacy-safe tlon command summaries", async () => {
+    const telemetry = createEnabledTelemetry();
+    const replyTelemetry = telemetry?.startReply({
+      sessionKey: "session-1",
+      ownerShip: "~zod",
+      botShip: "~nec",
+      chatType: "dm",
+      isThreadReply: false,
+      senderRole: "owner",
+      attachmentCount: 0,
+    });
+
+    recordToolCall({
+      sessionKey: "session-1",
+      toolName: "tlon",
+      durationMs: 80,
+      context: summarizeTlonCommand(
+        "groups invite ~zod/quiet-launch ~sampel-palnet ~marzod-marnec",
+      ),
+    });
+    recordToolCall({
+      sessionKey: "session-1",
+      toolName: "tlon",
+      durationMs: 40,
+      context: summarizeTlonCommand(
+        'contacts update-profile --nickname "PM Bot" --avatar https://assets.example.com/private.png',
+      ),
+    });
+    recordToolCall({
+      sessionKey: "session-1",
+      toolName: "tlon",
+      durationMs: 30,
+      context: summarizeTlonCommand(
+        'groups add-channel ~zod/quiet-launch "Photos" --kind heap',
+      ),
+    });
+
+    await replyTelemetry?.capture({
+      deliveredMessageCount: 1,
+      replyCharCount: 42,
+      replyWordCount: 7,
+      replyMediaCount: 0,
+      dispatchDurationMs: 250,
+      queuedFinal: false,
+      queuedFinalCount: 1,
+      queuedBlockCount: 0,
+      provider: "anthropic",
+      model: "claude-test",
+      thinkLevel: null,
+    });
+
+    expect(postHogMocks.capture).toHaveBeenCalledWith({
+      distinctId: "~zod",
+      event: "TlonBot Reply Handled",
+      properties: expect.objectContaining({
+        tlonToolCallCount: 3,
+        tlonToolSummaryKeys: [
+          "groups.invite",
+          "contacts.update-profile",
+          "groups.add-channel",
+        ],
+        tlonToolChannelKinds: ["heap"],
+        tlonToolUpdateFields: ["nickname", "avatar"],
+        toolCalls: [
+          {
+            toolName: "tlon",
+            durationMs: 80,
+            error: null,
+            summaryKey: "groups.invite",
+          },
+          {
+            toolName: "tlon",
+            durationMs: 40,
+            error: null,
+            summaryKey: "contacts.update-profile",
+          },
+          {
+            toolName: "tlon",
+            durationMs: 30,
+            error: null,
+            summaryKey: "groups.add-channel",
+          },
+        ],
+      }),
+    });
+
+    const capturedEvent = postHogMocks.capture.mock.calls.at(-1)?.[0];
+    expect(JSON.stringify(capturedEvent?.properties)).not.toContain(
+      "~zod/quiet-launch",
+    );
+    expect(JSON.stringify(capturedEvent?.properties)).not.toContain(
+      "~sampel-palnet",
+    );
+    expect(JSON.stringify(capturedEvent?.properties)).not.toContain("PM Bot");
+    expect(JSON.stringify(capturedEvent?.properties)).not.toContain(
+      "https://assets.example.com/private.png",
+    );
+
+    await telemetry?.close();
+  });
+
   describe("captureHeartbeatNudge", () => {
     it("emits correct PostHog event with all properties", () => {
       const telemetry = createEnabledTelemetry()!;