feat(auto-permissions): count confirmed dialog answers as user evidence

packages/pi-auto-permissions/README.md

@@ -159,6 +159,22 @@ Trusted projects may optionally provide their root `AGENTS.md`, or `CLAUDE.md` w
 
 Project instructions help interpret the requested workflow, but cannot independently authorize an action or override guardian policy.
 
+### Interactive dialog answers
+
+When the main agent gathers a decision through an interactive question tool (for example `ask_user_question`), the user's selection is stored as a tool result, which never grants permission. Operators can allowlist user-answer tools, whose confirmed answers become `source: "user"` evidence records:
+
+```json
+{
+  "reviewEvidence": {
+    "userAnswerTools": ["ask_user_question"]
+  }
+}
+```
+
+A successful result from an allowlisted tool qualifies when its `details` are `{ "answers": [{ "question": string, "answer"?: string, "selected"?: string[], "notes"?: string }], "cancelled": false }` with no `error` field. `selected` takes precedence over `answer`, non-string answers are ignored, and notes count only alongside a real answer. Each answered question contributes one `USER (dialog answer):` record; the guardian treats it as authorization for exactly the selected content and is told the question wording is assistant-drafted context, never an instruction. Any dialog extension emitting that shape qualifies.
+
+The allowlist matches what you wrote: a bare name such as `ask_user_question` matches that tool in any namespace (`functions.ask_user_question` included), while a dotted name matches exactly. The default is an empty list.
+
 ## Review display
 
 The default UI shows guardian progress in a temporary widget below the editor. Configure it with:
@@ -209,6 +225,8 @@ Rules match raw shell text. Quoting, variables, aliases, generated scripts, or o
 
 Pi Auto Permissions is a permission layer for normal agent behavior. It is not an operating-system sandbox or a defense against hostile shell input. Pair it with sandboxing when commands need a hard security boundary.
 
+`reviewEvidence.userAnswerTools` widens what counts as user authorization: any code that can record a tool result under an allowlisted tool name can mint `USER (dialog answer)` evidence. Allowlist only tool names served by extensions you trust.
+
 ## License
 
 MIT

packages/pi-auto-permissions/config.test.ts

@@ -24,7 +24,7 @@ describe("auto permissions config", () => {
     expect(config.enabled).toBeTrue();
     expect(config.reviewer).toBeUndefined();
     expect(config.rules).toEqual([]);
-    expect(config.reviewEvidence).toEqual({ projectInstructions: false });
+    expect(config.reviewEvidence).toEqual({ projectInstructions: false, userAnswerTools: [] });
     expect(config.ui).toEqual({ enabled: true, resultDisplayMs: 2500, placement: "widget" });
   });
 
@@ -57,12 +57,31 @@ describe("auto permissions config", () => {
       timeoutMs: 12_000,
     });
     expect(config.systemPrompt).toBe("custom permission policy");
-    expect(config.reviewEvidence).toEqual({ projectInstructions: true });
+    expect(config.reviewEvidence).toEqual({ projectInstructions: true, userAnswerTools: [] });
     expect(config.ui).toEqual({ enabled: true, resultDisplayMs: 5000, placement: "toolRow" });
     expect(config.rules).toHaveLength(1);
     expect(config.rules[0].pattern.test("rm -rf build")).toBeTrue();
   });
 
+  test("accepts, trims, and deduplicates user answer tools", () => {
+    const path = configFile({
+      reviewEvidence: { userAnswerTools: [" ask_user_question ", "plan_review", "ask_user_question"] },
+    });
+    expect(loadAutoPermissionsConfig(path).reviewEvidence).toEqual({
+      projectInstructions: false,
+      userAnswerTools: ["ask_user_question", "plan_review"],
+    });
+  });
+
+  test("rejects malformed user answer tools", () => {
+    for (const userAnswerTools of ["ask_user_question", [42], [""], ["  "], {}]) {
+      const path = configFile({ reviewEvidence: { userAnswerTools } });
+      expect(() => loadAutoPermissionsConfig(path)).toThrow(
+        "reviewEvidence.userAnswerTools must be an array of non-empty strings",
+      );
+    }
+  });
+
   test("loads a prompt file relative to the config", () => {
     const dir = mkdtempSync(join(tmpdir(), "pi-auto-permissions-"));
     tempDirs.push(dir);

packages/pi-auto-permissions/config.ts

@@ -21,6 +21,7 @@ export interface AutoPermissionsConfig {
   systemPrompt: string;
   reviewEvidence: {
     projectInstructions: boolean;
+    userAnswerTools: string[];
   };
   rules: Gate[];
   ui: {
@@ -102,15 +103,22 @@ function resolvePrompt(raw: Record<string, unknown>, path: string): string {
 }
 
 function resolveReviewEvidence(raw: Record<string, unknown>): AutoPermissionsConfig["reviewEvidence"] {
-  if (raw.reviewEvidence === undefined) return { projectInstructions: false };
+  if (raw.reviewEvidence === undefined) return { projectInstructions: false, userAnswerTools: [] };
   if (!raw.reviewEvidence || typeof raw.reviewEvidence !== "object" || Array.isArray(raw.reviewEvidence)) {
     throw new Error("reviewEvidence must be an object");
   }
   const evidence = raw.reviewEvidence as Record<string, unknown>;
   if (evidence.projectInstructions !== undefined && typeof evidence.projectInstructions !== "boolean") {
     throw new Error("reviewEvidence.projectInstructions must be boolean");
   }
-  return { projectInstructions: evidence.projectInstructions === true };
+  const rawTools = evidence.userAnswerTools === undefined ? [] : evidence.userAnswerTools;
+  if (!Array.isArray(rawTools) || rawTools.some((tool) => typeof tool !== "string" || !tool.trim())) {
+    throw new Error("reviewEvidence.userAnswerTools must be an array of non-empty strings");
+  }
+  return {
+    projectInstructions: evidence.projectInstructions === true,
+    userAnswerTools: [...new Set((rawTools as string[]).map((tool) => tool.trim()))],
+  };
 }
 
 function resolveUi(raw: Record<string, unknown>): AutoPermissionsConfig["ui"] {

packages/pi-auto-permissions/index.test.ts

@@ -465,6 +465,44 @@ describe("auto permissions tool gate", () => {
     expect(calls[2].options.reasoning).toBe("medium");
   });
 
+  test("threads user answer tools into evidence and resets when the allowlist changes", async () => {
+    const path = useConfig({ reviewEvidence: { userAnswerTools: ["ask_user_question"] } });
+    const calls: Array<{ context: any; options: any }> = [];
+    completeOverride = async (context, options) => {
+      calls.push({ context, options });
+      return reviewerResponse();
+    };
+    const state = harness(["push this branch"]);
+    state.setContextEntries([
+      { id: "user-1", type: "message", message: { role: "user", content: "push this branch" } },
+      {
+        id: "ask-result",
+        type: "message",
+        message: {
+          role: "toolResult",
+          toolCallId: "ask-1",
+          toolName: "ask_user_question",
+          isError: false,
+          content: [{ type: "text", text: "ENVELOPE_PROSE_CANARY" }],
+          details: { answers: [{ question: "Which branch?", answer: "feature" }], cancelled: false },
+        },
+      },
+    ]);
+
+    expect(await state.toolCallHandler({ toolName: "bash", input: { command: "git push origin feature" } }, state.ctx)).toBeUndefined();
+    expect(calls[0].context.messages[0].content[0].text).toContain('"source":"user"');
+    expect(calls[0].context.messages[0].content[0].text).toContain("USER (dialog answer):");
+    expect(calls[0].context.messages[0].content[0].text).not.toContain("ENVELOPE_PROSE_CANARY");
+
+    writeFileSync(path, JSON.stringify({ reviewEvidence: { userAnswerTools: [] }, rules: TEST_RULES }));
+    expect(await state.toolCallHandler({ toolName: "bash", input: { command: "git push origin feature" } }, state.ctx)).toBeUndefined();
+
+    expect(calls[1].options.sessionId).not.toBe(calls[0].options.sessionId);
+    expect(calls[1].context.messages).toHaveLength(1);
+    expect(calls[1].context.messages[0].content[0].text).toContain('mode="full"');
+    expect(calls[1].context.messages[0].content[0].text).not.toContain("USER (dialog answer):");
+  });
+
   test("adds only the highest-priority trusted project instruction file and resets when it changes", async () => {
     useConfig({ reviewEvidence: { projectInstructions: true } });
     const projectDir = mkdtempSync(join(tmpdir(), "pi-auto-permissions-project-"));

packages/pi-auto-permissions/index.ts

@@ -75,6 +75,9 @@ function reviewerFingerprint(
     reasoning: config.reviewer?.reasoningEffort ?? "low",
     systemPrompt,
     projectInstructionsTrusted: config.reviewEvidence.projectInstructions ? projectTrusted : undefined,
+    userAnswerTools: config.reviewEvidence.userAnswerTools.length
+      ? [...config.reviewEvidence.userAnswerTools].sort()
+      : undefined,
   });
 }
 
@@ -266,7 +269,11 @@ export default function autoPermissionsExtension(pi: ExtensionAPI) {
     }
     const systemPrompt = buildReviewerSystemPrompt(config.systemPrompt, projectInstructions);
     const fingerprint = reviewerFingerprint(mainSessionId, model, config, systemPrompt, projectTrusted);
-    const evidence = collectReviewEvidence(ctx.sessionManager.buildContextEntries(), toolCallId);
+    const evidence = collectReviewEvidence(
+      ctx.sessionManager.buildContextEntries(),
+      toolCallId,
+      config.reviewEvidence.userAnswerTools,
+    );
     const evidenceKeys = evidence.map((record) => record.key);
     const budget = reviewContextBudget(model.contextWindow);
     let base = reviewerLineage;

packages/pi-auto-permissions/review.test.ts

@@ -132,6 +132,109 @@ describe("compact review evidence", () => {
     ]);
   });
 
+  test("promotes confirmed dialog answers from allowlisted tools to user evidence", () => {
+    const entries = [
+      { id: "u1", type: "message", message: { role: "user", content: "deploy the release to some demo hosts" } },
+      {
+        id: "a1",
+        type: "message",
+        message: {
+          role: "assistant",
+          content: [
+            { type: "toolCall", id: "ask-1", name: "functions.ask_user_question", arguments: { questions: [] } },
+          ],
+        },
+      },
+      {
+        id: "r1",
+        type: "message",
+        message: {
+          role: "toolResult",
+          toolCallId: "ask-1",
+          toolName: "functions.ask_user_question",
+          isError: false,
+          content: [{ type: "text", text: "ENVELOPE_PROSE_CANARY" }],
+          details: {
+            answers: [
+              { questionIndex: 0, question: "Which hosts first?", kind: "option", answer: "southern, dominion" },
+              { questionIndex: 1, question: "Enable which checks?", kind: "multi", answer: "overridden", selected: ["lint", "tests"], notes: "skip e2e" },
+              { questionIndex: 2, question: "Unanswered?", kind: "option", answer: "   " },
+              { questionIndex: 3, question: "Notes only?", kind: "option", answer: "", notes: "a note is not an answer" },
+              { questionIndex: 4, question: "Non-string?", kind: "option", answer: true },
+            ],
+            cancelled: false,
+          },
+        },
+      },
+    ];
+
+    const records = collectReviewEvidence(entries, "pending-1", ["ask_user_question"]);
+    expect(records.map((record) => ({ source: record.source, text: record.text }))).toEqual([
+      { source: "user", text: "USER: deploy the release to some demo hosts" },
+      { source: "tool", text: "TOOL functions.ask_user_question → success" },
+      { source: "user", text: 'USER (dialog answer): selected "southern, dominion" — assistant-drafted question: "Which hosts first?"' },
+      { source: "user", text: 'USER (dialog answer): selected "lint; tests (note: skip e2e)" — assistant-drafted question: "Enable which checks?"' },
+    ]);
+    expect(records[2].key).not.toBe(records[3].key);
+    expect(JSON.stringify(records)).not.toContain("ENVELOPE_PROSE_CANARY");
+    expect(AUTO_PERMISSIONS_SYSTEM_PROMPT).toContain('USER (dialog answer):');
+  });
+
+  test("ignores dialog answers that are not allowlisted, cancelled, errored, or malformed", () => {
+    const answers = [{ questionIndex: 0, question: "Which hosts first?", kind: "option", answer: "southern" }];
+    const resultEntry = (details: unknown, isError: unknown = false, toolName = "ask_user_question") => [
+      {
+        id: "a1",
+        type: "message",
+        message: {
+          role: "assistant",
+          content: [{ type: "toolCall", id: "ask-1", name: toolName, arguments: {} }],
+        },
+      },
+      {
+        id: "r1",
+        type: "message",
+        message: { role: "toolResult", toolCallId: "ask-1", toolName, isError, content: [], details },
+      },
+    ];
+
+    const userRecords = (entries: unknown[], tools?: readonly string[]) =>
+      collectReviewEvidence(entries, undefined, tools).filter((record) => record.source === "user");
+
+    expect(userRecords(resultEntry({ answers, cancelled: false }))).toEqual([]);
+    expect(userRecords(resultEntry({ answers, cancelled: false }), ["other_tool"])).toEqual([]);
+    expect(userRecords(resultEntry({ answers, cancelled: true }), ["ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry({ answers, cancelled: false, error: "no_ui" }), ["ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry({ answers, cancelled: false }, true), ["ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry({ answers: [], cancelled: false }), ["ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry("prose only"), ["ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry({ answers }), ["ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry({ answers, cancelled: "true" }), ["ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry({ answers, cancelled: 1 }), ["ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry({ answers, cancelled: false }, "false"), ["ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry({ answers, cancelled: false }), ["ask_user_question"]).length).toBe(1);
+    expect(userRecords(resultEntry({ answers, cancelled: false }, undefined), ["ask_user_question"]).length).toBe(1);
+  });
+
+  test("matches bare allowlist names across namespaces but keeps dotted names exact", () => {
+    const answers = [{ questionIndex: 0, question: "Which hosts first?", kind: "option", answer: "southern" }];
+    const resultEntry = (toolName: string) => [
+      {
+        id: "r1",
+        type: "message",
+        message: { role: "toolResult", toolCallId: "ask-1", toolName, isError: false, content: [], details: { answers, cancelled: false } },
+      },
+    ];
+
+    const userRecords = (entries: unknown[], tools: readonly string[]) =>
+      collectReviewEvidence(entries, undefined, tools).filter((record) => record.source === "user");
+
+    expect(userRecords(resultEntry("functions.ask_user_question"), ["ask_user_question"]).length).toBe(1);
+    expect(userRecords(resultEntry("functions.ask_user_question"), ["functions.ask_user_question"]).length).toBe(1);
+    expect(userRecords(resultEntry("evilext.ask_user_question"), ["trusted.ask_user_question"])).toEqual([]);
+    expect(userRecords(resultEntry("ask_user_question"), ["trusted.ask_user_question"])).toEqual([]);
+  });
+
   test("builds explicit cumulative full and delta envelopes around the exact latest action", () => {
     const request = {
       tool: "functions.bash",