Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions surfaces/cli/src/features/health.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1859,3 +1859,76 @@ describe("doctor unknown target", () => {
}
});
});

describe("doctor memory extraction privacy gate", () => {
const remoteRoute = (privacy: string) =>
[
"inference:",
" defaultPolicy: background",
" targets:",
" background:",
" executor: openai-compatible",
" endpoint: https://open.bigmodel.cn/api/coding/paas/v4",
" models:",
" default:",
" model: glm-5.3-flash",
" policies:",
" background:",
" mode: automatic",
" defaultTargets: [background/default]",
" fallbackTargets: [background/default]",
" taskClasses:",
" memory_extraction:",
" reasoning: medium",
" toolsRequired: true",
` privacy: ${privacy}`,
" workloads:",
" memoryExtraction:",
" target: background/default",
" taskClass: memory_extraction",
"",
].join("\n");

async function doctorFindings(agentYaml: string): Promise<Array<{ code?: string; message?: string; fix?: string }>> {
const root = mkdtempSync(join(tmpdir(), "health-extraction-privacy-"));
const lines: string[] = [];
const oldLog = console.log;
try {
writeFileSync(join(root, "agent.yaml"), agentYaml);
console.log = (...args: unknown[]) => {
lines.push(args.join(" "));
};
await showDoctor(
{ json: true },
{
...depsFor(root),
detectInstallations: () => ({
target: { kind: "unsupported", executablePath: "/tmp/signet", reason: "test fixture" },
installations: [],
inactive: [],
}),
},
);
return (JSON.parse(lines.join("\n")) as { findings: Array<{ code?: string; message?: string; fix?: string }> })
.findings;
} finally {
console.log = oldLog;
rmSync(root, { recursive: true, force: true });
}
}

it("warns when a remote extraction target can never satisfy the task class privacy tier", async () => {
expect(await doctorFindings(remoteRoute("restricted_remote"))).toContainEqual(
expect.objectContaining({
code: "memory_extraction_privacy_blocked",
message: expect.stringContaining("privacy gate (restricted_remote) for background/default"),
fix: expect.stringContaining("remote_ok"),
}),
);
});

it("stays quiet once remote extraction is allowed", async () => {
const findings = await doctorFindings(remoteRoute("remote_ok"));
expect(findings.map((finding) => finding.code)).not.toContain("memory_extraction_privacy_blocked");
});
});
41 changes: 41 additions & 0 deletions surfaces/cli/src/features/health.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,15 @@ import {
detectSignetInstallations,
getMissingIdentityFiles,
hasValidIdentity,
allTargetRefs,
inactivePackageManagerInstallations,
loadIdentityMode,
parseRoutingConfig,
parseYamlDocument,
resolveRoutingDecision,
resolveWorkspaceLayout,
} from "@signet/core";
import { existsSync, readFileSync } from "node:fs";
import { join } from "node:path";
import chalk from "chalk";
import { daemonAccessLines } from "../lib/network.js";
Expand Down Expand Up @@ -818,6 +823,41 @@ function addReadinessFindings(report: StatusReport, findings: DoctorFinding[]):
fix: "Inspect the queue in `signet status`; repair only identified jobs with `signet repair queue requeue --apply` or retire obsolete jobs with `signet repair queue cancel --apply`.",
});
}
function addExtractionPrivacyFindings(basePath: string, findings: DoctorFinding[]): void {
const path = join(basePath, "agent.yaml");
if (!existsSync(path)) return;
let parsed: ReturnType<typeof parseRoutingConfig>;
try {
parsed = parseRoutingConfig(parseYamlDocument(readFileSync(path, "utf-8")));
} catch {
return;
}
if (!parsed.ok || !parsed.value.workloads?.memoryExtraction) return;
const config = parsed.value;
const targets = Object.fromEntries(
allTargetRefs(config).map((ref) => [
ref,
{ available: true, health: "healthy" as const, circuitOpen: false, accountState: "ready" as const },
]),
);
const decision = resolveRoutingDecision(config, { operation: "memory_extraction" }, { targets });
if (decision.ok) return;
const trace = decision.error.details?.trace as
| { readonly candidates?: readonly { readonly targetRef: string; readonly blockedBy: readonly string[] }[] }
| undefined;
const gated = (trace?.candidates ?? []).filter((candidate) =>
candidate.blockedBy.some((reason) => reason.startsWith("privacy gate")),
);
if (gated.length === 0) return;
const reason = gated[0]?.blockedBy.find((entry) => entry.startsWith("privacy gate")) ?? "privacy gate";
findings.push({
level: "warn",
code: "memory_extraction_privacy_blocked",
message: `Dreaming cannot run: memory extraction is blocked by the ${reason} for ${gated.map((candidate) => candidate.targetRef).join(", ")}.`,
fix: "To send memory and transcript text to this provider, choose Use remotely under Settings → Inference in the dashboard, or set inference.taskClasses.memory_extraction.privacy to remote_ok in agent.yaml. Otherwise point memory extraction at a local target.",
});
}

function addDaemonLifecycleExitFindings(
probe: NonNullable<DaemonStatus["probe"]>,
findings: DoctorFinding[],
Expand Down Expand Up @@ -1082,6 +1122,7 @@ function getDoctorFindings(report: StatusReport, installations: SignetInstallati
addPhysicalMemoryFinding(report, findings);
addReadinessFindings(report, findings);
addQueueBacklogFindings(report, findings);
if (hasAgentYaml) addExtractionPrivacyFindings(report.basePath, findings);

if (report.openclawWorkspaceUnprotected) {
findings.push({
Expand Down
60 changes: 59 additions & 1 deletion surfaces/cli/src/features/setup-pipeline.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { describe, expect, it } from "bun:test";
import { MODEL_DEFAULTS } from "@signet/core";
import { MODEL_DEFAULTS, allTargetRefs, parseRoutingConfig, resolveRoutingDecision } from "@signet/core";
import {
applyAggregateRecallRoute,
applySetupInferenceRoute,
Expand All @@ -8,8 +8,21 @@ import {
buildSetupPipeline,
defaultAcpxModel,
defaultExtractionModel,
type SetupInferenceConfig,
} from "./setup-pipeline";

function routeMemoryExtraction(inference: SetupInferenceConfig | undefined) {
const config = parseRoutingConfig({ inference });
if (!config.ok) throw new Error(config.error.message);
const targets = Object.fromEntries(
allTargetRefs(config.value).map((ref) => [
ref,
{ available: true, health: "healthy" as const, circuitOpen: false, accountState: "ready" as const },
]),
);
return resolveRoutingDecision(config.value, { operation: "memory_extraction" }, { targets });
}

describe("defaultExtractionModel", () => {
it("uses the checked Codex CLI model default", () => {
expect(defaultExtractionModel("codex")).toBe(MODEL_DEFAULTS.codex);
Expand Down Expand Up @@ -69,6 +82,40 @@ describe("buildSetupPipeline", () => {
});
});

it("records remote extraction consent on the task class for remote targets", () => {
expect(buildSetupInference("openrouter", "anthropic/claude-haiku")?.taskClasses.memory_extraction).toMatchObject({
privacy: "remote_ok",
});
expect(
buildSetupInference("openai-compatible", "glm-5.3-flash", [], [], undefined, "https://open.bigmodel.cn/api/v4")
?.taskClasses.memory_extraction,
).toMatchObject({ privacy: "remote_ok" });
expect(
buildSetupInference("openai-compatible", "local-model", [], [], undefined, "http://127.0.0.1:1234/v1")
?.taskClasses.memory_extraction,
).toMatchObject({ privacy: "restricted_remote" });
expect(buildSetupInference("ollama", "qwen3:4b")?.taskClasses.memory_extraction).toMatchObject({
privacy: "restricted_remote",
});
});

it("writes a memory extraction route the router can resolve for every provider", () => {
const routes = [
buildSetupInference("openrouter", "anthropic/claude-haiku"),
buildSetupInference("openai-compatible", "glm-5.3-flash", [], [], undefined, "https://open.bigmodel.cn/api/v4"),
buildSetupInference("openai-compatible", "local-model", [], [], undefined, "http://localhost:1234/v1"),
buildSetupInference("ollama", "qwen3:4b"),
buildSetupInference("llama-cpp", "qwen3:4b"),
buildSetupInference("acpx", "haiku", ["claude-code"], ["acpx"], "/usr/local/bin/bunx"),
];
for (const route of routes) {
const decision = routeMemoryExtraction(route);
expect(decision.ok ? decision.value.targetRef : decision.error.details?.trace).toBe(
route?.defaultPolicy === "background-acpx" ? "background-acpx/default" : "background/default",
);
}
});

it("preserves the configured endpoint for OpenAI-compatible extraction", () => {
expect(
buildSetupInference("openai-compatible", "local-model", [], [], undefined, "http://gw:8000/v1")?.targets
Expand Down Expand Up @@ -175,6 +222,17 @@ describe("buildSetupInference", () => {
expect(config).not.toHaveProperty("inference");
});

it("removes the generated remote task class when setup switches extraction off", () => {
const config: Record<string, unknown> = {
inference: buildSetupInference("openrouter", "anthropic/claude-haiku"),
};

applySetupInferenceRoute(config, undefined);

expect(config.inference).not.toHaveProperty("taskClasses");
expect(config.inference).not.toHaveProperty("workloads");
});

it("preserves aggregate recall when removing generated direct routing", () => {
const config: Record<string, unknown> = {
inference: {
Expand Down
20 changes: 16 additions & 4 deletions surfaces/cli/src/features/setup-pipeline.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { defaultPipelineModel } from "@signet/core";
import { defaultPipelineModel, isLocalInferenceEndpoint } from "@signet/core";
import type { ExtractionProviderChoice, HarnessChoice } from "./setup-shared.js";

export const EXTRACTION_SAFETY_WARNING =
Expand Down Expand Up @@ -161,14 +161,18 @@ export function buildSetupInference(
models: { default: { model: resolved, reasoning: "medium" } },
};
let accounts: Record<string, unknown> | undefined;
let remote = false;
if (provider === "openai-compatible") {
target.endpoint = endpoint?.trim() || "http://127.0.0.1:1234/v1";
const resolvedEndpoint = endpoint?.trim() || "http://127.0.0.1:1234/v1";
target.endpoint = resolvedEndpoint;
remote = !isLocalInferenceEndpoint(resolvedEndpoint);
}
if (provider === "openrouter") {
target.account = "extraction";
accounts = {
extraction: { kind: "api", providerFamily: "openrouter", credentialRef: "OPENROUTER_API_KEY" },
};
remote = true;
}
const targetRef = "background/default";
return {
Expand All @@ -183,7 +187,11 @@ export function buildSetupInference(
},
},
taskClasses: {
memory_extraction: { reasoning: "medium", toolsRequired: true, privacy: "restricted_remote" },
memory_extraction: {
reasoning: "medium",
toolsRequired: true,
privacy: remote ? "remote_ok" : "restricted_remote",
},
},
workloads: {
memoryExtraction: { target: targetRef, taskClass: "memory_extraction" },
Expand Down Expand Up @@ -236,7 +244,11 @@ export function applySetupInferenceRoute(
function isGeneratedSetupTaskClass(value: unknown): boolean {
if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
const record = value as { reasoning?: unknown; toolsRequired?: unknown; privacy?: unknown };
return record.reasoning === "medium" && record.toolsRequired === true && record.privacy === "restricted_remote";
return (
record.reasoning === "medium" &&
record.toolsRequired === true &&
(record.privacy === "restricted_remote" || record.privacy === "remote_ok")
);
}

function isGeneratedSetupWorkload(value: unknown, target: string): boolean {
Expand Down
6 changes: 3 additions & 3 deletions web/docs/src/content/docs/benchmarking.md
Original file line number Diff line number Diff line change
Expand Up @@ -263,9 +263,9 @@ a fresh non-interactive install cannot reach a remote model otherwise:

- `account`: a reference to `SIGNET_BENCH_DREAMING_API_KEY`, which the wrapper
fills from `ZAI_API_KEY`. Setup writes no credential for a remote endpoint.
- `privacy: restricted_remote`: setup's `memory_extraction` task class requires
it, and a remote target is otherwise inferred as `remote_ok`, which the
router blocks.
- `privacy: restricted_remote`: setup keeps the `memory_extraction` task class
at `restricted_remote` when it is given a local endpoint, and the replaced
executor is otherwise inferred as `remote_ok`, which the router blocks.
- `executor`: set to the catalog provider family (default `zai-coding-cn`)
with no endpoint, matching how an installed Z.ai target is configured. The
generic OpenAI-compatible executor appends `/v1` to the Z.ai `/v4` base URL.
Expand Down
2 changes: 2 additions & 0 deletions web/docs/src/content/docs/cli/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ When an explicitly selected Ollama service or model is unavailable during non-in

Validated extraction providers are `acpx`, `claude-code`, `codex`, `llama-cpp`, `ollama`, `opencode`, `openrouter`, `openai-compatible`, and `none`. Explicit provider flags override defaults inferred from `--deployment-type`.

Choosing `openrouter`, or `openai-compatible` with a non-loopback `--extraction-endpoint`, sends memory and transcript text to that provider. Setup records that choice as `privacy: remote_ok` on the `memory_extraction` task class. Local providers and ACPX keep `restricted_remote`, so Settings → Inference in the dashboard asks before memory extraction is moved to a remote provider. It also asks for an existing ACPX target, because the dashboard treats an ACPX harness as a remote executor. `signet doctor` warns when the configured extraction target can never pass that privacy tier.

Current `signet setup --help` lists `llama-cpp` as an embedding provider, but setup validation rejects it. Use the validated embedding values above until that discrepancy is fixed.

### Automation, safety, and data flags
Expand Down
Loading