Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ harnesses agents sync --check # doctor: link global AGENTS.md files to one mas
harnesses mcp # run the MCP server over stdio
```

`tools` defaults to `false` in the library and CLI. The MCP, Pi, and OMP tools require agents to choose it explicitly. `false` must use a native CLI flag that removes tools from the model context; it is a lightweight advisor, not an agent constrained only by prompt wording. Set `tools: true` (or CLI `--tools`) whenever the task needs harness tools, including Grok's native X search. Add `readOnly: true` when those tools must stay inside a sandbox enforced by the harness CLI; the agent tools pass it beside `tools: true`, while the library and CLI let it imply tools. Read-only mode is rejected when a harness has no verified native recipe, so it never falls back to broader access. Harnesses whose CLI cannot disable tools reject advisor mode instead of silently running an agent and return an explicit `tools` retry when their full agent mode can handle the request.
`tools` defaults to `false` in the library and CLI. The MCP, Pi, and OMP tools require agents to choose it explicitly. `false` must use a native CLI flag that removes tools from the model context; it is a lightweight advisor, not an agent constrained only by prompt wording. Set `tools: true` (or CLI `--tools`) whenever the task needs harness tools, including Grok's native X search. Add `readOnly: true` when those tools must stay inside a sandbox enforced by the harness CLI; the agent tools pass it beside `tools: true`, while the library and CLI let it imply tools. Read-only mode is rejected when a harness has no verified native recipe, so it never falls back to broader access. A recipe can also carry the lowest CLI version whose enforcement was verified, and `invoke()` rejects read-only runs on older or unknown versions: Grok runs `--sandbox read-only` from 1.0.13. Harnesses whose CLI cannot disable tools reject advisor mode instead of silently running an agent and return an explicit `tools` retry when their full agent mode can handle the request.

## How harnesses compares to unagent

Expand Down
21 changes: 18 additions & 3 deletions docs/app/data/harnesses.json
Original file line number Diff line number Diff line change
Expand Up @@ -1185,18 +1185,33 @@
"--output-format",
"json"
],
"readOnlyArgs": [
"-p",
"{prompt}",
"--sandbox",
"read-only"
],
"readOnlyJsonArgs": [
"-p",
"{prompt}",
"--sandbox",
"read-only",
"--output-format",
"json"
],
"readOnlyMinVersion": "1.0.13",
"modelArgs": [
"--model",
"{model}"
],
"level": "official",
"note": "-p is short for --single; add --output-format json for structured output."
"note": "-p is short for --single; add --output-format json for structured output. --sandbox read-only is kernel-enforced (Landlock, Seatbelt) regardless of the inherited permission mode and still allows writes to ~/.grok and temp dirs; verified on Linux with 1.0.13 and 1.0.25."
},
"invocationModes": {
"advisor": false,
"advisorStructured": false,
"readOnly": false,
"readOnlyStructured": false,
"readOnly": true,
"readOnlyStructured": true,
"agent": true,
"agentStructured": true
},
Expand Down
1 change: 1 addition & 0 deletions docs/app/utils/harnesses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export interface Invocation {
noToolsJsonArgs?: string[];
readOnlyArgs?: string[];
readOnlyJsonArgs?: string[];
readOnlyMinVersion?: string;
modelArgs?: string[];
level: EvidenceLevel;
note?: string;
Expand Down
39 changes: 39 additions & 0 deletions src/harness.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,25 @@ function invocationRetryHint(
return alternateAvailable ? (INVOCATION_RETRY_HINT[mode] ?? "") : "";
}

/**
* Orders two dotted versions numerically; a pre-release tag sorts below its release.
*
* @param a - Left version.
* @param b - Right version.
* @returns {number} Negative when a is older, positive when newer, zero when equal.
*/
function compareVersions(a: string, b: string): number {
const [aBase = "", aPre] = a.split("-", 2);
const [bBase = "", bPre] = b.split("-", 2);
const left = aBase.split(".").map(Number);
const right = bBase.split(".").map(Number);
for (let i = 0; i < Math.max(left.length, right.length); i++) {
const diff = (left[i] ?? 0) - (right[i] ?? 0);
if (diff !== 0) return diff;
}
return Number(aPre === undefined) - Number(bPre === undefined);
}

function buildInvocationArgs(
template: readonly string[],
prompt: string,
Expand Down Expand Up @@ -402,10 +421,30 @@ export abstract class Harness {
new Error(this.invocationError(invocationOptions) ?? "Invalid invocation"),
);
}
const versionError = this.readOnlyVersionError(invocationOptions);
if (versionError) return Promise.reject(new Error(versionError));

return executeCommand(built.command, built.args, options);
}

/**
* Explains why the installed CLI cannot run a read-only recipe that carries a
* version floor, or returns null when no floor applies or the installed
* version meets it. An unknown version fails closed, like a missing recipe.
*
* @param options - Requested execution mode.
* @returns {string | null} The version incompatibility, or null when the run may proceed.
*/
private readOnlyVersionError(options: InvocationOptions): string | null {
const floor = this.invocation?.readOnlyMinVersion;
if (floor === undefined || options.readOnly !== true) return null;
const installed = this.version;
const error = `Harness ${this.id} requires version ${floor} or newer for read-only runs`;
if (installed === null) return `${error}; installed version unknown`;
if (compareVersions(installed, floor) < 0) return `${error}; installed ${installed}`;
return null;
}

/**
* Expands the native model-listing recipe without spawning anything.
*
Expand Down
5 changes: 4 additions & 1 deletion src/harnesses/grok.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,12 @@ export default class Grok extends Harness {
readonly invocation: Harness["invocation"] = {
args: ["-p", "{prompt}"],
jsonArgs: ["-p", "{prompt}", "--output-format", "json"],
readOnlyArgs: ["-p", "{prompt}", "--sandbox", "read-only"],
readOnlyJsonArgs: ["-p", "{prompt}", "--sandbox", "read-only", "--output-format", "json"],
readOnlyMinVersion: "1.0.13",
modelArgs: ["--model", "{model}"],
level: "official",
note: "-p is short for --single; add --output-format json for structured output.",
note: "-p is short for --single; add --output-format json for structured output. --sandbox read-only is kernel-enforced (Landlock, Seatbelt) regardless of the inherited permission mode and still allows writes to ~/.grok and temp dirs; verified on Linux with 1.0.13 and 1.0.25.",
};
override readonly mcpConfigs: Harness["mcpConfigs"] = [
{
Expand Down
2 changes: 2 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ export interface HarnessInvocation {
readOnlyArgs?: string[];
/** Structured agent argument template with read-only tool access. */
readOnlyJsonArgs?: string[];
/** Lowest CLI version whose read-only enforcement was verified; older or unknown versions reject read-only runs. */
readOnlyMinVersion?: string;
/** Arguments appended when a model is selected; every "{model}" is replaced. */
modelArgs?: string[];
level: EvidenceLevel;
Expand Down
73 changes: 73 additions & 0 deletions test/invoke.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,20 @@ describe("normalized invocation", () => {
});
});

it("runs Grok read-only jobs inside its native sandbox", () => {
const grok = getHarness("grok");

expect(grok.invocationError({ readOnly: true })).toBeNull();
expect(grok.buildInvocation("review this", { readOnly: true })).toEqual({
command: "grok",
args: ["-p", "review this", "--sandbox", "read-only"],
});
expect(grok.buildInvocation("review this", { readOnly: true, structured: true })).toEqual({
command: "grok",
args: ["-p", "review this", "--sandbox", "read-only", "--output-format", "json"],
});
});

it("rejects OMP advisor mode because --no-tools only disables bundled tools", () => {
const omp = getHarness("omp");

Expand Down Expand Up @@ -647,6 +661,65 @@ describe("runHarness tool operation", () => {
});
});

it("refuses read-only runs below the verified CLI version", async () => {
const versions: Array<string | null> = ["1.0.5", "1.0.13-alpha.2", null];
for (const installed of versions) {
registerHarness(
class extends FakeCursor {
override readonly invocation: Harness["invocation"] = {
args: ["-e", "console.log('write')"],
readOnlyArgs: ["-e", "console.log('read')", "{prompt}"],
readOnlyMinVersion: "1.0.13",
level: "inferred",
};
override get version(): string | null {
return installed;
}
},
);

try {
const result = await runHarness("cursor", "review", { tools: true, readOnly: true });

expect(result.isError).toBe(true);
expect(result.content[0]?.text).toContain(
`requires version 1.0.13 or newer for read-only runs; installed ${installed ?? "version unknown"}`,
);
expect(result.details).not.toHaveProperty("retry");

const agent = await runHarness("cursor", "review", { tools: true });
expect(agent.isError).toBeUndefined();
} finally {
registerHarness(Cursor);
}
}
});

it("runs read-only jobs once the installed CLI meets the verified version", async () => {
registerHarness(
class extends FakeCursor {
override readonly invocation: Harness["invocation"] = {
args: ["-e", "console.log('write')"],
readOnlyArgs: ["-e", "console.log('read:' + process.argv[1])", "{prompt}"],
readOnlyMinVersion: "1.0.13",
level: "inferred",
};
override get version(): string | null {
return "1.0.25";
}
},
);

try {
const result = await runHarness("cursor", "review", { tools: true, readOnly: true });

expect(result.isError).toBeUndefined();
expect(result.details).toMatchObject({ stdout: "read:review\n", readOnly: true });
} finally {
registerHarness(Cursor);
}
});

it("keeps unsupported read-only access from widening to a full agent", async () => {
const result = await runHarness("claude", "inspect", { tools: true, readOnly: true });

Expand Down
Loading