Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 67 additions & 1 deletion e2e/e2e.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import { afterEach, describe, expect, it } from "vitest";
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const distCliPath = join(repoRoot, "dist", "cli.mjs");
const fixtureBinDir = join(repoRoot, "e2e", "fixtures");
const windowsFixtureBinDir = join(fixtureBinDir, "windows");

// Empty gitconfig pointed at by GIT_CONFIG_GLOBAL/GIT_CONFIG_SYSTEM so the
// developer's real ~/.gitconfig (which may enable commit.gpgsign, set a
Expand Down Expand Up @@ -165,7 +166,10 @@ function createTestEnv(
...sanitizedGitEnv,
HOME: home,
USERPROFILE: home,
PATH: `${fixtureBinDir}${process.platform === "win32" ? ";" : ":"}${process.env.PATH ?? ""}`,
PATH:
process.platform === "win32"
? `${windowsFixtureBinDir};${fixtureBinDir};${process.env.PATH ?? ""}`
: `${fixtureBinDir}:${process.env.PATH ?? ""}`,
GNHF_MOCK_OPENCODE_LOG_PATH: mockLogPath,
};
}
Expand Down Expand Up @@ -399,6 +403,68 @@ describe("gnhf e2e", () => {
expect(iterationEnd?.success).toBe(false);
}, 30_000);

it.each([
{
label: "carried on stdout",
mode: "stdout-error",
expected:
"claude exited with code 1: Invalid model name: claude-nonexistent-5",
},
{
label: "with both streams empty",
mode: "no-output",
expected: "claude exited with code 1 and produced no output",
},
])(
"surfaces the claude CLI's own failure text $label",
async ({ mode, expected }) => {
const cwd = createRepo();
tempDirs.push(cwd);
const logDir = mkdtempSync(join(tmpdir(), "gnhf-e2e-logs-"));
tempDirs.push(logDir);
const mockLogPath = join(logDir, "mock-opencode.jsonl");

const result = await runCli(
cwd,
[
"break the build",
"--agent",
"claude",
"--max-iterations",
"1",
"--prevent-sleep",
"off",
],
{
env: {
...createTestEnv(mockLogPath, tempDirs),
GNHF_MOCK_CLAUDE_MODE: mode,
},
},
);

expect(result.code).toBe(0);

const debugLogPath = findRunLogPath(cwd);
const agentRunErrorEntry = readJsonLines(debugLogPath).find(
(entry) => entry.event === "agent:run:error",
);
expect(agentRunErrorEntry).toBeDefined();
const agentError = agentRunErrorEntry?.error as
| { message?: string }
| undefined;
expect(agentError?.message).toBe(expected);

// The morning-after trace: notes.md is what the user actually reads.
const notes = readFileSync(
join(dirname(debugLogPath), "notes.md"),
"utf-8",
);
expect(notes).toContain(`[ERROR] ${expected}`);
},
30_000,
);

it("reads the objective from stdin", async () => {
const cwd = createRepo();
tempDirs.push(cwd);
Expand Down
4 changes: 4 additions & 0 deletions e2e/fixtures/claude
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#!/usr/bin/env sh

SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
exec node "$SCRIPT_DIR/mock-claude-cli.mjs" "$@"
27 changes: 27 additions & 0 deletions e2e/fixtures/mock-claude-cli.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/usr/bin/env node

// Stands in for the `claude` CLI on a failing run. The failure shape is picked
// with GNHF_MOCK_CLAUDE_MODE so one fixture covers every stream combination.

import process from "node:process";

const mode = process.env.GNHF_MOCK_CLAUDE_MODE ?? "stdout-error";

if (mode === "no-output") {
process.exit(1);
}

if (mode === "stderr-error") {
process.stderr.write("Invalid API key - please run /login\n");
process.exit(1);
}

process.stdout.write(
`${JSON.stringify({
type: "result",
subtype: "error_during_execution",
is_error: true,
result: "Invalid model name: claude-nonexistent-5",
})}\n`,
);
process.exit(1);
2 changes: 2 additions & 0 deletions e2e/fixtures/windows/claude.cmd
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
@echo off
node "%~dp0\..\mock-claude-cli.mjs" %*
149 changes: 149 additions & 0 deletions src/core/agents/claude.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -935,6 +935,155 @@ describe("ClaudeAgent", () => {
);
});

it("surfaces a structured stdout error when stderr is empty", async () => {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);

const promise = agent.run("prompt", "/cwd");

emitLine(proc, {
type: "result",
subtype: "error_during_execution",
is_error: true,
result: "Invalid model name: claude-nonexistent-5",
});
proc.emit("close", 1);

await expect(promise).rejects.toThrow(
"claude exited with code 1: Invalid model name: claude-nonexistent-5",
);
});

it("surfaces plain-text stdout output when stderr is empty", async () => {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);

const promise = agent.run("prompt", "/cwd");

proc.stdout.emit(
"data",
Buffer.from("Invalid API key - please run /login"),
);
proc.emit("close", 1);

await expect(promise).rejects.toThrow(
"claude exited with code 1: Invalid API key - please run /login",
);
});

it("reports both streams when stderr and stdout carry output", async () => {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);

const promise = agent.run("prompt", "/cwd");

proc.stderr.emit("data", Buffer.from("something broke"));
emitLine(proc, {
type: "error",
error: { message: "rate limit exceeded" },
});
proc.emit("close", 1);

await expect(promise).rejects.toThrow(
"claude exited with code 1: something broke\nrate limit exceeded",
);
});

it("bounds the stdout tail included in the failure detail", async () => {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);

const promise = agent.run("prompt", "/cwd");

proc.stdout.emit("data", Buffer.from("x".repeat(10_000) + "tail marker"));
proc.emit("close", 1);

const message = await promise.then(
() => "",
(err: Error) => err.message,
);
expect(message).toContain("tail marker");
expect(message).toContain("[...truncated");
expect(message.length).toBeLessThan(600);
});

it("says so when a non-zero exit produced no output at all", async () => {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);

const promise = agent.run("prompt", "/cwd");

proc.emit("close", 1);

await expect(promise).rejects.toThrow(
"claude exited with code 1 and produced no output",
);
});

it("marks a low credit balance reported on stdout as permanent", async () => {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);

const promise = agent.run("prompt", "/cwd");

emitLine(proc, {
type: "result",
subtype: "error_during_execution",
is_error: true,
result: "Credit balance is too low to access Claude Code",
});
proc.emit("close", 1);

await expect(promise).rejects.toBeInstanceOf(PermanentAgentError);
await expect(promise).rejects.toMatchObject({
detail:
"claude exited with code 1: Credit balance is too low to access Claude Code",
});
});

it("keeps a run retryable when only agent output quotes a permanent failure", async () => {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);

const promise = agent.run("prompt", "/cwd");

emitLine(proc, {
type: "assistant",
message: {
id: "msg-1",
usage: { input_tokens: 1, output_tokens: 1 },
content: [
{
type: "text",
text: "The docs say 'credit balance is too low' aborts the run.",
},
],
},
});
proc.emit("close", 1);

await expect(promise).rejects.not.toBeInstanceOf(PermanentAgentError);
await expect(promise).rejects.toThrow("claude exited with code 1:");
});

it("keeps a run retryable when unparseable stdout quotes a permanent failure", async () => {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);

const promise = agent.run("prompt", "/cwd");

proc.stdout.emit(
"data",
Buffer.from("grep: README.md: credit balance is too low"),
);
proc.emit("close", 1);

await expect(promise).rejects.not.toBeInstanceOf(PermanentAgentError);
await expect(promise).rejects.toThrow(
"claude exited with code 1: grep: README.md: credit balance is too low",
);
});

it("marks low credit balance exits as permanent", async () => {
const proc = createMockProcess();
mockSpawn.mockReturnValue(proc);
Expand Down
Loading
Loading