diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a92a5ae0..354d733a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,11 @@ jobs: run: npm run test:coverage timeout-minutes: 3 - - name: Integration tests (slow, Docker) + - name: E2E tests (fake provider) + run: npm run test:e2e + timeout-minutes: 5 + + - name: Integration tests (slow) run: npm run test:slow timeout-minutes: 15 @@ -96,6 +100,29 @@ jobs: fail_ci_if_error: false continue-on-error: true + # Tier 2: worktree/container isolation e2e (real Docker). Non-blocking and + # opt-in via workflow_dispatch - the Docker sub-tests in `check`'s + # "Integration tests (slow)" step self-skip under CI (this is what that + # step's Docker coverage was previously silently missing). + e2e-docker: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Docker isolation e2e + run: npm run test:e2e:docker + timeout-minutes: 15 + # Cross-platform/version compatibility (main only) # NOTE: macOS runners cost 10x Linux minutes - keep this scoped to main. install-matrix: diff --git a/.github/workflows/live-provider-smoke.yml b/.github/workflows/live-provider-smoke.yml new file mode 100644 index 00000000..b0229ead --- /dev/null +++ b/.github/workflows/live-provider-smoke.yml @@ -0,0 +1,183 @@ +name: Live Provider Smoke + +on: + workflow_dispatch: + inputs: + providers: + description: Comma-separated providers to test, or all. + required: true + default: claude,codex,gemini,copilot,gateway + timeout_ms: + description: Per-provider timeout in milliseconds. + required: true + default: '120000' + schedule: + - cron: '17 6 * * 1' + +permissions: + contents: read + +jobs: + matrix: + if: github.event_name != 'schedule' || vars.ZEROSHOT_LIVE_PROVIDER_SMOKE_ENABLED == 'true' + runs-on: ubuntu-latest + outputs: + providers: ${{ steps.providers.outputs.providers }} + steps: + - uses: actions/checkout@v4 + + - name: Build provider matrix + id: providers + env: + DISPATCH_PROVIDERS: ${{ github.event.inputs.providers }} + SCHEDULE_PROVIDERS: ${{ vars.ZEROSHOT_LIVE_PROVIDERS }} + run: | + node <<'NODE' >> "$GITHUB_OUTPUT" + const requested = + process.env.DISPATCH_PROVIDERS || + process.env.SCHEDULE_PROVIDERS || + 'claude,codex,gemini,copilot,gateway'; + const helper = require('./lib/agent-cli-provider'); + const valid = helper.listProviderAdapters(); + const selected = []; + for (const raw of requested.split(',')) { + const item = raw.trim().toLowerCase(); + if (!item) continue; + if (item === 'all') { + for (const provider of valid) { + if (!selected.includes(provider)) selected.push(provider); + } + continue; + } + if (!valid.includes(item)) { + throw new Error(`Unknown live provider "${raw}". Valid: ${valid.join(', ')}, all`); + } + if (!selected.includes(item)) selected.push(item); + } + if (selected.length === 0) { + throw new Error('No live providers selected.'); + } + console.log(`providers=${JSON.stringify(selected.map((provider) => ({ provider })))}`); + NODE + + live-smoke: + needs: matrix + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.matrix.outputs.providers) }} + env: + ZEROSHOT_LIVE_TIMEOUT_MS: ${{ github.event.inputs.timeout_ms || '120000' }} + ANTHROPIC_API_KEY: ${{ secrets.ZEROSHOT_LIVE_ANTHROPIC_API_KEY || secrets.ANTHROPIC_API_KEY }} + CLAUDE_API_KEY: ${{ secrets.ZEROSHOT_LIVE_CLAUDE_API_KEY || secrets.CLAUDE_API_KEY }} + OPENAI_API_KEY: ${{ secrets.ZEROSHOT_LIVE_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} + CODEX_API_KEY: ${{ secrets.ZEROSHOT_LIVE_CODEX_API_KEY || secrets.CODEX_API_KEY }} + GEMINI_API_KEY: ${{ secrets.ZEROSHOT_LIVE_GEMINI_API_KEY || secrets.GEMINI_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.ZEROSHOT_LIVE_GOOGLE_API_KEY || secrets.GOOGLE_API_KEY }} + OPENCODE_API_KEY: ${{ secrets.ZEROSHOT_LIVE_OPENCODE_API_KEY || secrets.OPENCODE_API_KEY }} + KIRO_API_KEY: ${{ secrets.ZEROSHOT_LIVE_KIRO_API_KEY || secrets.KIRO_API_KEY }} + COPILOT_GITHUB_TOKEN: ${{ secrets.ZEROSHOT_LIVE_COPILOT_GITHUB_TOKEN || secrets.COPILOT_GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.ZEROSHOT_LIVE_GH_TOKEN || secrets.GH_TOKEN }} + ZEROSHOT_LIVE_GATEWAY_BASE_URL: ${{ secrets.ZEROSHOT_LIVE_GATEWAY_BASE_URL || vars.ZEROSHOT_LIVE_GATEWAY_BASE_URL }} + ZEROSHOT_LIVE_GATEWAY_API_KEY: ${{ secrets.ZEROSHOT_LIVE_GATEWAY_API_KEY }} + ZEROSHOT_LIVE_GATEWAY_MODEL: ${{ secrets.ZEROSHOT_LIVE_GATEWAY_MODEL || vars.ZEROSHOT_LIVE_GATEWAY_MODEL }} + ZEROSHOT_LIVE_GATEWAY_HEADERS_JSON: ${{ secrets.ZEROSHOT_LIVE_GATEWAY_HEADERS_JSON }} + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Install provider CLI + run: | + case "${{ matrix.provider }}" in + claude) + npm install -g @anthropic-ai/claude-code + ;; + codex) + npm install -g @openai/codex + ;; + gemini) + npm install -g @google/gemini-cli + ;; + pi) + npm install -g --ignore-scripts @earendil-works/pi-coding-agent@0.80.3 + ;; + copilot) + npm install -g @github/copilot + ;; + gateway) + ;; + opencode) + command -v opencode >/dev/null 2>&1 || { + echo "::error::opencode is not installed on this runner. Use a self-hosted runner with opencode installed and authenticated." + exit 1 + } + ;; + kiro) + command -v kiro-cli >/dev/null 2>&1 || { + echo "::error::kiro-cli is not installed on this runner. Use a self-hosted runner with kiro-cli installed and authenticated." + exit 1 + } + ;; + *) + echo "::error::Unsupported provider ${{ matrix.provider }}" + exit 1 + ;; + esac + + - name: Verify live prerequisites + run: | + case "${{ matrix.provider }}" in + claude) + test -n "$ANTHROPIC_API_KEY$CLAUDE_API_KEY" || { + echo "::error::claude live smoke requires ZEROSHOT_LIVE_ANTHROPIC_API_KEY, ANTHROPIC_API_KEY, ZEROSHOT_LIVE_CLAUDE_API_KEY, or CLAUDE_API_KEY." + exit 1 + } + ;; + codex) + test -n "$OPENAI_API_KEY$CODEX_API_KEY" || { + echo "::error::codex live smoke requires ZEROSHOT_LIVE_OPENAI_API_KEY, OPENAI_API_KEY, ZEROSHOT_LIVE_CODEX_API_KEY, or CODEX_API_KEY." + exit 1 + } + ;; + gemini) + test -n "$GEMINI_API_KEY$GOOGLE_API_KEY" || { + echo "::error::gemini live smoke requires ZEROSHOT_LIVE_GEMINI_API_KEY, GEMINI_API_KEY, ZEROSHOT_LIVE_GOOGLE_API_KEY, or GOOGLE_API_KEY." + exit 1 + } + ;; + copilot) + test -n "$COPILOT_GITHUB_TOKEN$GH_TOKEN" || { + echo "::error::copilot live smoke requires ZEROSHOT_LIVE_COPILOT_GITHUB_TOKEN, COPILOT_GITHUB_TOKEN, ZEROSHOT_LIVE_GH_TOKEN, or GH_TOKEN." + exit 1 + } + ;; + gateway) + test -n "$ZEROSHOT_LIVE_GATEWAY_BASE_URL" && + test -n "$ZEROSHOT_LIVE_GATEWAY_API_KEY" && + test -n "$ZEROSHOT_LIVE_GATEWAY_MODEL" || { + echo "::error::gateway live smoke requires ZEROSHOT_LIVE_GATEWAY_BASE_URL, ZEROSHOT_LIVE_GATEWAY_API_KEY, and ZEROSHOT_LIVE_GATEWAY_MODEL." + exit 1 + } + ;; + kiro) + test -n "$KIRO_API_KEY" || { + echo "::error::kiro live smoke requires ZEROSHOT_LIVE_KIRO_API_KEY or KIRO_API_KEY." + exit 1 + } + ;; + esac + + - name: Run live provider smoke + env: + ZEROSHOT_LIVE_PROVIDERS: ${{ matrix.provider }} + run: npm run test:providers:live diff --git a/.releaserc.json b/.releaserc.json index 90723e6d..2b925492 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -3,12 +3,20 @@ "plugins": [ "@semantic-release/commit-analyzer", "@semantic-release/release-notes-generator", + "@semantic-release/changelog", [ "@semantic-release/npm", { "npmPublish": true } ], + [ + "@semantic-release/git", + { + "assets": ["package.json", "package-lock.json", "CHANGELOG.md"], + "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" + } + ], "@semantic-release/github" ] } diff --git a/AGENTS.md b/AGENTS.md index edb27f86..e1abfe5c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,25 +21,28 @@ Destructive commands (need permission): `zeroshot kill`, `zeroshot clear`, `zero ## Where to Look -| Concept | File | -| ------------------------ | ----------------------------------- | -| Conductor classification | `src/conductor-bootstrap.js` | -| Base templates | `cluster-templates/base-templates/` | -| Message bus | `src/message-bus.js` | -| Ledger (SQLite) | `src/ledger.js` | -| Guidance topics | `src/guidance-topics.js` | -| Guidance mailbox helper | `src/ledger.js` | -| Guidance live injection | `src/orchestrator.js` | -| Trigger evaluation | `src/logic-engine.js` | -| Agent wrapper | `src/agent-wrapper.js` | -| Providers registry | `src/providers/index.js` | -| Provider implementations | `src/providers/` | -| Provider detection | `lib/provider-detection.js` | -| Provider capabilities | `src/providers/capabilities.js` | -| Start-cluster helper | `lib/start-cluster.js` | -| Docker mounts/env | `lib/docker-config.js` | -| Container lifecycle | `src/isolation-manager.js` | -| Settings | `lib/settings.js` | +| Concept | File | +| ------------------------ | --------------------------------------------- | +| Conductor classification | `src/conductor-bootstrap.js` | +| Base templates | `cluster-templates/base-templates/` | +| Message bus | `src/message-bus.js` | +| Ledger (SQLite) | `src/ledger.js` | +| Guidance topics | `src/guidance-topics.js` | +| Guidance mailbox helper | `src/ledger.js` | +| Guidance live injection | `src/orchestrator.js` | +| Trigger evaluation | `src/logic-engine.js` | +| Agent wrapper | `src/agent-wrapper.js` | +| Providers registry | `src/providers/index.js` | +| Provider implementations | `src/providers/` | +| Provider engine registry | `src/agent-cli-provider/provider-registry.ts` | +| Gateway runner | `src/agent-cli-provider/gateway-runner.ts` | +| Gateway tools/policy | `src/agent-cli-provider/gateway-tools.ts` | +| Provider detection | `lib/provider-detection.js` | +| Provider capabilities | `src/providers/capabilities.js` | +| Start-cluster helper | `lib/start-cluster.js` | +| Docker mounts/env | `lib/docker-config.js` | +| Container lifecycle | `src/isolation-manager.js` | +| Settings | `lib/settings.js` | The TUI is not included in this release. Use `zeroshot list`, `zeroshot status `, and `zeroshot logs -f` or `zeroshot logs -w` for monitoring. @@ -77,7 +80,13 @@ UX modes: - Daemon (`-d`): background, Ctrl+C detaches. - Attach (`zeroshot attach`): connect to daemon, Ctrl+C detaches only. -Settings: `defaultProvider`, `providerSettings` (claude/codex/gemini), legacy `maxModel`, `defaultConfig`, `logLevel`, robustness (`maxRetries`, `backoffBaseMs`, `backoffMaxMs`, `jitterFactor`, `maxRestartAttempts`, `maxTotalRestarts`, `staleWarningsBeforeKill`). +Settings: `defaultProvider`, `providerSettings` (claude/codex/gateway/gemini/opencode/pi/copilot), legacy `maxModel`, `defaultConfig`, `logLevel`, robustness (`maxRetries`, `backoffBaseMs`, `backoffMaxMs`, `jitterFactor`, `maxRestartAttempts`, `maxTotalRestarts`, `staleWarningsBeforeKill`). + +Provider engines are registry-owned: adding an engine means one entry in `src/agent-cli-provider/provider-registry.ts`, plus the provider-specific adapter and tests. Docker credential mount/env presets, CLI aliases, visible preset lists, and any nontrivial availability probe rules must derive from that registry entry; do not add new provider identity lists or provider preset lists elsewhere. +Model gateways stay behind the single bundled `gateway` engine. Do not add `openrouter`, `ollama`, `vllm`, `hermes`, or similar model-only targets as standalone provider ids. + +ACP-native engines use one shared stdio adapter lane. New ACP engines must be added with registry metadata plus helper fixtures only; do not add engine-specific ACP parsers or invoke runners. +ACP fixtures must use protocol-shaped chunk payloads: `agent_message_chunk.content` is a single `ContentBlock` object, and thought deltas are covered with `agent_thought_chunk` fixtures so parser tests catch spec drift. ## Architecture @@ -127,7 +136,7 @@ Restart persistence: orchestrator publishes `AGENT_RESTART_ATTEMPT` to the ledge - Use `modelLevel` (`level1`/`level2`/`level3`) for provider-agnostic configs. - Set `provider` per agent or `defaultProvider`/`forceProvider` at cluster level. -- Provider names use CLI identifiers: `claude`, `codex`, `gemini` (legacy `anthropic`/`openai`/`google` map to these). +- Provider names use CLI identifiers: `claude`, `codex`, `gemini`, `opencode`, `pi`, `copilot` (legacy `anthropic`/`openai`/`google` map to these). - `model` remains a provider-specific escape hatch. - Codex/Opencode only: `reasoningEffort` (`low|medium|high|xhigh`). @@ -185,13 +194,13 @@ Docker: fresh git clone in container, credentials mounted, auto-cleanup. Configurable credential mounts for `--docker` mode. See `lib/docker-config.js`. -| Setting | Type | Default | Description | +| Setting | Type | Default | Description | | ---------------------- | ------------- | -------- | ----------------------------------------------------- | ---------------------------------------- | -| `dockerMounts` | `Array` | `['gh','git','ssh']` | Presets or `{host, container, readonly}` | -| `dockerEnvPassthrough` | `string[]` | `[]` | Extra env vars (supports `VAR`, `VAR_*`, `VAR=value`) | -| `dockerContainerHome` | `string` | `/root` | Container home for `$HOME` expansion | +| `dockerMounts` | `Array` | `['gh','git','ssh']` | Presets or `{host, container, readonly}` | +| `dockerEnvPassthrough` | `string[]` | `[]` | Extra env vars (supports `VAR`, `VAR_*`, `VAR=value`) | +| `dockerContainerHome` | `string` | `/root` | Container home for `$HOME` expansion | -Mount presets: `gh`, `git`, `ssh`, `aws`, `azure`, `kube`, `terraform`, `gcloud`, `claude`, `codex`, `gemini`, `opencode`. +Mount presets: infrastructure presets plus provider ids from `src/agent-cli-provider/provider-registry.ts`. Provider CLIs in Docker require credential mounts; Zeroshot warns when missing. diff --git a/FIX_TESTS.md b/FIX_TESTS.md deleted file mode 100644 index d220f7d3..00000000 --- a/FIX_TESTS.md +++ /dev/null @@ -1,125 +0,0 @@ -# Test Failures Snapshot - -## Overall - -- 853 passing (24m) -- 16 pending -- 15 failing - -## Failing Tests (15) - -1. MessageBus Integration - - Suite: Batch Publishing (Atomic) - - Test: should prevent interleaving with concurrent agents - - Error: AssertionError [ERR_ASSERTION]: Agent A's messages should be contiguous - - Expected vs actual: 1 expected, 2 actual - - Location: tests/integration/message-bus.test.js:441:14 - - Status: Fixed (verified locally) - -2. npm install retry logic - - Suite: Retry mechanism - - Test: retries npm install on failure with exponential backoff - - Error: AssertionError [ERR_ASSERTION]: Should retry twice after initial failure - - Expected vs actual: 3 expected, 2 actual - - Location: tests/integration/npm-install-retry.test.js:110:14 - - Status: Fixed (pending verification) - -3. npm install retry logic - - Suite: Retry mechanism - - Test: fails after max retries exceeded - - Error: AssertionError [ERR_ASSERTION]: Should attempt 3 times total - - Expected vs actual: 3 expected, 2 actual - - Location: tests/integration/npm-install-retry.test.js:148:14 - - Status: Fixed (pending verification) - -4. npm install retry logic - - Suite: Retry mechanism - - Test: does not retry if first attempt succeeds - - Error: AssertionError [ERR_ASSERTION]: Should only attempt once on success - - Expected vs actual: 1 expected, 4 actual - - Location: tests/integration/npm-install-retry.test.js:183:14 - - Status: Fixed (pending verification) - -5. npm install retry logic - - Suite: Retry mechanism - - Test: handles execution errors during retry - - Error: AssertionError [ERR_ASSERTION]: Should retry after execution errors - - Expected vs actual: 3 expected, 1 actual - - Location: tests/integration/npm-install-retry.test.js:249:14 - - Status: Fixed (pending verification) - -6. npm install retry logic - - Suite: Exponential backoff timing - - Test: uses correct delay calculation: 2s, 4s, 8s - - Error: AssertionError [ERR_ASSERTION]: Should have 3 attempts - - Expected vs actual: 3 expected, 2 actual - - Location: tests/integration/npm-install-retry.test.js:291:14 - - Status: Fixed (pending verification) - -7. npm install retry logic - - Suite: Non-fatal failure behavior - - Test: logs warning but continues when all retries fail - - Error: AssertionError [ERR_ASSERTION]: Should log warnings about npm install failures - - Expected vs actual: true expected, false actual - - Location: tests/integration/npm-install-retry.test.js:343:9 - - Status: Fixed (pending verification) - -8. Orchestrator Isolation Mode Integration - - Suite: Container Lifecycle - - Test: should preserve workspace on stop for resume capability - - Error: AssertionError [ERR_ASSERTION]: Isolated workspace should be PRESERVED for resume: /tmp/zeroshot-isolated/flaming-surge-53 - - Expected vs actual: true expected, false actual - - Location: tests/integration/orchestrator-isolation.test.js:250:7 - - Status: Fixed (pending verification) - -9. Orchestrator Isolation Mode Integration - - Suite: Resume Capability - - Test: should recreate container on resume using preserved workspace - - Error: AssertionError [ERR_ASSERTION]: Workspace should be preserved for resume - - Expected vs actual: true expected, false actual - - Location: tests/integration/orchestrator-isolation.test.js:451:7 - - Status: Fixed (pending verification) - -10. Orchestrator Isolation Mode Integration - - Suite: Resume Capability - - Test: should not be resumable after kill (cluster removed) - - Error: AssertionError [ERR_ASSERTION]: Workspace should exist before kill - - Expected vs actual: true expected, false actual - - Location: tests/integration/orchestrator-isolation.test.js:547:7 - - Status: Fixed (pending verification) - -11. Orchestrator Isolation Mode Integration - - Suite: Container Lifecycle - - Test: should preserve workspace on stop for resume capability - - Error: AssertionError [ERR_ASSERTION]: Isolated workspace should be PRESERVED for resume: /tmp/zeroshot-isolated/sonic-glyph-68 - - Expected vs actual: true expected, false actual - - Location: tests/integration/slow/orchestrator-isolation.test.js:253:7 - - Status: Fixed (pending verification) - -12. Orchestrator Isolation Mode Integration - - Suite: Resume Capability - - Test: should recreate container on resume using preserved workspace - - Error: AssertionError [ERR_ASSERTION]: Workspace should be preserved for resume - - Expected vs actual: true expected, false actual - - Location: tests/integration/slow/orchestrator-isolation.test.js:454:7 - - Status: Fixed (pending verification) - -13. Orchestrator Isolation Mode Integration - - Suite: Resume Capability - - Test: should not be resumable after kill (cluster removed) - - Error: AssertionError [ERR_ASSERTION]: Workspace should exist before kill - - Expected vs actual: true expected, false actual - - Location: tests/integration/slow/orchestrator-isolation.test.js:550:7 - - Status: Fixed (pending verification) - -14. Isolated Mode Output Capture - - Test: should read agent output from log file, not spawn stdout - - Error: Timeout of 150000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. - - Location: tests/unit/isolated-mode-output-capture.test.js - - Status: Fixed (pending verification) - -15. Isolated Mode Output Capture - - Hook: "after each" hook for "should read agent output from log file, not spawn stdout" - - Error: Timeout of 2000ms exceeded. For async tests and hooks, ensure "done()" is called; if returning a Promise, ensure it resolves. - - Location: tests/unit/isolated-mode-output-capture.test.js - - Status: Fixed (pending verification) diff --git a/cli/commands/providers.js b/cli/commands/providers.js index 77c81a82..9d3b707e 100644 --- a/cli/commands/providers.js +++ b/cli/commands/providers.js @@ -1,8 +1,14 @@ const readline = require('readline'); const { loadSettings, saveSettings } = require('../../lib/settings'); -const { VALID_PROVIDERS, normalizeProviderName } = require('../../lib/provider-names'); +const { + VALID_PROVIDERS, + normalizeProviderName, + providerSupportsCapability, +} = require('../../lib/provider-names'); const { detectProviders, getProvider } = require('../../src/providers'); +const PROVIDER_CHOICES = VALID_PROVIDERS.join(', '); + function question(rl, prompt) { return new Promise((resolve) => { rl.question(prompt, (answer) => resolve(answer.trim())); @@ -45,7 +51,7 @@ async function providersCommand() { function setDefaultCommand(args) { const provider = normalizeProviderName(args[0]); if (!VALID_PROVIDERS.includes(provider)) { - console.error(`Invalid provider: ${args[0]}`); + console.error(`Invalid provider: ${args[0]}. Valid: ${PROVIDER_CHOICES}`); process.exit(1); } @@ -58,8 +64,8 @@ function setDefaultCommand(args) { async function setupCommand(args) { const provider = normalizeProviderName(args[0]); - if (!provider) { - console.error('Provider is required (claude, codex, gemini, opencode)'); + if (!provider || !VALID_PROVIDERS.includes(provider)) { + console.error(`Provider is required (${PROVIDER_CHOICES})`); process.exit(1); } @@ -115,7 +121,7 @@ async function setupCommand(args) { for (const level of levelKeys) { const modelChoice = await question(rl, `Model for ${level} (${catalog.join(', ')}): `); if (modelChoice) levelOverrides[level] = { model: modelChoice }; - if (provider !== 'codex') continue; + if (!providerSupportsCapability(provider, 'reasoningEffort')) continue; const reasoning = await question(rl, `Reasoning for ${level} (low|medium|high|xhigh): `); if (reasoning) { levelOverrides[level] = { diff --git a/cli/index.js b/cli/index.js index 45949724..67509c48 100755 --- a/cli/index.js +++ b/cli/index.js @@ -46,8 +46,9 @@ const { validateSetting, coerceValue, DEFAULT_SETTINGS, + settingsFileExists, } = require('../lib/settings'); -const { normalizeProviderName } = require('../lib/provider-names'); +const { VALID_PROVIDERS, normalizeProviderName } = require('../lib/provider-names'); const { getProvider, parseProviderChunk } = require('../src/providers'); const { readClustersFileSync } = require('../lib/clusters-registry'); const { MOUNT_PRESETS, resolveEnvs } = require('../lib/docker-config'); @@ -89,6 +90,7 @@ const { EVENT_COPY, formatMergeStatus } = require('./event-copy'); let activeClusterId = null; /** @type {import('../src/orchestrator') | null} */ let orchestratorInstance = null; +const PROVIDER_CHOICES = VALID_PROVIDERS.join(', '); // Track active status footer for safe output routing // When set, all output routes through statusFooter.print() to prevent garbling @@ -189,7 +191,9 @@ function normalizeRunOptions(options) { if (options.docker) { options.worktree = false; } - options.autoMerge = Boolean(options.ship); + // autoMerge is NOT stored here — it is derived from the run plan (delivery === + // 'ship') at every consumer. Writing it here unconditionally would clobber an + // explicit autoMerge intent (e.g. a future `--auto-merge` flag) back to false. } async function runClusterPreflight({ input, options, providerOverride, settings, forceProvider }) { @@ -2411,10 +2415,7 @@ program 'When to close issue after merge: auto|always|never (default: from .zeroshot/settings.json or never)' ) .option('--workers ', 'Max sub-agents for worker to spawn in parallel', parseInt) - .option( - '--provider ', - 'Override all agents to use a provider (claude, codex, gemini, opencode)' - ) + .option('--provider ', `Override all agents to use a provider (${PROVIDER_CHOICES})`) .option('--model ', 'Override all agent models (provider-specific model id)') .option( '--sim ', @@ -2610,6 +2611,7 @@ Force provider flags: -G (GitHub), -L (GitLab), -J (Jira), -D (DevOps), -N (Line if (!process.env.ZEROSHOT_DAEMON) { await streamClusterInForeground(cluster, orchestrator, clusterId, options); + orchestrator.close(); } setupDaemonCleanup(orchestrator, clusterId); @@ -2679,7 +2681,7 @@ taskCmd .command('run ') .description('Run a single-agent background task') .option('-C, --cwd ', 'Working directory for task') - .option('--provider ', 'Provider to use (claude, codex, gemini, opencode)') + .option('--provider ', `Provider to use (${PROVIDER_CHOICES})`) .option('--model ', 'Model id override for the provider') .option('--model-level ', 'Model level override (level1, level2, level3)') .option('--reasoning-effort ', 'Reasoning effort (low, medium, high, xhigh)') @@ -2691,6 +2693,12 @@ taskCmd 'stream-json' ) .option('--json-schema ', 'JSON schema for structured output') + .option( + '--mcp-config ', + 'MCP server config for providers that accept an MCP config flag (e.g. Copilot). ' + + 'Inline JSON string ({"mcpServers":{...}}) or @. Repeatable.', + (value, previous) => (previous || []).concat([value]) + ) .option('--silent-json-output', 'Log ONLY final structured output') .action(async (prompt, options) => { try { @@ -3602,15 +3610,15 @@ function registerTuiEntrypoint(commandName, providerName) { .action(failTuiUnavailable); } -registerTuiEntrypoint('codex', 'codex'); -registerTuiEntrypoint('claude', 'claude'); -registerTuiEntrypoint('gemini', 'gemini'); -registerTuiEntrypoint('opencode', 'opencode'); +for (const providerName of VALID_PROVIDERS) { + registerTuiEntrypoint(providerName, providerName); +} // Settings management const settingsCmd = program.command('settings').description('Manage zeroshot settings'); function printSettingsUsage() { + const mountPresetList = Object.keys(MOUNT_PRESETS).join(', '); console.log(chalk.dim('Usage:')); console.log(chalk.dim(' zeroshot settings set ')); console.log(chalk.dim(' zeroshot settings get ')); @@ -3621,11 +3629,7 @@ function printSettingsUsage() { console.log(chalk.dim(' zeroshot settings set dockerMounts \'["gh","git","ssh","aws"]\'')); console.log(chalk.dim(' zeroshot settings set dockerEnvPassthrough \'["AWS_*","TF_VAR_*"]\'')); console.log(''); - console.log( - chalk.dim( - 'Available mount presets: gh, git, ssh, aws, azure, kube, terraform, gcloud, claude, codex, gemini' - ) - ); + console.log(chalk.dim(`Available mount presets: ${mountPresetList}`)); console.log(''); } @@ -4126,7 +4130,7 @@ providersCmd.action(async () => { providersCmd .command('set-default ') - .description('Set default provider (claude, codex, gemini, opencode)') + .description(`Set default provider (${PROVIDER_CHOICES})`) .action(async (provider) => { await setDefaultCommand([provider]); }); @@ -4138,6 +4142,68 @@ providersCmd await setupCommand([provider]); }); +// Setup wizard (read-only facts + setup contract; apply/undo/TTY wizard land separately) +const setupCmd = program.command('setup').description('Setup and configuration wizard'); +setupCmd + .command('plan') + .description('Show read-only setup facts and proposed contract (no writes)') + .option('--json', 'Output as JSON') + .action(() => { + const { buildSetupPlan } = require('../lib/setup-plan'); + const { readRepoSettings } = require('../lib/repo-settings'); + + const cwd = process.cwd(); + const settings = loadSettings(); + settings.__meta = { fileExists: settingsFileExists() }; + const { settings: repoSettings } = readRepoSettings(cwd); + const plan = buildSetupPlan({ + cwd, + settings, + repoSettings, + env: { ...process.env, __isTTY: !!process.stdout.isTTY }, + }); + console.log(JSON.stringify(plan, null, 2)); + }); + +setupCmd + .command('apply') + .description('Apply a decisions file to global/repo settings (writes)') + .requiredOption('--decisions ', 'Path to decisions JSON file') + .option('--json', 'Output as JSON') + .option( + '--allow-risky-defaults', + 'Allow storing defaultDelivery=ship as a global default (auto-merge by default)' + ) + .action((opts) => { + const { applyDecisions } = require('../lib/setup-apply'); + try { + const results = applyDecisions({ + decisionsPath: opts.decisions, + cwd: process.cwd(), + allowRiskyDefaults: !!opts.allowRiskyDefaults, + }); + console.log(JSON.stringify({ results }, null, 2)); + } catch (err) { + console.error(JSON.stringify({ error: err.message }, null, 2)); + process.exitCode = 1; + } + }); + +setupCmd + .command('undo') + .description('Undo writes made by the most recent `zeroshot setup apply` runs') + .option('--json', 'Output as JSON') + .action(() => { + const { undo } = require('../lib/setup-undo'); + try { + const results = undo({}); + console.log(JSON.stringify({ results }, null, 2)); + } catch (err) { + console.error(JSON.stringify({ error: err.message }, null, 2)); + process.exitCode = 1; + } + }); + // Update command program .command('update') diff --git a/cli/lib/first-run.js b/cli/lib/first-run.js index d40eb73c..df587ff0 100644 --- a/cli/lib/first-run.js +++ b/cli/lib/first-run.js @@ -10,6 +10,7 @@ const readline = require('readline'); const { loadSettings, saveSettings } = require('../../lib/settings'); +const { listProviderMetadata } = require('../../lib/provider-names'); const { detectProviders } = require('../../src/providers'); /** @@ -39,6 +40,16 @@ function createReadline() { }); } +function printProviderInstallChoices() { + for (const provider of listProviderMetadata()) { + const [firstLine, ...remainingLines] = provider.installInstructions.split('\n'); + console.log(` - ${provider.displayName}: ${firstLine}`); + for (const line of remainingLines) { + console.log(` ${line}`); + } + } +} + /** * Prompt for provider selection * @param {readline.Interface} rl @@ -52,9 +63,7 @@ function promptProvider(rl, detected) { if (available.length === 0) { console.log('No AI CLI tools detected. Please install one of:'); - console.log(' - Claude Code: npm install -g @anthropic-ai/claude-code'); - console.log(' - Codex CLI: npm install -g @openai/codex'); - console.log(' - Gemini CLI: npm install -g @google/gemini-cli'); + printProviderInstallChoices(); process.exit(1); } @@ -208,4 +217,6 @@ module.exports = { detectFirstRun, printWelcome, printComplete, + promptProvider, + printProviderInstallChoices, }; diff --git a/docker/zeroshot-cluster/Dockerfile b/docker/zeroshot-cluster/Dockerfile index 2929bc38..cc9f52ac 100644 --- a/docker/zeroshot-cluster/Dockerfile +++ b/docker/zeroshot-cluster/Dockerfile @@ -8,7 +8,9 @@ FROM node:20-slim # Upgrade npm to fix Arborist isDescendantOf bug (npm 10.x crash on complex peer deps) # See: https://github.com/npm/cli/issues/7682 -RUN npm install -g npm@latest +# Pinned to npm@11 (the latest line that still supports the node:20-slim base); npm@latest (12.x) +# now requires node >=22 and breaks the build with EBADENGINE. +RUN npm install -g npm@11 # Version pinning for infrastructure tools ARG AWS_CLI_VERSION=2.15.10 @@ -146,6 +148,14 @@ RUN mkdir -p /home/node/.claude /home/node/.config/gh \ RUN mkdir -p /workspace && chown node:node /workspace WORKDIR /workspace +# Install the running provider's CLI as a docker-cached layer (per-provider image variant). +# Claude is baked in above, so its variant passes an empty PROVIDER_INSTALL and skips this step. +# Kept last in the root phase so every heavy layer above is shared across all provider images; +# Docker caches this layer keyed on the install command, so it builds once per provider and is +# then reused. The command comes from the provider registry (docker.install), never hardcoded. +ARG PROVIDER_INSTALL="" +RUN if [ -n "$PROVIDER_INSTALL" ]; then sh -c "$PROVIDER_INSTALL"; fi + # Switch to non-root user (required for --dangerously-skip-permissions) USER node diff --git a/docs/providers.md b/docs/providers.md index 0dc0d695..53c4f93f 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -1,16 +1,23 @@ # Providers -Zeroshot shells out to provider CLIs. It does not store API keys or manage -authentication. Use each CLI's login flow or API key setup. +Zeroshot supports two provider shapes: + +- CLI-backed providers that shell out to a full agent CLI +- One bundled `gateway` provider that wraps OpenAI-compatible model APIs with a + Zeroshot-owned tool runner ## Supported Providers -| Provider | CLI | Install | -| -------- | ----------- | ------------------------------------------ | -| Claude | Claude Code | `npm install -g @anthropic-ai/claude-code` | -| Codex | Codex | `npm install -g @openai/codex` | -| Gemini | Gemini | `npm install -g @google/gemini-cli` | -| Opencode | Opencode | See https://opencode.ai | +| Provider | CLI | Install | +| -------- | ----------- | ------------------------------------------------------------------------ | +| Claude | Claude Code | `npm install -g @anthropic-ai/claude-code` | +| Codex | Codex | `npm install -g @openai/codex` | +| Gateway | Bundled | No external CLI required | +| Gemini | Gemini | `npm install -g @google/gemini-cli` | +| Opencode | Opencode | See https://opencode.ai | +| Pi | Pi | `npm install -g --ignore-scripts @earendil-works/pi-coding-agent@0.80.3` | +| Kiro | Kiro | See https://kiro.dev/docs/cli/ | +| Copilot | Copilot | `npm install -g @github/copilot` | ## Selecting a Provider @@ -20,6 +27,36 @@ authentication. Use each CLI's login flow or API key setup. - Override per run: `zeroshot run ... --provider ` - Env override: `ZEROSHOT_PROVIDER=codex` +## Gateway Provider + +Use `gateway` for OpenAI-compatible model endpoints such as OpenRouter, +Ollama, vLLM, or self-hosted gateways. These stay model configs behind one +provider engine; do not add them as standalone provider ids. + +Required settings: + +```json +{ + "providerSettings": { + "gateway": { + "baseUrl": "http://127.0.0.1:11434", + "apiKey": "gateway-key", + "model": "openrouter/meta-llama/test-model", + "toolPolicy": { + "roots": ["/absolute/path/to/worktree"], + "commands": ["node"] + } + } + } +} +``` + +Notes: + +- `toolPolicy` is required. There is no default file or shell access. +- `headers` is optional for extra gateway-specific request headers. +- `model` may be any non-empty provider-specific model id. + ## Model Levels Zeroshot uses provider-agnostic levels: @@ -53,8 +90,8 @@ Notes: ## Docker Isolation and Credentials -Zeroshot does not inject credentials for non-Claude CLIs. When using -`--docker`, mount your provider config directories explicitly. +Zeroshot does not inject credentials for external CLIs. When using `--docker`, +mount your provider config directories explicitly. Examples: @@ -89,3 +126,61 @@ internals. See `docs/provider-cli-helper.md` for the ownership boundary, non-goals, rollout rules, and required verification commands. + +## Live Provider Smoke Tests + +The normal test suite is deterministic and offline. To verify a provider against +the real installed CLI or a real gateway endpoint, run the opt-in live smoke +command: + +```bash +ZEROSHOT_LIVE_PROVIDERS=all npm run test:providers:live +ZEROSHOT_LIVE_PROVIDERS=claude,codex,gemini npm run test:providers:live +ZEROSHOT_LIVE_PROVIDERS=pi npm run test:providers:live +ZEROSHOT_LIVE_PROVIDERS=copilot npm run test:providers:live +``` + +Gateway requires endpoint settings: + +```bash +ZEROSHOT_LIVE_PROVIDERS=gateway \ + ZEROSHOT_LIVE_GATEWAY_BASE_URL=https://openrouter.ai/api/v1 \ + ZEROSHOT_LIVE_GATEWAY_API_KEY=... \ + ZEROSHOT_LIVE_GATEWAY_MODEL=openai/gpt-5.4 \ + npm run test:providers:live +``` + +The live command invokes the provider through Zeroshot's executable provider +contract and requires the provider to return the sentinel +`ZEROSHOT_LIVE_SMOKE_OK`. It is not part of CI because it may require local +auth, network access, and paid API calls. + +### GitHub Actions Live Smoke + +Use the `Live Provider Smoke` workflow for release-gating real providers. It is +manual by default and scheduled only when the repository variable +`ZEROSHOT_LIVE_PROVIDER_SMOKE_ENABLED` is set to `true`. + +Recommended release gate: + +```text +claude,codex,gemini,copilot,gateway +``` + +Run `all` only on a runner that also has Opencode, Pi, and Kiro installed and +authenticated. The workflow fails selected providers when the executable or +required credential is missing; it does not convert missing live coverage into a +passing skip. + +Credential names: + +| Provider | Required CI credential | +| -------- | ------------------------------------------------------------------------------------------------ | +| Claude | `ZEROSHOT_LIVE_ANTHROPIC_API_KEY` or `ANTHROPIC_API_KEY` | +| Codex | `ZEROSHOT_LIVE_OPENAI_API_KEY` or `OPENAI_API_KEY` | +| Gemini | `ZEROSHOT_LIVE_GEMINI_API_KEY` / `ZEROSHOT_LIVE_GOOGLE_API_KEY` | +| Copilot | `ZEROSHOT_LIVE_COPILOT_GITHUB_TOKEN` | +| Gateway | `ZEROSHOT_LIVE_GATEWAY_BASE_URL`, `ZEROSHOT_LIVE_GATEWAY_API_KEY`, `ZEROSHOT_LIVE_GATEWAY_MODEL` | +| Kiro | `ZEROSHOT_LIVE_KIRO_API_KEY` plus a runner with `kiro-cli` installed | +| Pi | A runner with `pi` installed and authenticated | +| Opencode | A runner with `opencode` installed and authenticated | diff --git a/lib/detached-startup.js b/lib/detached-startup.js index 98bb6e05..bc4eb48c 100644 --- a/lib/detached-startup.js +++ b/lib/detached-startup.js @@ -2,6 +2,7 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); const lockfile = require('proper-lockfile'); +const { resolveRunPlan } = require('./run-plan'); const DEFAULT_WAIT_TIMEOUT_SECONDS = 180; const DEFAULT_WAIT_POLL_MS = 1000; @@ -116,6 +117,7 @@ async function registerDetachedSetupCluster({ runOptions = {}, cwd, }) { + const plan = resolveRunPlan(runOptions); await updateClustersFile(storageDir, (clusters) => { clusters[clusterId] = { id: clusterId, @@ -125,13 +127,13 @@ async function registerDetachedSetupCluster({ setupLogPath: logPath || null, setupStartedAt: Date.now(), setupStage: 'starting', - autoPr: Boolean(runOptions.pr || runOptions.ship), + autoPr: plan.delivery !== 'none', prOptions: runOptions.prBase ? { prBase: runOptions.prBase, mergeQueue: runOptions.mergeQueue || false, closeIssue: runOptions.closeIssue || null, - autoMerge: Boolean(runOptions.ship), + autoMerge: plan.autoMerge, cwd: cwd || null, } : null, diff --git a/lib/docker-config.js b/lib/docker-config.js index d68802da..295fd984 100644 --- a/lib/docker-config.js +++ b/lib/docker-config.js @@ -3,13 +3,14 @@ * Fully generic - no hardcoded paths, all configurable */ +const { listProviderMetadata } = require('./provider-names'); /** * Built-in mount presets * Uses $HOME placeholder - resolved at runtime based on dockerContainerHome setting * Host paths use ~ (expanded to host user's home) * Container paths use $HOME (expanded to configured container home) */ -const MOUNT_PRESETS = { +const BASE_MOUNT_PRESETS = { gh: { host: '~/.config/gh', container: '$HOME/.config/gh', readonly: false }, git: { host: '~/.gitconfig', container: '$HOME/.gitconfig', readonly: true }, ssh: { host: '~/.ssh', container: '$HOME/.ssh', readonly: true }, @@ -18,14 +19,6 @@ const MOUNT_PRESETS = { kube: { host: '~/.kube', container: '$HOME/.kube', readonly: true }, terraform: { host: '~/.terraform.d', container: '$HOME/.terraform.d', readonly: false }, gcloud: { host: '~/.config/gcloud', container: '$HOME/.config/gcloud', readonly: true }, - claude: { host: '~/.claude', container: '$HOME/.claude', readonly: true }, - codex: { host: '~/.config/codex', container: '$HOME/.config/codex', readonly: true }, - gemini: { host: '~/.config/gemini', container: '$HOME/.config/gemini', readonly: true }, - opencode: { - host: '~/.local/share/opencode', - container: '$HOME/.local/share/opencode', - readonly: true, - }, }; /** @@ -36,17 +29,35 @@ const MOUNT_PRESETS = { * - Forced: 'VAR=value' (always set to value) * - Empty: 'VAR=' (always set to empty string) */ -const { CLAUDE_AUTH_ENV_VARS } = require('./settings/claude-auth'); - -const ENV_PRESETS = { +const BASE_ENV_PRESETS = { aws: ['AWS_REGION', 'AWS_DEFAULT_REGION', 'AWS_PROFILE', 'AWS_PAGER='], azure: ['AZURE_SUBSCRIPTION_ID', 'AZURE_TENANT_ID', 'AZURE_CLIENT_ID'], gcloud: ['CLOUDSDK_CORE_PROJECT', 'GOOGLE_CLOUD_PROJECT'], kube: ['KUBECONFIG'], terraform: ['TF_VAR_*'], - claude: CLAUDE_AUTH_ENV_VARS, }; +function listProviderDockerPresetEntries() { + return listProviderMetadata().map((metadata) => [metadata.id, metadata.docker]); +} + +const MOUNT_PRESETS = Object.freeze({ + ...BASE_MOUNT_PRESETS, + ...Object.fromEntries( + listProviderDockerPresetEntries().map(([providerId, docker]) => [providerId, docker.mount]) + ), +}); + +const ENV_PRESETS = Object.freeze({ + ...BASE_ENV_PRESETS, + ...Object.fromEntries( + listProviderDockerPresetEntries().map(([providerId, docker]) => [ + providerId, + [...docker.envPassthrough], + ]) + ), +}); + /** * Resolve mount config to actual mount specs * @param {Array} config - Preset names or {host, container, readonly?} objects diff --git a/lib/provider-defaults.js b/lib/provider-defaults.js index ac3c6daf..92840680 100644 --- a/lib/provider-defaults.js +++ b/lib/provider-defaults.js @@ -10,6 +10,7 @@ // Cache provider defaults to avoid repeated instantiation let _providerDefaultsCache = null; +const { VALID_PROVIDERS, getProviderMetadata } = require('./provider-names'); /** * Build provider default settings by instantiating each provider @@ -25,16 +26,26 @@ function buildProviderDefaults() { const provider = getProvider(providerName); defaults[providerName] = provider.getDefaultSettings(); } catch (error) { - // If provider fails to instantiate, use basic defaults + const metadata = getProviderMetadata(providerName); console.warn(`Warning: Could not get defaults for ${providerName}: ${error.message}`); defaults[providerName] = { - maxLevel: 'level3', - minLevel: 'level1', - defaultLevel: 'level2', + maxLevel: metadata.defaultLevels.max, + minLevel: metadata.defaultLevels.min, + defaultLevel: metadata.defaultLevels.default, levelOverrides: {}, }; } } + for (const providerName of VALID_PROVIDERS) { + if (defaults[providerName]) continue; + const metadata = getProviderMetadata(providerName); + defaults[providerName] = { + maxLevel: metadata.defaultLevels.max, + minLevel: metadata.defaultLevels.min, + defaultLevel: metadata.defaultLevels.default, + levelOverrides: {}, + }; + } return defaults; } diff --git a/lib/provider-detection.js b/lib/provider-detection.js index 65573868..720c2760 100644 --- a/lib/provider-detection.js +++ b/lib/provider-detection.js @@ -36,6 +36,7 @@ function getHelpOutput(command, args = []) { const attempt = (flag) => { const result = spawnSync(command, [...args, flag], { encoding: 'utf8' }); + if (result.status !== 0) return ''; const output = `${result.stdout || ''}${result.stderr || ''}`; return output.trim(); }; @@ -50,6 +51,7 @@ function getHelpOutput(command, args = []) { function getVersionOutput(command, args = []) { if (!commandExists(command)) return ''; const result = spawnSync(command, [...args, '--version'], { encoding: 'utf8' }); + if (result.status !== 0) return ''; const output = `${result.stdout || ''}${result.stderr || ''}`; return output.trim(); } diff --git a/lib/provider-names.js b/lib/provider-names.js index 39f09264..188dc9df 100644 --- a/lib/provider-names.js +++ b/lib/provider-names.js @@ -1,19 +1,17 @@ -const PROVIDER_ALIASES = { - anthropic: 'claude', - openai: 'codex', - google: 'gemini', - claude: 'claude', - codex: 'codex', - gemini: 'gemini', - opencode: 'opencode', -}; +const registry = require('./agent-cli-provider/provider-registry'); -const VALID_PROVIDERS = ['claude', 'codex', 'gemini', 'opencode']; +const VALID_PROVIDERS = [...registry.providerIds]; +const KNOWN_PROVIDER_NAMES = [...registry.knownProviderNames]; +const PROVIDER_ALIASES = Object.freeze({ ...registry.providerAliasMap }); +const PROVIDER_CAPABILITIES = Object.freeze( + Object.fromEntries( + registry.listProviderRegistryEntries().map((entry) => [entry.id, entry.capabilities]) + ) +); function normalizeProviderName(name) { if (!name || typeof name !== 'string') return name; - const normalized = PROVIDER_ALIASES[name.toLowerCase()]; - return normalized || name; + return registry.normalizeProviderName(name); } function normalizeProviderSettings(providerSettings) { @@ -49,9 +47,31 @@ function normalizeProviderSettings(providerSettings) { return normalized; } +function getProviderMetadata(name) { + return registry.getProviderRegistryEntry(name); +} + +function listProviderMetadata() { + return registry.listProviderRegistryEntries(); +} + +function resolveProviderCommand(name) { + return registry.resolveProviderCommand(name); +} + +function providerSupportsCapability(name, capability) { + return registry.supportsProviderCapability(name, capability); +} + module.exports = { + KNOWN_PROVIDER_NAMES, PROVIDER_ALIASES, + PROVIDER_CAPABILITIES, VALID_PROVIDERS, + getProviderMetadata, + listProviderMetadata, normalizeProviderName, normalizeProviderSettings, + providerSupportsCapability, + resolveProviderCommand, }; diff --git a/lib/repo-settings.js b/lib/repo-settings.js index 05625079..6297c7f5 100644 --- a/lib/repo-settings.js +++ b/lib/repo-settings.js @@ -66,4 +66,18 @@ function readRepoSettings(startDir) { return { repoRoot, settings, settingsPath }; } -module.exports = { readRepoSettings }; +/** + * Write repo-local settings. + * + * @param {string} repoRoot - Repo root directory (as returned by readRepoSettings). + * @param {object} settings - Settings object to persist. + * @returns {string} The path written to. + */ +function writeRepoSettings(repoRoot, settings) { + const settingsPath = path.join(repoRoot, '.zeroshot', 'settings.json'); + fs.mkdirSync(path.dirname(settingsPath), { recursive: true }); + fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2), 'utf8'); + return settingsPath; +} + +module.exports = { readRepoSettings, writeRepoSettings }; diff --git a/lib/run-mode.js b/lib/run-mode.js index dc9fb9ea..32034bb7 100644 --- a/lib/run-mode.js +++ b/lib/run-mode.js @@ -1,11 +1,23 @@ -function resolveRunMode(options) { - if (options.ship) return options.docker ? 'ship+docker' : 'ship'; - if (options.pr) return options.docker ? 'pr+docker' : 'pr'; - if (options.docker) return 'docker'; - if (options.worktree) return 'worktree'; +const { resolveRunPlan } = require('./run-plan'); + +// The run-mode label is a VIEW of the canonical plan, never an independent +// cascade. Deriving it from the plan is what keeps the user-facing label and the +// actual isolation/delivery/autoMerge behavior from drifting apart. Callers that +// already hold a plan (e.g. the effective plan with env/settings folded in) use +// runModeFromPlan directly so the label reflects the SAME plan as behavior. +function runModeFromPlan({ isolation, delivery }) { + const dockerSuffix = isolation === 'docker' ? '+docker' : ''; + if (delivery === 'ship') return `ship${dockerSuffix}`; + if (delivery === 'pr') return `pr${dockerSuffix}`; + if (isolation === 'docker') return 'docker'; + if (isolation === 'worktree') return 'worktree'; return null; } +function resolveRunMode(options) { + return runModeFromPlan(resolveRunPlan(options)); +} + const RUN_MODE_LABELS = { ship: 'ship (worktree + PR + auto-merge)', 'ship+docker': 'ship (docker + PR + auto-merge)', @@ -19,4 +31,4 @@ function describeRunMode(mode) { return RUN_MODE_LABELS[mode] || 'local (no isolation)'; } -module.exports = { resolveRunMode, describeRunMode }; +module.exports = { resolveRunMode, runModeFromPlan, describeRunMode }; diff --git a/lib/run-plan.js b/lib/run-plan.js new file mode 100644 index 00000000..dde30f02 --- /dev/null +++ b/lib/run-plan.js @@ -0,0 +1,32 @@ +/** + * The single canonical run-mode resolver. + * + * Every consumer (orchestrator, daemon startup, CLI label) derives isolation, + * delivery, and autoMerge from THIS function and nowhere else. The raw + * worktree/docker/pr/ship/autoMerge booleans are inputs; the frozen plan is the + * one truth. Deriving autoMerge separately (the old `Boolean(ship)` scatter) is + * what let `--pr` merge and let the run-mode label drift from behavior. + */ + +function resolveIsolation(options) { + if (options.docker) return 'docker'; + if (options.worktree || options.pr || options.ship) return 'worktree'; + return 'none'; +} + +function resolveDelivery(options) { + // Explicit autoMerge (e.g. a future `--auto-merge` flag) is ship-equivalent: + // "merge it" implies the ship delivery. This is the ONLY place that intent is + // interpreted, so it can never be silently overwritten downstream. + if (options.ship || options.autoMerge === true) return 'ship'; + if (options.pr) return 'pr'; + return 'none'; +} + +function resolveRunPlan(options = {}) { + const isolation = resolveIsolation(options); + const delivery = resolveDelivery(options); + return Object.freeze({ isolation, delivery, autoMerge: delivery === 'ship' }); +} + +module.exports = { resolveRunPlan }; diff --git a/lib/settings.js b/lib/settings.js index a7428da3..c5b5047f 100644 --- a/lib/settings.js +++ b/lib/settings.js @@ -33,6 +33,14 @@ function getSettingsFile() { ); } +/** + * Whether the global settings file exists on disk (vs. running on defaults). + * @returns {boolean} + */ +function settingsFileExists() { + return fs.existsSync(getSettingsFile()); +} + // Import provider defaults from separate module to avoid circular dependency const { getProviderDefaults, clearProviderDefaultsCache } = require('./provider-defaults'); @@ -103,6 +111,7 @@ const DEFAULT_SETTINGS_BASE = { }, defaultConfig: 'conductor-bootstrap', defaultDocker: false, + defaultDelivery: 'none', // 'none' | 'pr' | 'ship' - folded into resolveEffectiveRunPlan same as defaultDocker strictSchema: true, // true = reliable json output (default), false = live streaming (may crash - see bold-meadow-11) logLevel: 'normal', // Auto-update settings @@ -113,7 +122,7 @@ const DEFAULT_SETTINGS_BASE = { // Example: 'ccr code' for claude-code-router integration claudeCommand: 'claude', // Docker isolation mounts - preset names or {host, container, readonly?} objects - // Valid presets: gh, git, ssh, aws, azure, kube, terraform, gcloud, claude, codex, gemini, opencode + // Valid presets: infrastructure presets plus provider ids from the provider registry dockerMounts: ['gh', 'git', 'ssh'], // Extra env vars to pass to Docker container (in addition to preset-implied ones) // Supports: VAR (if set), VAR_* (pattern), VAR=value (forced), VAR= (empty) @@ -323,7 +332,7 @@ function validateProviderSettings(value) { for (const [providerName, settings] of Object.entries(normalizedSettings || {})) { if (!VALID_PROVIDERS.includes(providerName)) { - return `Unknown provider in providerSettings: ${providerName}`; + return `Unknown provider in providerSettings: ${providerName}. Valid providers: ${VALID_PROVIDERS.join(', ')}`; } // Delegate validation to the provider @@ -391,6 +400,10 @@ function validateSetting(key, value) { return `Invalid log level: ${value}. Valid levels: quiet, normal, verbose`; } + if (key === 'defaultDelivery' && !['none', 'pr', 'ship'].includes(value)) { + return `Invalid defaultDelivery: ${value}. Valid: none, pr, ship`; + } + if (key === 'claudeCommand') { return validateClaudeCommand(value); } @@ -528,6 +541,7 @@ module.exports = { coerceValue, DEFAULT_SETTINGS, getSettingsFile, + settingsFileExists, getClaudeCommand, // Model validation exports MODEL_HIERARCHY, @@ -535,6 +549,7 @@ module.exports = { validateModelAgainstMax, // Provider defaults exports clearProviderDefaultsCache, + mapLegacyModelToLevel, // Backward compatibility: SETTINGS_FILE as getter (reads env var dynamically) get SETTINGS_FILE() { return getSettingsFile(); diff --git a/lib/settings/claude-auth.js b/lib/settings/claude-auth.js index 22632bcd..7a369991 100644 --- a/lib/settings/claude-auth.js +++ b/lib/settings/claude-auth.js @@ -3,6 +3,8 @@ * Extracted from settings.js for provider-specific isolation */ +const { getProviderMetadata } = require('../provider-names'); + /** * Anthropic API key prefix for validation */ @@ -11,12 +13,7 @@ const ANTHROPIC_KEY_PREFIX = 'sk-ant-'; /** * Environment variables used for Claude authentication */ -const CLAUDE_AUTH_ENV_VARS = [ - 'ANTHROPIC_API_KEY', - 'AWS_BEARER_TOKEN_BEDROCK', - 'AWS_REGION', - 'CLAUDE_CODE_USE_BEDROCK', -]; +const CLAUDE_AUTH_ENV_VARS = [...getProviderMetadata('claude').docker.envPassthrough]; /** * Validate an Anthropic API key format diff --git a/lib/setup-apply.js b/lib/setup-apply.js new file mode 100644 index 00000000..3501cb8d --- /dev/null +++ b/lib/setup-apply.js @@ -0,0 +1,300 @@ +/** + * `zeroshot setup apply` — writes decisions from #A's plan/decision contract + * (lib/setup-plan.js) to global settings, repo-local settings, and the undo + * journal (lib/setup-journal.js). + * + * Fail-closed: every input decisionId + value is resolved and validated + * against its domain before ANY write happens. Writes are then applied only + * to settings keys a run-mode resolver actually reads (setup-plan.js's + * CONSUMED_PATHS/isConsumedPath — the same set that filters proposedWrites) + * — a settings key nobody reads is dead config, the exact drift the + * canonical-path rule (issue #605) forbids. + */ + +const { resolveDecisionPath, domainFor, isConsumedPath, CONSUMED_PATHS } = require('./setup-plan'); +const { validateMountConfig, validateEnvPassthrough } = require('./docker-config'); +const { VALID_PROVIDERS } = require('./provider-names'); +const { VALID_MODELS, mapLegacyModelToLevel } = require('./settings'); +const { + loadJournal, + saveJournal, + upsertJournalEntry, + getNestedValue, + setNestedValue, + deepEqual, +} = require('./setup-journal'); + +const SECRET_PATTERN = /token|secret|password|api[_-]?key|credential/i; + +function assertSecretSafePath(targetPath) { + if (SECRET_PATTERN.test(targetPath)) { + throw new Error(`Refusing to write secret-shaped settings path: ${targetPath}`); + } +} + +function domainError(decisionId, value) { + return new Error( + `Invalid value for decision "${decisionId}": expected ${domainFor(decisionId)}, got ${JSON.stringify(value)}` + ); +} + +// Resolves a submitted decision value into the raw form actually stored at +// its target settings path, validating it against #A's domain first. +// (defaultIsolation's domain is worktree|docker|none, but its target path is +// the boolean settings.defaultDocker — this is where that translation lives.) +function convertDecisionValue({ decisionId, value, globalSettings, deps }) { + switch (decisionId) { + case 'defaultProvider': + if (typeof value !== 'string' || !deps.VALID_PROVIDERS.includes(value)) { + throw domainError(decisionId, value); + } + return value; + + case 'defaultIsolation': + if (!['worktree', 'docker', 'none'].includes(value)) throw domainError(decisionId, value); + return value === 'docker'; + + case 'allowLocalNoIsolation': + if (typeof value !== 'boolean') throw domainError(decisionId, value); + return value; + + case 'defaultDelivery': + if (!['none', 'pr', 'ship'].includes(value)) throw domainError(decisionId, value); + return value; + + case 'defaultIssueSource': + if (typeof value !== 'string' || !deps.listIssueProviders().includes(value)) { + throw domainError(decisionId, value); + } + return value; + + case 'prBase': + if (typeof value !== 'string' || value.trim() === '') throw domainError(decisionId, value); + return value; + + case 'dockerMounts': { + const err = validateMountConfig(value); + if (err) throw new Error(`Invalid value for decision "${decisionId}": ${err}`); + return value; + } + + case 'dockerEnvPassthrough': { + const err = validateEnvPassthrough(value); + if (err) throw new Error(`Invalid value for decision "${decisionId}": ${err}`); + return value; + } + + case 'updatePolicy': + if (!['off', 'notify', 'auto'].includes(value)) throw domainError(decisionId, value); + return value; + + default: + if (decisionId.startsWith('providerLevel.')) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw domainError(decisionId, value); + } + for (const key of ['min', 'default', 'max']) { + if (!VALID_MODELS.includes(value[key])) throw domainError(decisionId, value); + } + const providerName = decisionId.slice('providerLevel.'.length); + const existing = getNestedValue(globalSettings, `providerSettings.${providerName}`) || {}; + return { + ...existing, + minLevel: mapLegacyModelToLevel(value.min), + defaultLevel: mapLegacyModelToLevel(value.default), + maxLevel: mapLegacyModelToLevel(value.max), + }; + } + throw new Error(`Unknown decision ID: ${decisionId}`); + } +} + +function defaultApplyDeps() { + const fs = require('fs'); + const { loadSettings, saveSettings } = require('./settings'); + const { readRepoSettings, writeRepoSettings } = require('./repo-settings'); + const { checkGhAuth } = require('../src/preflight'); + const { listProviders: listIssueProviders } = require('../src/issue-providers'); + + return { + readFile: (filePath) => fs.readFileSync(filePath, 'utf8'), + loadSettings, + saveSettings, + readRepoSettings, + writeRepoSettings, + checkGhAuth, + listIssueProviders, + VALID_PROVIDERS, + now: () => new Date().toISOString(), + }; +} + +// Phase 1: resolve + validate EVERY input decision before any write happens +// (fail-closed — a single bad decisionId or out-of-domain value rejects the +// whole request, never a partial apply). +function resolveAndValidateDecisions(input, globalSettings, deps) { + const resolved = []; + for (const [decisionId, value] of Object.entries(input)) { + const target = resolveDecisionPath(decisionId); + if (!target) { + throw new Error(`Unknown decision ID: ${decisionId}`); + } + assertSecretSafePath(target.path); + const writeValue = convertDecisionValue({ decisionId, value, globalSettings, deps }); + resolved.push({ decisionId, target, inputValue: value, writeValue }); + } + return resolved; +} + +// Decides the outcome for a single resolved decision: a skip (with reason) +// or the write to perform. Returns the write descriptor rather than mutating +// anything, so the caller controls when settings objects/journal are touched. +function resolveDecisionOutcome( + { decisionId, target, inputValue, writeValue }, + { globalSettings, repoSettings, allowRiskyDefaults } +) { + const settingsObj = target.scope === 'repo' ? repoSettings : globalSettings; + const currentValue = getNestedValue(settingsObj, target.path) ?? null; + const base = { decisionId, from: currentValue, to: writeValue }; + + if (decisionId === 'defaultDelivery' && inputValue === 'ship' && !allowRiskyDefaults) { + return { result: { ...base, applied: false, skippedReason: 'requires-explicit-opt-in' } }; + } + if (!isConsumedPath(target.scope, target.path)) { + return { result: { ...base, applied: false, skippedReason: 'no-consumer' } }; + } + if (deepEqual(currentValue, writeValue)) { + return { result: { ...base, applied: false, skippedReason: 'unchanged' } }; + } + + return { + result: { ...base, applied: true }, + write: { + settingsObj, + scope: target.scope, + path: target.path, + writeValue, + priorValue: currentValue, + }, + }; +} + +// Applies one decision's write to its in-memory settings object and journals +// it (deferred persistence — the caller flushes to disk once at the end). +function applyWrite(write, { repoRoot, journal, deps }) { + setNestedValue(write.settingsObj, write.path, write.writeValue); + upsertJournalEntry(journal, { + scope: write.scope, + path: write.path, + repoRoot: write.scope === 'repo' ? repoRoot : null, + priorValue: write.priorValue, + appliedValue: write.writeValue, + appliedAt: deps.now(), + }); + return write.scope; +} + +function persistIfDirty({ + globalDirty, + repoDirty, + globalSettings, + repoSettings, + repoRoot, + journal, + deps, +}) { + if (globalDirty) deps.saveSettings(globalSettings); + if (repoDirty) deps.writeRepoSettings(repoRoot, repoSettings); + if (globalDirty || repoDirty) saveJournal(journal); +} + +// Phase 2: write every resolved decision, journaling each mutation and +// skipping (never throwing) decisions that are unchanged, risky-without-optin, +// or targeted at a settings key no resolver consumes. +function writeResolvedDecisions( + resolved, + { globalSettings, repoSettings, repoRoot, journal, allowRiskyDefaults, deps } +) { + const results = []; + let globalDirty = false; + let repoDirty = false; + + for (const decision of resolved) { + const { result, write } = resolveDecisionOutcome(decision, { + globalSettings, + repoSettings, + allowRiskyDefaults, + }); + results.push(result); + if (!write) continue; + + const scope = applyWrite(write, { repoRoot, journal, deps }); + if (scope === 'repo') repoDirty = true; + else globalDirty = true; + } + + persistIfDirty({ globalDirty, repoDirty, globalSettings, repoSettings, repoRoot, journal, deps }); + + return results; +} + +/** + * Apply a decisions file `{ "": , ... }` to settings. + * + * @param {Object} params + * @param {string} params.decisionsPath - Path to the decisions JSON file. + * @param {string} params.cwd - Working directory (for repo-scope settings lookup). + * @param {boolean} [params.allowRiskyDefaults] - Required to store defaultDelivery='ship'. + * @param {Object} [params.deps] - Injected dependencies (for testing). + * @returns {Array<{decisionId: string, applied: boolean, from: *, to: *, skippedReason?: string}>} + */ +function applyDecisions({ decisionsPath, cwd, allowRiskyDefaults = false, deps = {} }) { + const resolvedDeps = { ...defaultApplyDeps(), ...deps }; + + let input; + try { + input = JSON.parse(resolvedDeps.readFile(decisionsPath)); + } catch (err) { + throw new Error(`Failed to read decisions file "${decisionsPath}": ${err.message}`); + } + if (!input || typeof input !== 'object' || Array.isArray(input)) { + throw new Error('Decisions file must be a JSON object of { decisionId: value }'); + } + + const globalSettings = resolvedDeps.loadSettings(); + const { repoRoot, settings: repoSettingsRaw } = resolvedDeps.readRepoSettings(cwd); + const repoSettings = repoSettingsRaw || {}; + + const resolved = resolveAndValidateDecisions(input, globalSettings, resolvedDeps); + + const results = writeResolvedDecisions(resolved, { + globalSettings, + repoSettings, + repoRoot, + journal: loadJournal(), + allowRiskyDefaults, + deps: resolvedDeps, + }); + + // Never store provider-auth secrets: print the login command instead. + const issueSourceApplied = results.find( + (r) => r.decisionId === 'defaultIssueSource' && r.applied + ); + if (issueSourceApplied && issueSourceApplied.to === 'github') { + const auth = resolvedDeps.checkGhAuth(); + if (!auth || !auth.authenticated) { + console.log('Run: gh auth login'); + } + } + + return results; +} + +module.exports = { + applyDecisions, + resolveAndValidateDecisions, + writeResolvedDecisions, + assertSecretSafePath, + isConsumedPath, + CONSUMED_PATHS, +}; diff --git a/lib/setup-journal.js b/lib/setup-journal.js new file mode 100644 index 00000000..1baa76b2 --- /dev/null +++ b/lib/setup-journal.js @@ -0,0 +1,109 @@ +/** + * Undo journal for `zeroshot setup apply` / `zeroshot setup undo`. + * + * One journal entry per setting this setup owns: { scope, path, repoRoot, + * priorValue, appliedValue, appliedAt }. `priorValue: null` means the key + * did not exist before apply, so undo deletes it rather than restoring it. + * Shared by lib/setup-apply.js (writer) and lib/setup-undo.js (reader) so the + * nested-path mutation and equality semantics can't drift between the two. + */ + +const fs = require('fs'); +const path = require('path'); +const { getSettingsFile } = require('./settings'); +const { getNestedValue } = require('./setup-plan'); + +function getJournalPath() { + return path.join(path.dirname(getSettingsFile()), 'setup-undo-journal.json'); +} + +function loadJournal() { + const journalPath = getJournalPath(); + if (!fs.existsSync(journalPath)) { + return { version: 1, entries: [] }; + } + try { + const parsed = JSON.parse(fs.readFileSync(journalPath, 'utf8')); + if (!parsed || !Array.isArray(parsed.entries)) { + return { version: 1, entries: [] }; + } + return parsed; + } catch { + return { version: 1, entries: [] }; + } +} + +function saveJournal(journal) { + const journalPath = getJournalPath(); + fs.mkdirSync(path.dirname(journalPath), { recursive: true }); + fs.writeFileSync(journalPath, JSON.stringify(journal, null, 2), 'utf8'); +} + +function entryKey(entry) { + return `${entry.scope}:${entry.repoRoot || ''}:${entry.path}`; +} + +// Re-applying an already-journaled write updates appliedValue/appliedAt but +// keeps the original priorValue — that's the true pre-apply state undo must +// restore to, and it must not drift across repeated `apply` runs. +function upsertJournalEntry(journal, entry) { + const key = entryKey(entry); + const existingIndex = journal.entries.findIndex((e) => entryKey(e) === key); + if (existingIndex === -1) { + journal.entries.push(entry); + return; + } + journal.entries[existingIndex] = { + ...entry, + priorValue: journal.entries[existingIndex].priorValue, + }; +} + +function setNestedValue(target, pathStr, value) { + const keys = pathStr.split('.'); + let node = target; + for (let i = 0; i < keys.length - 1; i++) { + const key = keys[i]; + if (typeof node[key] !== 'object' || node[key] === null) { + node[key] = {}; + } + node = node[key]; + } + node[keys[keys.length - 1]] = value; +} + +function deleteNestedKey(target, pathStr) { + const keys = pathStr.split('.'); + let node = target; + for (let i = 0; i < keys.length - 1; i++) { + const key = keys[i]; + if (typeof node[key] !== 'object' || node[key] === null) return; + node = node[key]; + } + delete node[keys[keys.length - 1]]; +} + +function deepEqual(a, b) { + if (a === b) return true; + if (a === null || b === null || typeof a !== 'object' || typeof b !== 'object') return false; + if (Array.isArray(a) !== Array.isArray(b)) return false; + if (Array.isArray(a)) { + if (a.length !== b.length) return false; + return a.every((v, i) => deepEqual(v, b[i])); + } + const aKeys = Object.keys(a); + const bKeys = Object.keys(b); + if (aKeys.length !== bKeys.length) return false; + return aKeys.every((k) => Object.prototype.hasOwnProperty.call(b, k) && deepEqual(a[k], b[k])); +} + +module.exports = { + getJournalPath, + loadJournal, + saveJournal, + upsertJournalEntry, + getNestedValue, + setNestedValue, + deleteNestedKey, + deepEqual, +}; diff --git a/lib/setup-plan.js b/lib/setup-plan.js new file mode 100644 index 00000000..388b1a2f --- /dev/null +++ b/lib/setup-plan.js @@ -0,0 +1,406 @@ +/** + * Setup plan — read-only facts collector + the pinned, versioned setup contract. + * + * buildSetupPlan() is pure over its injected inputs: no direct process.env/fs + * reads for config data, no writes, no prompts. Everything downstream (apply, + * undo, the TTY wizard, agents) consumes the object this returns, so its shape + * (schemaVersion, facts, decisions, recommended, risk, proposedWrites) and the + * decisionId registry below are a stable contract — do not rename IDs or add a + * parallel run-mode field without updating every consumer. + */ + +const path = require('path'); + +const SCHEMA_VERSION = 1; + +// Canonical settings key each decision maps to. `defaultIsolation` and +// `defaultDelivery` MUST stay pinned to the keys resolveEffectiveRunPlan +// already reads (today: settings.defaultDocker) — see the canonical-path +// rule in issue #605. Never introduce a second parallel key here. +const DECISION_PATHS = { + defaultProvider: { scope: 'global', path: 'defaultProvider' }, + defaultIsolation: { scope: 'global', path: 'defaultDocker' }, + allowLocalNoIsolation: { scope: 'global', path: 'allowLocalNoIsolation' }, + defaultDelivery: { scope: 'global', path: 'defaultDelivery' }, + defaultIssueSource: { scope: 'global', path: 'defaultIssueSource' }, + prBase: { scope: 'repo', path: 'prBase' }, + dockerMounts: { scope: 'global', path: 'dockerMounts' }, + dockerEnvPassthrough: { scope: 'global', path: 'dockerEnvPassthrough' }, + updatePolicy: { scope: 'global', path: 'updatePolicy' }, +}; + +function providerLevelDecisionId(providerName) { + return `providerLevel.${providerName}`; +} + +// Settings keys a run-mode resolver actually reads today. Shared by +// buildProposedWrites below (never propose a write nobody will read) and by +// lib/setup-apply.js (never perform a write nobody will read) — one +// canonical answer to "is this key consumed?" so the two can't drift. +const CONSUMED_PATHS = new Set([ + 'global:defaultProvider', + 'global:defaultDocker', + 'global:defaultDelivery', + 'global:defaultIssueSource', + 'global:dockerMounts', + 'global:dockerEnvPassthrough', + // No resolver consumes this yet; consumption is deferred to a future + // update-policy issue, but issue #606 explicitly sanctions writing it now. + 'global:updatePolicy', +]); + +function isConsumedPath(scope, targetPath) { + if (scope === 'global' && targetPath.startsWith('providerSettings.')) return true; + return CONSUMED_PATHS.has(`${scope}:${targetPath}`); +} + +// providerLevel. decisionIds map to a per-provider settings key +// (providerSettings.) not present in DECISION_PATHS above, since +// the provider name is only known at runtime. +function resolveDecisionPath(decisionId) { + if (decisionId.startsWith('providerLevel.')) { + const providerName = decisionId.slice('providerLevel.'.length); + return { scope: 'global', path: `providerSettings.${providerName}` }; + } + return DECISION_PATHS[decisionId] || null; +} + +function getNestedValue(source, pathStr) { + return pathStr + .split('.') + .reduce((acc, key) => (acc === null || acc === undefined ? undefined : acc[key]), source); +} + +function defaultDeps() { + const { commandExists, getCommandPath } = require('./provider-detection'); + const { checkDocker, checkGhAuth } = require('../src/preflight'); + const { execSync } = require('../src/lib/safe-exec'); + const { listProviders, getProvider } = require('../src/providers'); + const { getProviderDefaults } = require('./provider-defaults'); + const packageJson = require('../package.json'); + + return { + commandExists, + getCommandPath, + checkDocker, + checkGhAuth, + execSync, + listProviders, + getProvider, + getProviderDefaults, + getNodeVersion: () => process.version, + getPackageVersion: () => packageJson.version, + }; +} + +function detectInstallSource(cwd, env) { + if (env.npm_config_global === 'true') return 'npm-global'; + if (env.npm_execpath && /_npx|npx/.test(env.npm_execpath)) return 'npx'; + try { + const ownNodeModules = path.join(__dirname, '..', 'node_modules'); + if (cwd && (cwd === path.join(__dirname, '..') || cwd.startsWith(ownNodeModules))) { + return 'local'; + } + } catch { + // fall through to unknown + } + return 'unknown'; +} + +function buildNodeFacts({ cwd, env, deps }) { + return { + version: deps.getNodeVersion(), + packageVersion: deps.getPackageVersion(), + installSource: detectInstallSource(cwd, env), + }; +} + +function buildProviderFacts(deps) { + const providers = {}; + for (const name of deps.listProviders()) { + let cliCommand = name; + try { + cliCommand = deps.getProvider(name).cliCommand || name; + } catch { + // Provider metadata unavailable — fall back to the provider name as the CLI command. + } + const cliAvailable = deps.commandExists(cliCommand); + providers[name] = { + cliAvailable, + path: cliAvailable ? deps.getCommandPath(cliCommand) : null, + }; + } + return providers; +} + +function safeExecTrim(deps, command, cwd) { + try { + return deps.execSync(command, { cwd, stdio: 'pipe', encoding: 'utf8' }).trim(); + } catch { + return null; + } +} + +function buildGitFacts({ cwd, deps }) { + const isRepo = safeExecTrim(deps, 'git rev-parse --is-inside-work-tree', cwd) === 'true'; + const ghAvailable = deps.commandExists('gh'); + + if (!isRepo) { + return { isRepo: false, branch: null, remote: null, ghAvailable, ghAuthed: null }; + } + + const branch = safeExecTrim(deps, 'git rev-parse --abbrev-ref HEAD', cwd); + const remote = safeExecTrim(deps, 'git remote get-url origin', cwd); + const ghAuthed = ghAvailable ? !!deps.checkGhAuth().authenticated : null; + + return { isRepo: true, branch, remote, ghAvailable, ghAuthed }; +} + +function buildFacts({ cwd, settings, repoSettings, env, deps }) { + return { + node: buildNodeFacts({ cwd, env, deps }), + providers: buildProviderFacts(deps), + git: buildGitFacts({ cwd, deps }), + docker: { available: !!deps.checkDocker().available }, + settings: { + hasGlobal: settings?.__meta ? !!settings.__meta.fileExists : true, + hasRepo: repoSettings !== null && repoSettings !== undefined, + }, + }; +} + +function inferIssueSource(facts) { + const { parseGitRemoteUrl } = require('./git-remote-utils'); + const parsed = parseGitRemoteUrl(facts.git.remote); + return parsed?.provider || null; +} + +function inferPrBase(cwd, deps) { + const branch = safeExecTrim(deps, 'git rev-parse --abbrev-ref origin/HEAD', cwd); + if (!branch) return null; + return branch.replace(/^origin\//, ''); +} + +function buildProviderLevelRecommendation(name, deps) { + const providerDefaults = deps.getProviderDefaults()[name] || {}; + const minLevel = providerDefaults.minLevel; + const defaultLevel = providerDefaults.defaultLevel; + const maxLevel = providerDefaults.maxLevel; + const overrides = providerDefaults.levelOverrides || {}; + + try { + const provider = deps.getProvider(name); + return { + min: provider.resolveModelSpec(minLevel, overrides).model, + default: provider.resolveModelSpec(defaultLevel, overrides).model, + max: provider.resolveModelSpec(maxLevel, overrides).model, + }; + } catch { + return { min: null, default: null, max: null }; + } +} + +function buildRecommendedAndRisk({ cwd, facts, env, deps }) { + const recommended = {}; + const risk = {}; + + const firstAvailableProvider = Object.entries(facts.providers).find( + ([, info]) => info.cliAvailable + ); + recommended.defaultProvider = firstAvailableProvider ? firstAvailableProvider[0] : 'claude'; + risk.defaultProvider = firstAvailableProvider ? 'low' : 'medium'; + + for (const name of Object.keys(facts.providers)) { + recommended[providerLevelDecisionId(name)] = buildProviderLevelRecommendation(name, deps); + risk[providerLevelDecisionId(name)] = 'low'; + } + + if (facts.git.isRepo) { + recommended.defaultIsolation = 'worktree'; + risk.defaultIsolation = 'low'; + } else if (facts.docker.available) { + recommended.defaultIsolation = 'docker'; + risk.defaultIsolation = 'low'; + } else { + recommended.defaultIsolation = 'none'; + risk.defaultIsolation = 'high'; + } + + recommended.allowLocalNoIsolation = false; + risk.allowLocalNoIsolation = 'low'; + + recommended.defaultDelivery = 'none'; + risk.defaultDelivery = 'low'; + + const inferredIssueSource = inferIssueSource(facts); + recommended.defaultIssueSource = inferredIssueSource || 'github'; + risk.defaultIssueSource = inferredIssueSource ? 'low' : 'medium'; + + const inferredPrBase = inferPrBase(cwd, deps); + recommended.prBase = inferredPrBase || 'main'; + risk.prBase = inferredPrBase ? 'low' : 'medium'; + + recommended.dockerMounts = ['gh', 'git', 'ssh']; + risk.dockerMounts = 'low'; + + recommended.dockerEnvPassthrough = []; + risk.dockerEnvPassthrough = 'low'; + + const nonInteractive = env.CI === 'true' || env.__isTTY === false; + recommended.updatePolicy = nonInteractive ? 'off' : 'notify'; + risk.updatePolicy = 'low'; + + return { recommended, risk, inferredIssueSource, inferredPrBase }; +} + +function currentValueFor(decisionId, settings, repoSettings) { + const target = resolveDecisionPath(decisionId); + if (!target) return null; + const source = target.scope === 'repo' ? repoSettings || {} : settings || {}; + const value = getNestedValue(source, target.path); + return value === undefined ? null : value; +} + +function buildDecisions({ facts, settings, repoSettings, inferredIssueSource, inferredPrBase }) { + const decisions = []; + const hasGlobal = facts.settings.hasGlobal; + const hasRepo = facts.settings.hasRepo; + + const globalDecisionIds = [ + 'defaultProvider', + ...Object.keys(facts.providers).map(providerLevelDecisionId), + 'defaultIsolation', + 'allowLocalNoIsolation', + 'defaultDelivery', + 'defaultIssueSource', + 'dockerMounts', + 'dockerEnvPassthrough', + 'updatePolicy', + ]; + + for (const decisionId of globalDecisionIds) { + const shouldInclude = + !hasGlobal || (decisionId === 'defaultIssueSource' && !inferredIssueSource); + if (!shouldInclude) continue; + decisions.push({ + decisionId, + domain: domainFor(decisionId), + currentValue: currentValueFor(decisionId, settings, repoSettings), + }); + } + + if (!hasRepo || !inferredPrBase) { + decisions.push({ + decisionId: 'prBase', + domain: domainFor('prBase'), + currentValue: currentValueFor('prBase', settings, repoSettings), + }); + } + + return decisions; +} + +function domainFor(decisionId) { + if (decisionId.startsWith('providerLevel.')) return '{ min, default, max } of haiku|sonnet|opus'; + const domains = { + defaultProvider: 'claude | codex | gemini | opencode', + defaultIsolation: 'worktree | docker | none', + allowLocalNoIsolation: 'boolean', + defaultDelivery: 'none | pr | ship', + defaultIssueSource: 'github | gitlab | jira | azure-devops | linear', + prBase: 'string (branch)', + dockerMounts: 'array of presets/objects', + dockerEnvPassthrough: 'string[]', + updatePolicy: 'off | notify | auto', + }; + return domains[decisionId] || 'unknown'; +} + +function buildProposedWrites({ decisions, recommended }) { + const writes = []; + for (const decision of decisions) { + const target = resolveDecisionPath(decision.decisionId); + if (!target) continue; + // A decision can still be surfaced for the user to make (e.g. a future + // wizard asking about prBase), but proposing a *write* for a settings key + // no resolver reads would advertise a write that apply will always skip — + // dead config. Only propose writes apply will actually perform. + if (!isConsumedPath(target.scope, target.path)) continue; + let to = recommended[decision.decisionId]; + if (decision.decisionId === 'defaultIsolation') { + to = to === 'docker'; + } + if (to === decision.currentValue) continue; + writes.push({ + scope: target.scope, + path: target.path, + from: decision.currentValue ?? null, + to, + decisionId: decision.decisionId, + }); + } + return writes; +} + +/** + * Build the pinned, versioned setup contract. Pure over injected inputs — + * performs only cheap, read-only detection (no writes, no prompts). + * + * @param {Object} params + * @param {string} params.cwd + * @param {Object} params.settings - loaded global settings (optionally with __meta.fileExists) + * @param {Object|null} params.repoSettings - repo-local settings object, or null + * @param {Object} params.env - caller-provided env (e.g. process.env), plus optional __isTTY + * @param {Object} [params.deps] - injected dependencies (for testing) + * @returns {Object} plan + */ +function buildSetupPlan({ cwd, settings, repoSettings, env, deps } = {}) { + const resolvedDeps = { ...defaultDeps(), ...(deps || {}) }; + const resolvedEnv = env || {}; + const resolvedCwd = cwd || '.'; + const resolvedSettings = settings || {}; + + const facts = buildFacts({ + cwd: resolvedCwd, + settings: resolvedSettings, + repoSettings, + env: resolvedEnv, + deps: resolvedDeps, + }); + + const { recommended, risk, inferredIssueSource, inferredPrBase } = buildRecommendedAndRisk({ + cwd: resolvedCwd, + facts, + env: resolvedEnv, + deps: resolvedDeps, + }); + + const decisions = buildDecisions({ + facts, + settings: resolvedSettings, + repoSettings, + inferredIssueSource, + inferredPrBase, + }); + + const proposedWrites = buildProposedWrites({ decisions, recommended }); + + return { + schemaVersion: SCHEMA_VERSION, + facts, + decisions, + recommended, + risk, + proposedWrites, + }; +} + +module.exports = { + buildSetupPlan, + resolveDecisionPath, + domainFor, + DECISION_PATHS, + getNestedValue, + isConsumedPath, + CONSUMED_PATHS, +}; diff --git a/lib/setup-undo.js b/lib/setup-undo.js new file mode 100644 index 00000000..6a4e7d92 --- /dev/null +++ b/lib/setup-undo.js @@ -0,0 +1,88 @@ +/** + * `zeroshot setup undo` — reverts writes made by `zeroshot setup apply` using + * the journal recorded at apply time (lib/setup-journal.js). + * + * Three-way conflict rule per journaled write, comparing `current` against + * `appliedValue`: + * - current === appliedValue -> restore priorValue (delete if null) + * - current === priorValue -> already-restored (no-op) + * - otherwise (changed elsewhere) -> skipped-modified, never clobbered + */ + +const { + loadJournal, + getNestedValue, + setNestedValue, + deleteNestedKey, + deepEqual, +} = require('./setup-journal'); + +function defaultUndoDeps() { + const { loadSettings, saveSettings } = require('./settings'); + const { readRepoSettings, writeRepoSettings } = require('./repo-settings'); + + return { loadSettings, saveSettings, readRepoSettings, writeRepoSettings }; +} + +/** + * Undo every journaled write, per the three-way conflict rule. + * + * @param {Object} [params] + * @param {Object} [params.deps] - Injected dependencies (for testing). + * @returns {Array<{scope: string, path: string, repoRoot: string|null, status: string, current?: *, wouldRestore?: *}>} + */ +function undo({ deps = {} } = {}) { + const resolvedDeps = { ...defaultUndoDeps(), ...deps }; + const journal = loadJournal(); + + const globalSettings = resolvedDeps.loadSettings(); + const repoSettingsCache = new Map(); + let globalDirty = false; + const dirtyRepoRoots = new Set(); + + function repoSettingsFor(repoRoot) { + if (!repoSettingsCache.has(repoRoot)) { + const { settings } = resolvedDeps.readRepoSettings(repoRoot); + repoSettingsCache.set(repoRoot, settings || {}); + } + return repoSettingsCache.get(repoRoot); + } + + const results = journal.entries.map((entry) => { + const settingsObj = entry.scope === 'repo' ? repoSettingsFor(entry.repoRoot) : globalSettings; + const current = getNestedValue(settingsObj, entry.path) ?? null; + + if (deepEqual(current, entry.priorValue)) { + return { ...entry, status: 'already-restored' }; + } + + if (!deepEqual(current, entry.appliedValue)) { + return { ...entry, status: 'skipped-modified', current, wouldRestore: entry.priorValue }; + } + + if (entry.priorValue === null) { + deleteNestedKey(settingsObj, entry.path); + } else { + setNestedValue(settingsObj, entry.path, entry.priorValue); + } + + if (entry.scope === 'repo') { + dirtyRepoRoots.add(entry.repoRoot); + } else { + globalDirty = true; + } + + return { ...entry, status: entry.priorValue === null ? 'deleted' : 'restored' }; + }); + + if (globalDirty) resolvedDeps.saveSettings(globalSettings); + for (const repoRoot of dirtyRepoRoots) { + resolvedDeps.writeRepoSettings(repoRoot, repoSettingsFor(repoRoot)); + } + + // Journal entries are left in place (not cleared) so a re-run reports + // 'already-restored' instead of finding nothing to undo. + return results; +} + +module.exports = { undo }; diff --git a/lib/start-cluster.js b/lib/start-cluster.js index d13a5874..ac1fd363 100644 --- a/lib/start-cluster.js +++ b/lib/start-cluster.js @@ -5,7 +5,8 @@ const { normalizeProviderName } = require('./provider-names'); const { getProvider } = require('../src/providers'); const { detectProvider } = require('../src/issue-providers'); const TemplateResolver = require('../src/template-resolver'); -const { resolveRunMode } = require('./run-mode'); +const { runModeFromPlan } = require('./run-mode'); +const { resolveRunPlan } = require('./run-plan'); const PACKAGE_ROOT = path.resolve(__dirname, '..'); @@ -198,8 +199,20 @@ function mergeRunOptions(options) { return envRunOptions ? { ...envRunOptions, ...options } : options; } -function resolveIsolation(mergedOptions, settings) { - return mergedOptions.docker || process.env.ZEROSHOT_DOCKER === '1' || settings.defaultDocker; +// The single producer of the run plan for a cluster start: fold env + settings +// into the flags, then resolve the canonical isolation/delivery/autoMerge plan. +// buildStartOptions reads EVERY mode field off this one plan — no field +// (isolation, worktree, autoPr, autoMerge, runMode) is derived independently. +function resolveEffectiveRunPlan(mergedOptions, settings) { + return resolveRunPlan({ + ...mergedOptions, + docker: anyTruthy( + mergedOptions.docker, + process.env.ZEROSHOT_DOCKER === '1', + settings.defaultDocker + ), + worktree: anyTruthy(mergedOptions.worktree, process.env.ZEROSHOT_WORKTREE === '1'), + }); } function resolveMergeQueue(mergedOptions) { @@ -238,14 +251,15 @@ function buildStartOptions({ forceProvider, }) { const mergedOptions = mergeRunOptions(options); + const plan = resolveEffectiveRunPlan(mergedOptions, settings); return { clusterId, cwd: resolveTargetCwd(), - isolation: resolveIsolation(mergedOptions, settings), + isolation: plan.isolation === 'docker', isolationImage: firstTruthy(mergedOptions.dockerImage, process.env.ZEROSHOT_DOCKER_IMAGE), - worktree: anyTruthy(mergedOptions.worktree, process.env.ZEROSHOT_WORKTREE === '1'), - autoPr: anyTruthy(mergedOptions.pr, process.env.ZEROSHOT_PR === '1'), - autoMerge: Boolean(mergedOptions.ship), + worktree: plan.isolation === 'worktree', + autoPr: plan.delivery !== 'none', + autoMerge: plan.autoMerge, autoPush: process.env.ZEROSHOT_PUSH === '1', modelOverride: optionalValue(modelOverride), providerOverride: optionalValue(providerOverride), @@ -257,7 +271,7 @@ function buildStartOptions({ mergeQueue: resolveMergeQueue(mergedOptions), closeIssue: resolveCloseIssue(mergedOptions), ship: mergedOptions.ship, - runMode: resolveRunMode(mergedOptions) || null, + runMode: runModeFromPlan(plan), requiredQualityGates: mergedOptions.requiredQualityGates, settings, }; @@ -281,6 +295,16 @@ function resolveConfigOrThrow({ return loadClusterConfig(orchestrator, resolvedPath, settings, providerOverride); } +function applyDefaultDeliveryOptions(options, settings) { + if (settings.defaultDelivery === 'pr') { + return { ...options, pr: true }; + } + if (settings.defaultDelivery === 'ship') { + return { ...options, ship: true }; + } + return options; +} + function startClusterWithInput(args, input) { const { orchestrator, config, configPath, configName, settings, providerOverride } = args; if (!orchestrator) { @@ -296,7 +320,7 @@ function startClusterWithInput(args, input) { }); const startOptions = buildStartOptions({ clusterId: args.clusterId, - options: args.options || {}, + options: applyDefaultDeliveryOptions(args.options || {}, settings), settings, providerOverride, modelOverride: args.modelOverride, @@ -330,6 +354,7 @@ module.exports = { resolveConfigPath, loadClusterConfig, buildStartOptions, + resolveEffectiveRunPlan, startClusterFromText, startClusterFromIssue, startClusterFromFile, diff --git a/lib/stream-json-parser.js b/lib/stream-json-parser.js index 6797795b..d1c7734a 100644 --- a/lib/stream-json-parser.js +++ b/lib/stream-json-parser.js @@ -4,10 +4,11 @@ * Provider parsing is delegated to the helper-backed runtime provider facade. */ -const { getProvider } = require('../src/providers'); +const { getProvider, listProviders } = require('../src/providers'); -const providerOrder = ['claude', 'codex', 'opencode', 'gemini']; -const providerParsers = providerOrder.map((name) => getProvider(name)); +function createProviderParsers() { + return listProviders().map((name) => getProvider(name)); +} function stripTimestampPrefix(line) { if (!line || typeof line !== 'string') return ''; @@ -28,7 +29,7 @@ function stripTimestampPrefix(line) { return trimmed; } -function parseEvent(line) { +function parseEvent(line, providerParsers = createProviderParsers()) { const content = stripTimestampPrefix(line); if (!content) return null; @@ -52,10 +53,11 @@ function collectEvent(events, event) { function parseChunk(chunk) { const events = []; const lines = String(chunk || '').split('\n'); + const providerParsers = createProviderParsers(); for (const line of lines) { if (!line.trim()) continue; - collectEvent(events, parseEvent(line)); + collectEvent(events, parseEvent(line, providerParsers)); } return events; diff --git a/package-lock.json b/package-lock.json index 209a3445..d418ea9f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@the-open-engine/zeroshot", - "version": "5.4.0", + "version": "6.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@the-open-engine/zeroshot", - "version": "5.4.0", + "version": "6.4.0", "hasInstallScript": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 853b4028..857f3096 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@the-open-engine/zeroshot", - "version": "5.4.0", + "version": "6.4.0", "description": "Multi-agent orchestration engine for Claude, Codex, and Gemini", "main": "src/orchestrator.js", "bin": { @@ -15,8 +15,10 @@ "pretest": "npm run build:agent-cli-provider", "test": "node tests/run-tests.js", "test:unit": "node tests/run-tests.js", - "test:slow": "mocha 'tests/integration/**/*.test.js' --timeout 180000", - "test:all": "npm run test && npm run test:slow", + "test:e2e": "mocha 'tests/e2e/**/*.test.js' --timeout 120000", + "test:e2e:docker": "bash tests/integration/e2e-isolation-and-auto.test.sh", + "test:slow": "mocha 'tests/integration/**/*.test.js' 'tests/providers/**/*.test.js' --timeout 180000", + "test:all": "npm run test && npm run test:e2e && npm run test:slow", "test:coverage": "c8 npm run test:unit", "test:coverage:report": "c8 --reporter=html npm run test:unit && echo 'Coverage report generated at coverage/index.html'", "postinstall": "node scripts/fix-node-pty-permissions.js && node scripts/check-path.js", @@ -26,6 +28,8 @@ "lint:agent-cli-provider": "eslint \"src/agent-cli-provider/**/*.ts\" \"tests/agent-cli-provider/**/*.ts\"", "build:agent-cli-provider": "tsc --project tsconfig.agent-cli-provider.build.json", "test:agent-cli-provider": "node --test tests/agent-cli-provider/*.test.js", + "test:providers:live": "npm run build:agent-cli-provider && node scripts/live-provider-smoke.js", + "check:agent-cli-provider": "npm run check:agent-cli-provider:ci", "check:agent-cli-provider:ci": "npm run typecheck:agent-cli-provider && npm run lint:agent-cli-provider && npm run build:agent-cli-provider && npm run test:agent-cli-provider", "dev:link": "npm link", "lint": "eslint .", diff --git a/scripts/live-provider-smoke.js b/scripts/live-provider-smoke.js new file mode 100644 index 00000000..56b84e94 --- /dev/null +++ b/scripts/live-provider-smoke.js @@ -0,0 +1,221 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const helper = require('../lib/agent-cli-provider'); + +const SENTINEL = process.env.ZEROSHOT_LIVE_SENTINEL || 'ZEROSHOT_LIVE_SMOKE_OK'; +const TIMEOUT_MS = Number(process.env.ZEROSHOT_LIVE_TIMEOUT_MS || 120000); +const DEFAULT_PROMPT = `Reply with exactly this text and nothing else: ${SENTINEL}`; + +function fail(message) { + console.error(message); + process.exitCode = 1; +} + +function usage() { + return [ + 'Live provider smoke requires explicit provider selection.', + '', + 'Examples:', + ' ZEROSHOT_LIVE_PROVIDERS=all npm run test:providers:live', + ' ZEROSHOT_LIVE_PROVIDERS=claude,codex,gemini npm run test:providers:live', + ' ZEROSHOT_LIVE_PROVIDERS=pi npm run test:providers:live', + ' ZEROSHOT_LIVE_PROVIDERS=copilot npm run test:providers:live', + ' ZEROSHOT_LIVE_PROVIDERS=gateway \\', + ' ZEROSHOT_LIVE_GATEWAY_BASE_URL=https://openrouter.ai/api/v1 \\', + ' ZEROSHOT_LIVE_GATEWAY_API_KEY=... \\', + ' ZEROSHOT_LIVE_GATEWAY_MODEL=openai/gpt-5.4 \\', + ' npm run test:providers:live', + '', + 'This command invokes real provider CLIs or a real gateway endpoint. It is intentionally', + 'not part of the normal CI suite because it can require user auth and paid API calls.', + ].join('\n'); +} + +function parseProviders() { + const raw = process.env.ZEROSHOT_LIVE_PROVIDERS; + if (!raw) { + console.error(usage()); + process.exit(2); + } + + const providers = []; + const allProviders = helper.listProviderAdapters(); + for (const item of raw.split(',')) { + const trimmed = item.trim(); + if (trimmed.toLowerCase() === 'all') { + for (const provider of allProviders) { + if (!providers.includes(provider)) providers.push(provider); + } + continue; + } + + const normalized = helper.normalizeProviderName(trimmed); + if (!normalized) continue; + if (!allProviders.includes(normalized)) { + throw new Error( + `Unknown provider "${item}". Valid providers: ${allProviders.join(', ')}, all` + ); + } + if (!providers.includes(normalized)) providers.push(normalized); + } + + if (providers.length === 0) { + throw new Error('ZEROSHOT_LIVE_PROVIDERS did not contain any provider names.'); + } + return providers; +} + +function readJsonEnv(key) { + const value = process.env[key]; + if (!value) return undefined; + try { + return JSON.parse(value); + } catch (error) { + throw new Error(`${key} must be valid JSON: ${error.message}`); + } +} + +function liveGatewayOptions(cwd) { + const baseUrl = process.env.ZEROSHOT_LIVE_GATEWAY_BASE_URL; + const apiKey = process.env.ZEROSHOT_LIVE_GATEWAY_API_KEY; + const model = process.env.ZEROSHOT_LIVE_GATEWAY_MODEL; + const headers = readJsonEnv('ZEROSHOT_LIVE_GATEWAY_HEADERS_JSON'); + + for (const [key, value] of [ + ['ZEROSHOT_LIVE_GATEWAY_BASE_URL', baseUrl], + ['ZEROSHOT_LIVE_GATEWAY_API_KEY', apiKey], + ['ZEROSHOT_LIVE_GATEWAY_MODEL', model], + ]) { + if (!value) throw new Error(`gateway live smoke requires ${key}.`); + } + + return { + cwd, + gateway: { + baseUrl, + apiKey, + model, + ...(headers === undefined ? {} : { headers }), + toolPolicy: { + roots: ['.'], + commands: [], + }, + }, + }; +} + +function providerOptions(provider, cwd) { + if (provider === 'gateway') return liveGatewayOptions(cwd); + + return { + cwd, + outputFormat: 'stream-json', + }; +} + +function eventText(event) { + if (!event || typeof event !== 'object') return ''; + if (event.type === 'text' || event.type === 'thinking') return event.text || ''; + if (event.type === 'result') { + return [event.result, event.text, event.message].filter(Boolean).join('\n'); + } + return ''; +} + +function summarizeResult(envelope) { + if (!envelope || typeof envelope !== 'object') return 'no envelope'; + if (!envelope.ok) { + const error = envelope.error || {}; + return `${error.code || 'error'}: ${error.message || 'unknown failure'}`; + } + + const result = envelope.result || {}; + const events = Array.isArray(result.events) ? result.events : []; + const text = events.map(eventText).join('\n').trim(); + const last = events.at(-1); + return JSON.stringify( + { + provider: envelope.provider, + exitCode: result.exitCode, + signal: result.signal, + timedOut: result.timedOut, + classification: result.classification || null, + eventCount: events.length, + lastEventType: last && typeof last === 'object' ? last.type : null, + text: text.slice(0, 500), + }, + null, + 2 + ); +} + +function assertSmokePassed(provider, response) { + if (response.exitCode !== 0) { + throw new Error( + `${provider} contract exited ${response.exitCode}: ${summarizeResult(response.envelope)}` + ); + } + if (!response.envelope.ok) { + throw new Error(`${provider} returned a contract error: ${summarizeResult(response.envelope)}`); + } + + const result = response.envelope.result || {}; + const events = Array.isArray(result.events) ? result.events : []; + const text = events.map(eventText).join('\n'); + const finalResult = events.findLast((event) => event && event.type === 'result'); + + if (result.timedOut) { + throw new Error(`${provider} timed out after ${result.timeoutMs || TIMEOUT_MS}ms.`); + } + if (finalResult && finalResult.success === false) { + throw new Error( + `${provider} produced a failed result event: ${summarizeResult(response.envelope)}` + ); + } + if (!text.includes(SENTINEL)) { + throw new Error( + `${provider} did not return sentinel ${SENTINEL}.\nResult:\n${summarizeResult(response.envelope)}` + ); + } +} + +async function smokeProvider(provider) { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), `zeroshot-live-${provider}-`)); + try { + const prompt = process.env.ZEROSHOT_LIVE_PROMPT || DEFAULT_PROMPT; + const response = await helper.runProviderExecutable( + JSON.stringify({ + schemaVersion: 1, + command: 'invoke', + provider, + context: prompt, + options: providerOptions(provider, tempDir), + timeoutMs: TIMEOUT_MS, + }) + ); + + assertSmokePassed(provider, response); + console.log(`✓ ${provider} live smoke returned ${SENTINEL}`); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +async function main() { + const providers = parseProviders(); + console.log(`Running live provider smoke for: ${providers.join(', ')}`); + console.log(`Sentinel: ${SENTINEL}`); + + for (const provider of providers) { + await smokeProvider(provider); + } +} + +main().catch((error) => { + fail(error.stack || error.message || String(error)); +}); diff --git a/src/agent-cli-provider/acp-stdio-runner.ts b/src/agent-cli-provider/acp-stdio-runner.ts new file mode 100644 index 00000000..a8626bbb --- /dev/null +++ b/src/agent-cli-provider/acp-stdio-runner.ts @@ -0,0 +1,314 @@ +/* eslint-disable max-lines-per-function */ +import { spawn } from 'node:child_process'; +import { omitProcessControlEnv, omitUnsafeProviderEnv } from './env-safety'; +import type { CommandSpec, ProviderId } from './types'; +import type { ProcessResult, ProcessRunnerOptions } from './process-runner'; + +const DEFAULT_TIMEOUT_KILL_GRACE_MS = 100; + +interface JsonRpcRequest { + readonly jsonrpc: '2.0'; + readonly id: number; + readonly method: string; + readonly params: Record; +} + +interface JsonRpcResponse { + readonly jsonrpc?: string; + readonly id?: number | string | null; + readonly result?: unknown; + readonly error?: unknown; +} + +interface PendingRequest { + resolve(response: JsonRpcResponse): void; + reject(error: Error): void; +} + +function spawnOptions(commandSpec: CommandSpec): { + readonly shell: false; + readonly env: NodeJS.ProcessEnv; + readonly stdio: ['pipe', 'pipe', 'pipe']; + cwd?: string; +} { + const options: { + readonly shell: false; + readonly env: NodeJS.ProcessEnv; + readonly stdio: ['pipe', 'pipe', 'pipe']; + cwd?: string; + } = { + shell: false, + env: { ...omitProcessControlEnv(process.env), ...omitUnsafeProviderEnv(commandSpec.env) }, + stdio: ['pipe', 'pipe', 'pipe'], + }; + if (commandSpec.cwd !== undefined) options.cwd = commandSpec.cwd; + return options; +} + +function processResult(input: { + readonly stdout: readonly Buffer[]; + readonly stderr: readonly Buffer[]; + readonly exitCode: number | null; + readonly signal: string | null; + readonly startedAt: number; + readonly timedOut: boolean; + readonly timeoutMs?: number; +}): ProcessResult { + return { + stdout: Buffer.concat(input.stdout).toString('utf8'), + stderr: Buffer.concat(input.stderr).toString('utf8'), + exitCode: input.exitCode, + signal: input.signal, + durationMs: Date.now() - input.startedAt, + timedOut: input.timedOut, + ...(input.timedOut && input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs } : {}), + }; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parseProtocolLine(line: string): Record { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch (error) { + const reason = error instanceof Error ? error.message : 'Unknown JSON parse error.'; + throw new Error(`malformed ACP stdout JSON: ${reason}`); + } + if (!isRecord(parsed)) throw new Error('malformed ACP stdout JSON: expected a JSON object.'); + return parsed; +} + +function createPromptParams(sessionId: string, prompt: string): Record { + return { + sessionId, + prompt: [ + { + role: 'user', + content: [{ type: 'text', text: prompt }], + }, + ], + }; +} + +export function runAcpStdioPrompt( + provider: ProviderId, + commandSpec: CommandSpec, + prompt: string, + options: ProcessRunnerOptions = {} +): Promise { + return new Promise((resolve, reject) => { + const startedAt = Date.now(); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + const child = spawn(commandSpec.binary, [...commandSpec.args], spawnOptions(commandSpec)); + const pending = new Map(); + let stdoutBuffer = ''; + let nextId = 1; + let timedOut = false; + let protocolFailure: Error | null = null; + let sessionId: string | null = null; + let closeResolved = false; + let timeout: NodeJS.Timeout | undefined; + let timeoutKill: NodeJS.Timeout | undefined; + + function cleanupTimers(): void { + if (timeout !== undefined) clearTimeout(timeout); + if (timeoutKill !== undefined) clearTimeout(timeoutKill); + } + + function finalize(exitCode: number | null, signal: string | null): void { + if (closeResolved) return; + closeResolved = true; + cleanupTimers(); + if (protocolFailure) { + const failureMessage = protocolFailure.message; + if (!stderr.some((chunk) => chunk.toString('utf8').includes(failureMessage))) { + stderr.push(Buffer.from(`${failureMessage}\n`, 'utf8')); + } + } + const effectiveExitCode = + protocolFailure && exitCode === 0 && signal === null ? 1 : exitCode; + resolve( + processResult({ + stdout, + stderr, + exitCode: effectiveExitCode, + signal, + startedAt, + timedOut, + ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), + }) + ); + } + + function failClosed(message: string): void { + if (protocolFailure) return; + protocolFailure = new Error(`${provider} ACP stdio fail-closed: ${message}`); + for (const request of pending.values()) { + request.reject(protocolFailure); + } + pending.clear(); + if (sessionId && child.stdin && !child.stdin.destroyed) { + child.stdin.write( + `${JSON.stringify({ + jsonrpc: '2.0', + method: 'session/cancel', + params: { sessionId }, + })}\n` + ); + } + child.stdin?.end(); + child.kill('SIGTERM'); + } + + function handleProtocolMessage(message: Record): void { + const responseId = typeof message.id === 'number' ? message.id : null; + if (responseId !== null && (message.result !== undefined || message.error !== undefined)) { + const request = pending.get(responseId); + if (request) { + pending.delete(responseId); + request.resolve(message); + } + return; + } + + const method = typeof message.method === 'string' ? message.method : null; + if (!method) return; + if (method === 'session/update') return; + if (method.startsWith('_')) return; + if (method === 'session/request_permission') { + failClosed('unsupported session/request_permission callback.'); + return; + } + if (method.startsWith('fs/')) { + failClosed(`unsupported ${method} callback.`); + return; + } + if (method.startsWith('terminal/')) { + failClosed(`unsupported ${method} callback.`); + return; + } + if (method.startsWith('session/')) { + failClosed(`unsupported ${method} session-control callback.`); + } + } + + function consumeStdoutLine(line: string): void { + try { + handleProtocolMessage(parseProtocolLine(line)); + } catch (error) { + failClosed(error instanceof Error ? error.message : 'malformed ACP stdout JSON.'); + } + } + + function flushStdoutRemainder(): void { + const line = stdoutBuffer.trim(); + stdoutBuffer = ''; + if (line) consumeStdoutLine(line); + } + + function flushStdout(data: Buffer): void { + stdout.push(data); + stdoutBuffer += data.toString('utf8'); + let newline = stdoutBuffer.indexOf('\n'); + while (newline !== -1) { + const line = stdoutBuffer.slice(0, newline).trim(); + stdoutBuffer = stdoutBuffer.slice(newline + 1); + if (line) consumeStdoutLine(line); + newline = stdoutBuffer.indexOf('\n'); + } + } + + function sendRequest(method: string, params: Record): Promise { + const id = nextId; + nextId += 1; + const request: JsonRpcRequest = { + jsonrpc: '2.0', + id, + method, + params, + }; + return new Promise((requestResolve, requestReject) => { + pending.set(id, { resolve: requestResolve, reject: requestReject }); + child.stdin?.write(`${JSON.stringify(request)}\n`); + }); + } + + async function driveProtocol(): Promise { + const initialize = await sendRequest('initialize', { + protocolVersion: 1, + clientInfo: { + name: '@the-open-engine/zeroshot', + version: '1', + }, + clientCapabilities: { + fs: false, + terminal: false, + promptCapabilities: { + image: false, + }, + }, + }); + if (initialize.error !== undefined) { + child.stdin?.end(); + return; + } + + const session = await sendRequest('session/new', { + cwd: commandSpec.cwd ?? process.cwd(), + mcpServers: [], + }); + if (session.error !== undefined) { + child.stdin?.end(); + return; + } + const sessionResult = isRecord(session.result) ? session.result : {}; + const nextSessionId = typeof sessionResult.sessionId === 'string' ? sessionResult.sessionId : null; + if (!nextSessionId) { + failClosed('session/new response omitted sessionId.'); + return; + } + sessionId = nextSessionId; + + await sendRequest('session/prompt', createPromptParams(sessionId, prompt)); + child.stdin?.end(); + } + + if (options.timeoutMs !== undefined) { + timeout = setTimeout(() => { + timedOut = true; + if (sessionId && child.stdin && !child.stdin.destroyed) { + child.stdin.write( + `${JSON.stringify({ + jsonrpc: '2.0', + method: 'session/cancel', + params: { sessionId }, + })}\n` + ); + } + child.kill('SIGTERM'); + timeoutKill = setTimeout(() => { + if (!closeResolved) child.kill('SIGKILL'); + }, options.timeoutKillGraceMs ?? DEFAULT_TIMEOUT_KILL_GRACE_MS); + }, options.timeoutMs); + } + + child.stdout?.on('data', flushStdout); + child.stdout?.once('end', flushStdoutRemainder); + child.stderr?.on('data', (data: Buffer) => stderr.push(data)); + child.once('error', (error) => { + cleanupTimers(); + reject(error); + }); + child.once('close', (exitCode, signal) => { + finalize(exitCode, signal); + }); + + driveProtocol().catch((error: unknown) => { + failClosed(error instanceof Error ? error.message : 'ACP protocol failed.'); + }); + }); +} diff --git a/src/agent-cli-provider/adapters/acp.ts b/src/agent-cli-provider/adapters/acp.ts new file mode 100644 index 00000000..9cc74bd7 --- /dev/null +++ b/src/agent-cli-provider/adapters/acp.ts @@ -0,0 +1,493 @@ +import { appendJsonSchemaPrompt } from '../schema'; +import { contractError } from '../contract-errors'; +import { + getArray, + getBoolean, + getNumber, + getOptionalString, + getRecord, + getString, + isRecord, + tryParseJson, +} from '../json'; +import { + type AcpCliFeatures, + type BuildProviderCommandOptions, + type CommandSpec, + type ErrorClassification, + type LevelModelSpec, + type LevelOverrides, + type ModelCatalogEntry, + type ModelLevel, + type OutputEvent, + type ProviderAdapter, + type ProviderId, + type ProviderParseResult, + type ProviderParserState, + type ResolvedModelSpec, + type WarningMetadata, +} from '../types'; +import { + classifyBaseProviderError, + commandSpec, + createParserState, + optionFeatures, + resolveModelSpecWithConfig, + validateModelIdFromCatalog, + warning, +} from './common'; + +export interface AcpAdapterMetadata { + readonly provider: ProviderId; + readonly displayName: string; + readonly binary: string; + readonly commandArgs: readonly string[]; + readonly credentialEnvKeys: readonly string[]; + readonly modelCatalog?: Readonly>; + readonly levelMapping?: Readonly>; + readonly defaultLevel?: ModelLevel; + readonly defaultMaxLevel?: ModelLevel; + readonly defaultMinLevel?: ModelLevel; + readonly supportsPromptImages: boolean; + readonly supportsLoadSession: boolean; + readonly supportsSessionCancel: boolean; + readonly supportsSessionSetModel: boolean; + readonly supportsSessionSetMode: boolean; + readonly retryableErrorPatterns?: readonly RegExp[]; + readonly permanentErrorPatterns?: readonly RegExp[]; +} + +const DEFAULT_MODEL_CATALOG: Readonly> = {}; +const DEFAULT_LEVEL_MAPPING: Readonly> = { + level1: { rank: 1, model: null }, + level2: { rank: 2, model: null }, + level3: { rank: 3, model: null }, +}; + +function detectCliFeatures( + meta: AcpAdapterMetadata, + helpText?: string | null +): AcpCliFeatures { + const help = helpText ?? ''; + const unknown = !help; + return { + provider: meta.provider, + supportsAcpStdio: unknown ? true : /\bacp\b/.test(help), + supportsPromptImages: meta.supportsPromptImages, + supportsLoadSession: meta.supportsLoadSession, + supportsSessionCancel: meta.supportsSessionCancel, + supportsSessionSetModel: meta.supportsSessionSetModel, + supportsSessionSetMode: meta.supportsSessionSetMode, + supportsRemoteTransport: false, + supportsCustomTransport: false, + supportsPermissionRequests: false, + supportsFsTools: false, + supportsTerminalTools: false, + unknown, + }; +} + +function failClosedUnsupportedSessionControl( + provider: ProviderId, + options: BuildProviderCommandOptions +): void { + const hasResumeSessionId = options.resumeSessionId !== undefined; + if (!hasResumeSessionId && !options.continueSession) return; + const field = hasResumeSessionId ? 'options.resumeSessionId' : 'options.continueSession'; + throw contractError({ + code: 'invalid-field', + field, + exitCode: 2, + message: `${provider} ACP stdio adapter only supports fresh headless sessions; resume/continue is capability-gated off.`, + }); +} + +function failClosedUnsupportedAcpStdio( + meta: AcpAdapterMetadata, + options: BuildProviderCommandOptions +): void { + if (optionFeatures(options).supportsAcpStdio !== false) return; + throw contractError({ + code: 'invalid-field', + field: 'options.cliFeatures.supportsAcpStdio', + exitCode: 2, + message: `${meta.displayName} CLI does not advertise ACP stdio support; fail closed instead of launching the unsupported ACP lane.`, + }); +} + +function collectWarnings( + meta: AcpAdapterMetadata, + options: BuildProviderCommandOptions +): WarningMetadata[] { + const warnings: WarningMetadata[] = []; + if (options.autoApprove) { + warnings.push( + warning( + meta.provider, + 'acp-auto-approve', + `${meta.displayName} ACP stdio does not support interactive approval callbacks; failing closed if the agent requests one.` + ) + ); + } + if (options.jsonSchema) { + warnings.push( + warning( + meta.provider, + 'acp-jsonschema', + `${meta.displayName} ACP stdio has no provider-native JSON schema lane; schema instructions are appended to the prompt.` + ) + ); + } + return warnings; +} + +function buildCommand( + meta: AcpAdapterMetadata, + _context: string, + options: BuildProviderCommandOptions = {} +): CommandSpec { + failClosedUnsupportedAcpStdio(meta, options); + failClosedUnsupportedSessionControl(meta.provider, options); + return commandSpec({ + binary: meta.binary, + args: [...meta.commandArgs], + env: {}, + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + warnings: collectWarnings(meta, options), + }); +} + +export function buildAcpPrompt(context: string, options: BuildProviderCommandOptions = {}): string { + return options.jsonSchema ? appendJsonSchemaPrompt(context, options.jsonSchema) : context; +} + +function createAcpState(provider: ProviderId): ProviderParserState { + return { + ...createParserState(provider), + lastAssistantText: '', + lastAssistantThinking: '', + assistantTextByMessageId: new Map(), + assistantThinkingByMessageId: new Map(), + toolCalls: new Map(), + usage: null, + }; +} + +function textSnapshots(state: ProviderParserState): Map { + if (!state.assistantTextByMessageId) state.assistantTextByMessageId = new Map(); + return state.assistantTextByMessageId; +} + +function thinkingSnapshots(state: ProviderParserState): Map { + if (!state.assistantThinkingByMessageId) state.assistantThinkingByMessageId = new Map(); + return state.assistantThinkingByMessageId; +} + +function toolCalls(state: ProviderParserState): Map { + if (!state.toolCalls) state.toolCalls = new Map(); + return state.toolCalls; +} + +function snapshotDelta(previous: string, current: string): string | null { + if (!current || previous === current) return null; + if (current.startsWith(previous)) return current.slice(previous.length) || null; + return current; +} + +function resolveChunkSnapshot(previous: string, chunk: string): string { + if (!previous) return chunk; + if (!chunk || previous === chunk) return previous; + if (chunk.startsWith(previous)) return chunk; + if (previous.startsWith(chunk)) return chunk; + return previous + chunk; +} + +function joinTextBlocks(content: unknown, allowedTypes: readonly string[]): string | null { + if (typeof content === 'string') return content; + const blocks = Array.isArray(content) ? content : [content]; + let value = ''; + let matched = false; + for (const block of blocks) { + if (!isRecord(block)) continue; + const type = getString(block, 'type'); + if (type !== null && !allowedTypes.includes(type)) continue; + const text = getOptionalString(block, 'text') ?? getOptionalString(block, 'thinking'); + if (text === null) continue; + value += text; + matched = true; + } + return matched ? value : null; +} + +function normalizeToolInput(input: unknown): unknown { + if (typeof input !== 'string') return input ?? {}; + const parsed = tryParseJson(input); + return parsed ?? input; +} + +function normalizeToolContent(content: unknown): unknown { + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return content; + const text = content + .map((item) => (isRecord(item) ? getString(item, 'text') : null)) + .filter((value): value is string => typeof value === 'string') + .join(''); + return text || content; +} + +function updateUsage(state: ProviderParserState, update: Record): void { + const usage = getRecord(update, 'usage') ?? update; + const inputTokens = getNumber(usage, 'inputTokens'); + const outputTokens = getNumber(usage, 'outputTokens'); + const cacheReadInputTokens = getNumber(usage, 'cacheReadInputTokens'); + const cacheCreationInputTokens = getNumber(usage, 'cacheCreationInputTokens'); + if ( + inputTokens === null && + outputTokens === null && + cacheReadInputTokens === null && + cacheCreationInputTokens === null + ) { + return; + } + state.usage = { + ...(inputTokens === null ? {} : { inputTokens }), + ...(outputTokens === null ? {} : { outputTokens }), + ...(cacheReadInputTokens === null ? {} : { cacheReadInputTokens }), + ...(cacheCreationInputTokens === null ? {} : { cacheCreationInputTokens }), + }; +} + +function parseAssistantChunk( + update: Record, + state: ProviderParserState, + options: { + readonly textTypes?: readonly string[]; + readonly thinkingTypes?: readonly string[]; + } +): readonly OutputEvent[] { + const messageId = getOptionalString(update, 'messageId') ?? 'assistant'; + const content = update.content ?? getArray(update, 'content'); + const textByMessageId = textSnapshots(state); + const thinkingByMessageId = thinkingSnapshots(state); + const nextText = options.textTypes ? joinTextBlocks(content, options.textTypes) : null; + const nextThinking = options.thinkingTypes ? joinTextBlocks(content, options.thinkingTypes) : null; + + const events: OutputEvent[] = []; + if (nextText !== null) { + const previousText = textByMessageId.get(messageId) ?? ''; + const currentText = resolveChunkSnapshot(previousText, nextText); + const textDelta = snapshotDelta(previousText, currentText); + textByMessageId.set(messageId, currentText); + if (currentText) state.lastAssistantText = currentText; + if (textDelta) events.push({ type: 'text', text: textDelta }); + } + if (nextThinking !== null) { + const previousThinking = thinkingByMessageId.get(messageId) ?? ''; + const currentThinking = resolveChunkSnapshot(previousThinking, nextThinking); + const thinkingDelta = snapshotDelta(previousThinking, currentThinking); + thinkingByMessageId.set(messageId, currentThinking); + if (currentThinking) state.lastAssistantThinking = currentThinking; + if (thinkingDelta) events.push({ type: 'thinking', text: thinkingDelta }); + } + return events; +} + +function parseToolCall( + update: Record, + state: ProviderParserState +): OutputEvent | null { + const toolId = + getOptionalString(update, 'toolCallId') ?? + getOptionalString(update, 'toolId') ?? + getOptionalString(update, 'id'); + if (!toolId) return null; + const toolName = + getOptionalString(update, 'title') ?? + getOptionalString(update, 'toolName') ?? + getOptionalString(update, 'name'); + const input = normalizeToolInput(update.rawInput ?? update.input ?? update.arguments ?? {}); + state.lastToolId = toolId; + toolCalls(state).set(toolId, { name: toolName, input }); + return { + type: 'tool_call', + toolName, + toolId, + input, + }; +} + +function parseToolCallUpdate( + update: Record, + state: ProviderParserState +): OutputEvent | null { + const toolId = + getOptionalString(update, 'toolCallId') ?? + getOptionalString(update, 'toolId') ?? + getOptionalString(update, 'id') ?? + state.lastToolId; + if (!toolId) return null; + state.lastToolId = toolId; + const content = + update.rawOutput ?? update.output ?? update.partialResult ?? update.content ?? update.result; + if (content === undefined) return null; + const status = getOptionalString(update, 'status'); + const isError = + getBoolean(update, 'isError') ?? (status === 'failed' || status === 'cancelled'); + return { + type: 'tool_result', + toolId, + content: normalizeToolContent(content), + isError, + }; +} + +function resultFromStopReason( + state: ProviderParserState, + response: Record +): OutputEvent { + updateUsage(state, response); + const stopReason = + getOptionalString(response, 'stopReason') ?? getOptionalString(response, 'stop_reason'); + const explicitError = + getOptionalString(response, 'error') ?? + getOptionalString(response, 'errorMessage') ?? + getOptionalString(response, 'message'); + const success = stopReason !== 'cancelled' && stopReason !== 'refusal'; + const resultText = + typeof state.lastAssistantText === 'string' && state.lastAssistantText.length > 0 + ? state.lastAssistantText + : null; + return { + type: 'result', + success, + result: success ? resultText : null, + error: success ? null : explicitError ?? stopReason ?? 'ACP prompt failed.', + ...(state.usage?.inputTokens === undefined ? {} : { inputTokens: state.usage.inputTokens }), + ...(state.usage?.outputTokens === undefined ? {} : { outputTokens: state.usage.outputTokens }), + ...(state.usage?.cacheReadInputTokens === undefined + ? {} + : { cacheReadInputTokens: state.usage.cacheReadInputTokens }), + ...(state.usage?.cacheCreationInputTokens === undefined + ? {} + : { cacheCreationInputTokens: state.usage.cacheCreationInputTokens }), + cost: null, + modelUsage: state.usage, + }; +} + +function resultFromRpcError(error: Record): OutputEvent { + return { + type: 'result', + success: false, + result: null, + error: getString(error, 'message') ?? 'ACP request failed.', + }; +} + +function parseSessionUpdate( + params: Record, + state: ProviderParserState +): ProviderParseResult { + const update = getRecord(params, 'update') ?? params; + const sessionUpdate = + getOptionalString(update, 'sessionUpdate') ?? + getOptionalString(update, 'session_update') ?? + getOptionalString(update, 'type'); + if (!sessionUpdate || sessionUpdate.startsWith('_')) return null; + if (sessionUpdate === 'agent_message_chunk') { + return parseAssistantChunk(update, state, { + textTypes: ['text'], + thinkingTypes: ['thinking'], + }); + } + if (sessionUpdate === 'agent_thought_chunk') { + return parseAssistantChunk(update, state, { + thinkingTypes: ['thinking', 'text'], + }); + } + if (sessionUpdate === 'tool_call') return parseToolCall(update, state); + if (sessionUpdate === 'tool_call_update') return parseToolCallUpdate(update, state); + if (sessionUpdate === 'usage_update') { + updateUsage(state, update); + return null; + } + return null; +} + +function parseRpcObject(parsed: Record, state: ProviderParserState): ProviderParseResult { + const method = getOptionalString(parsed, 'method'); + if (method === 'session/update') { + const params = getRecord(parsed, 'params') ?? {}; + return parseSessionUpdate(params, state); + } + if (method?.startsWith('_')) return null; + if (parsed.error && isRecord(parsed.error)) return resultFromRpcError(parsed.error); + const result = getRecord(parsed, 'result'); + if (!result) return null; + if (getOptionalString(result, 'stopReason') || getOptionalString(result, 'stop_reason')) { + return resultFromStopReason(state, result); + } + return null; +} + +function parseEvent(line: string, state: ProviderParserState): OutputEvent | readonly OutputEvent[] | null { + const parsed = tryParseJson(line); + if (!isRecord(parsed)) return null; + return parseRpcObject(parsed, state); +} + +function resolveModelSpec( + meta: AcpAdapterMetadata, + level: ModelLevel, + overrides?: LevelOverrides +): ResolvedModelSpec { + const mapping = meta.levelMapping ?? DEFAULT_LEVEL_MAPPING; + return resolveModelSpecWithConfig({ + mapping, + defaultLevel: meta.defaultLevel ?? 'level2', + level, + overrides, + validateModelId: (modelId) => validateModelId(meta, modelId), + }); +} + +function validateModelId( + meta: AcpAdapterMetadata, + modelId: string | null | undefined +): string | null | undefined { + return validateModelIdFromCatalog( + meta.provider, + meta.modelCatalog ?? DEFAULT_MODEL_CATALOG, + modelId + ); +} + +function classifyError(meta: AcpAdapterMetadata, error: unknown): ErrorClassification { + return classifyBaseProviderError( + error, + meta.retryableErrorPatterns ?? [], + meta.permanentErrorPatterns ?? [] + ); +} + +export function createAcpAdapter(meta: AcpAdapterMetadata): ProviderAdapter { + return { + id: meta.provider, + displayName: meta.displayName, + binary: meta.binary, + adapterVersion: '1', + credentialEnvKeys: meta.credentialEnvKeys, + modelCatalog: meta.modelCatalog ?? DEFAULT_MODEL_CATALOG, + levelMapping: meta.levelMapping ?? DEFAULT_LEVEL_MAPPING, + defaultLevel: meta.defaultLevel ?? 'level2', + defaultMaxLevel: meta.defaultMaxLevel ?? 'level3', + defaultMinLevel: meta.defaultMinLevel ?? 'level1', + detectCliFeatures: (helpText) => detectCliFeatures(meta, helpText), + buildCommand: (context, options) => buildCommand(meta, context, options), + parseEvent, + createParserState: () => createAcpState(meta.provider), + resolveModelSpec: (level, overrides) => resolveModelSpec(meta, level, overrides), + validateModelId: (modelId) => validateModelId(meta, modelId), + classifyError: (error) => classifyError(meta, error), + }; +} diff --git a/src/agent-cli-provider/adapters/claude.ts b/src/agent-cli-provider/adapters/claude.ts index c1c9c030..e2d22fb5 100644 --- a/src/agent-cli-provider/adapters/claude.ts +++ b/src/agent-cli-provider/adapters/claude.ts @@ -1,7 +1,4 @@ -import { existsSync, readFileSync } from 'node:fs'; -import { homedir } from 'node:os'; -import { join } from 'node:path'; -import { getString, isRecord, stringifyJson } from '../json'; +import { stringifyJson } from '../json'; import { type BuildProviderCommandOptions, type ClaudeCliFeatures, @@ -16,6 +13,7 @@ import { type ResolvedModelSpec, type WarningMetadata, } from '../types'; +import { resolveClaudeCommand } from '../claude-command'; import { classifyBaseProviderError, commandSpec, @@ -107,34 +105,6 @@ function addSessionArgs(args: string[], options: BuildProviderCommandOptions): v } } -function readConfiguredClaudeCommand(): string { - if (process.env.ZEROSHOT_CLAUDE_COMMAND?.trim()) { - return process.env.ZEROSHOT_CLAUDE_COMMAND; - } - - const settingsPath = - process.env.ZEROSHOT_SETTINGS_FILE || join(homedir(), '.zeroshot', 'settings.json'); - if (!existsSync(settingsPath)) return 'claude'; - - try { - const settings: unknown = JSON.parse(readFileSync(settingsPath, 'utf8')); - const configured = isRecord(settings) ? getString(settings, 'claudeCommand') : null; - if (configured?.trim()) return configured; - } catch { - return 'claude'; - } - - return 'claude'; -} - -function resolveClaudeCommand(): { readonly command: string; readonly args: readonly string[] } { - const parts = readConfiguredClaudeCommand().trim().split(/\s+/); - return { - command: parts[0] ?? 'claude', - args: parts.slice(1), - }; -} - function collectWarnings(options: BuildProviderCommandOptions): WarningMetadata[] { const features = optionFeatures(options); const warnings: WarningMetadata[] = []; diff --git a/src/agent-cli-provider/adapters/copilot-parser.ts b/src/agent-cli-provider/adapters/copilot-parser.ts new file mode 100644 index 00000000..26275bb7 --- /dev/null +++ b/src/agent-cli-provider/adapters/copilot-parser.ts @@ -0,0 +1,166 @@ +// Copilot `--output-format json` emits JSONL; payload under `data`, dot-namespaced `type` (verified +// v1.0.69). Unknown types are ignored (fail-open). Mapping: +// assistant.message_delta/message → text, or thinking when phase==='commentary' (final answer is +// the result text). assistant.reasoning → thinking. tool.execution_start → tool_call. +// tool.execution_complete → tool_result. result → terminal (success = top-level exitCode === 0). +import { getBoolean, getNumber, getRecord, getString, isRecord, tryParseJson } from '../json'; +import type { OutputEvent, ProviderParseResult, ProviderParserState } from '../types'; + +const IGNORED_TYPES = new Set([ + 'user.message', + 'assistant.turn_start', + 'assistant.turn_end', + 'assistant.idle', + 'assistant.tool_call_delta', + 'tool.execution_partial_result', +]); + +function messageTextMap(state: ProviderParserState): Map { + if (!state.assistantTextByMessageId) state.assistantTextByMessageId = new Map(); + return state.assistantTextByMessageId; +} + +function messagePhaseMap(state: ProviderParserState): Map { + if (!state.messagePhaseById) state.messagePhaseById = new Map(); + return state.messagePhaseById; +} + +// `commentary` = the model narrating its plan (thinking); anything else is user-facing output. +function isCommentaryPhase(phase: string | null | undefined): boolean { + return phase === 'commentary'; +} + +function snapshotDelta(previous: string, current: string): string | null { + if (!current) return null; + if (previous === current) return null; + if (current.startsWith(previous)) return current.slice(previous.length) || null; + return current; +} + +function accrueOutputTokens(state: ProviderParserState, data: Record): void { + const tokens = getNumber(data, 'outputTokens'); + if (tokens === null) return; + const usage = state.usage ?? {}; + state.usage = { ...usage, outputTokens: (usage.outputTokens ?? 0) + tokens }; +} + +function parseMessageStart(data: Record, state: ProviderParserState): null { + const messageId = getString(data, 'messageId'); + if (messageId === null) return null; + messageTextMap(state).set(messageId, ''); + const phase = getString(data, 'phase'); + if (phase !== null) messagePhaseMap(state).set(messageId, phase); + return null; +} + +function parseMessageDelta( + data: Record, + state: ProviderParserState +): OutputEvent | null { + const delta = getString(data, 'deltaContent'); + if (!delta) return null; + const messageId = getString(data, 'messageId') ?? ''; + const map = messageTextMap(state); + map.set(messageId, (map.get(messageId) ?? '') + delta); + const commentary = isCommentaryPhase(messagePhaseMap(state).get(messageId)); + return commentary ? { type: 'thinking', text: delta } : { type: 'text', text: delta }; +} + +function parseMessage( + data: Record, + state: ProviderParserState +): OutputEvent | null { + const content = getString(data, 'content') ?? ''; + accrueOutputTokens(state, data); + const messageId = getString(data, 'messageId') ?? ''; + const map = messageTextMap(state); + const delta = snapshotDelta(map.get(messageId) ?? '', content); + map.set(messageId, content); + const commentary = isCommentaryPhase(getString(data, 'phase') ?? messagePhaseMap(state).get(messageId)); + // Only the final answer (not commentary narration) is the run's result text. + if (content && !commentary) state.lastAssistantText = content; + if (!delta) return null; + return commentary ? { type: 'thinking', text: delta } : { type: 'text', text: delta }; +} + +function parseReasoning(data: Record): OutputEvent | null { + const text = getString(data, 'content'); + return text ? { type: 'thinking', text } : null; +} + +function parseToolExecutionStart( + data: Record, + state: ProviderParserState +): OutputEvent { + const toolId = getString(data, 'toolCallId'); + state.lastToolId = toolId; + return { + type: 'tool_call', + toolName: getString(data, 'toolName'), + toolId, + input: Object.prototype.hasOwnProperty.call(data, 'arguments') ? data.arguments : {}, + }; +} + +function toolResultContent(data: Record): unknown { + const result = getRecord(data, 'result'); + if (result !== null) return result.content ?? result.detailedContent ?? ''; + return data.result ?? ''; +} + +function parseToolExecutionComplete( + data: Record, + state: ProviderParserState +): OutputEvent { + const success = getBoolean(data, 'success'); + return { + type: 'tool_result', + toolId: getString(data, 'toolCallId') ?? state.lastToolId, + content: toolResultContent(data), + isError: success === false, + }; +} + +function parseResult(event: Record, state: ProviderParserState): OutputEvent { + const exitCode = getNumber(event, 'exitCode') ?? 0; + const success = exitCode === 0; + const usage = state.usage ?? {}; + return { + type: 'result', + success, + result: success ? (state.lastAssistantText ?? null) : null, + error: success ? null : `Copilot exited with code ${exitCode}`, + inputTokens: usage.inputTokens ?? 0, + outputTokens: usage.outputTokens ?? 0, + cacheReadInputTokens: usage.cacheReadInputTokens ?? 0, + cacheCreationInputTokens: usage.cacheCreationInputTokens ?? 0, + }; +} + +export function parseCopilotEvent(line: string, state: ProviderParserState): ProviderParseResult { + const event = tryParseJson(line); + if (!isRecord(event)) return null; + + const type = getString(event, 'type'); + if (type === null || type.startsWith('session.') || IGNORED_TYPES.has(type)) return null; + + if (type === 'result') return parseResult(event, state); + + const data = getRecord(event, 'data') ?? {}; + switch (type) { + case 'assistant.message_start': + return parseMessageStart(data, state); + case 'assistant.message_delta': + return parseMessageDelta(data, state); + case 'assistant.message': + return parseMessage(data, state); + case 'assistant.reasoning': + return parseReasoning(data); + case 'tool.execution_start': + return parseToolExecutionStart(data, state); + case 'tool.execution_complete': + return parseToolExecutionComplete(data, state); + default: + return null; + } +} diff --git a/src/agent-cli-provider/adapters/copilot.ts b/src/agent-cli-provider/adapters/copilot.ts new file mode 100644 index 00000000..f9300a00 --- /dev/null +++ b/src/agent-cli-provider/adapters/copilot.ts @@ -0,0 +1,231 @@ +import { appendJsonSchemaPrompt } from '../schema'; +import { unknownToMessage } from '../json'; +import { + type BuildProviderCommandOptions, + type CommandSpec, + type CopilotCliFeatures, + type ErrorClassification, + type LevelModelSpec, + type LevelOverrides, + type ModelCatalogEntry, + type ModelLevel, + type ProviderAdapter, + type ResolvedModelSpec, + type WarningMetadata, +} from '../types'; +import { + classifyBaseProviderError, + commandSpec, + createParserState, + optionFeatures, + resolveModelSpecWithConfig, + unsupportedSessionControlWarnings, + warning, +} from './common'; +import { parseCopilotEvent } from './copilot-parser'; +import type { ProviderParserState } from '../types'; + +// Empty catalog: Copilot's models are plan-dependent, so modelLevel is a no-op (uses Copilot's +// default). Pin a model via the `model` field or COPILOT_MODEL. +const MODEL_CATALOG: Readonly> = {}; + +const LEVEL_MAPPING: Readonly> = { + level1: { rank: 1, model: null }, + level2: { rank: 2, model: null }, + level3: { rank: 3, model: null }, +}; + +function createCopilotState(): ProviderParserState { + return { + ...createParserState('copilot'), + lastAssistantText: '', + messagePhaseById: new Map(), + assistantTextByMessageId: new Map(), + usage: { outputTokens: 0 }, + }; +} + +function supports(help: string, pattern: RegExp): boolean { + return help ? pattern.test(help) : true; +} + +function detectCliFeatures(helpText?: string | null): CopilotCliFeatures { + const help = helpText ?? ''; + const unknown = !help; + return { + provider: 'copilot', + supportsJsonOutput: supports(help, /--output-format\b/), + supportsModel: supports(help, /--model\b/), + supportsAllowAll: supports(help, /--allow-all\b/), + supportsNoAskUser: supports(help, /--no-ask-user\b/), + supportsAddDir: supports(help, /--add-dir\b/), + supportsMcpConfig: supports(help, /--additional-mcp-config\b/), + unknown, + }; +} + +function addOutputArgs(args: string[], options: BuildProviderCommandOptions): void { + const features = optionFeatures(options); + if ( + (options.outputFormat === 'json' || options.outputFormat === 'stream-json') && + features.supportsJsonOutput !== false + ) { + args.push('--output-format', 'json'); + } +} + +function addModelArgs(args: string[], options: BuildProviderCommandOptions): void { + const features = optionFeatures(options); + if (options.modelSpec?.model && features.supportsModel !== false) { + args.push('--model', options.modelSpec.model); + } +} + +function addAddDirArgs(args: string[], options: BuildProviderCommandOptions): void { + const features = optionFeatures(options); + if (options.cwd && features.supportsAddDir !== false) { + args.push('--add-dir', options.cwd); + } +} + +function addAutoApproveArgs(args: string[], options: BuildProviderCommandOptions): void { + const features = optionFeatures(options); + if (!options.autoApprove) return; + if (features.supportsAllowAll !== false) args.push('--allow-all'); + if (features.supportsNoAskUser !== false) args.push('--no-ask-user'); +} + +// Copilot consumes MCP servers via the `--additional-mcp-config` CLI flag (each value augments the +// session config from ~/.copilot/mcp-config.json). Unlike Claude — which reads a `.mcp.json` file +// overlaid into its config dir — Copilot's config lives under $HOME/.copilot, unreachable by the +// worktree overlay, so the CLI flag is the only delivery path. Copilot and Claude share the same +// `{"mcpServers": {...}}` envelope, so entries are forwarded verbatim (inline JSON string or +// `@`) with no translation. +function addMcpArgs(args: string[], options: BuildProviderCommandOptions): void { + const features = optionFeatures(options); + if (!options.mcpConfig || options.mcpConfig.length === 0) return; + if (features.supportsMcpConfig === false) return; + for (const entry of options.mcpConfig) { + args.push('--additional-mcp-config', entry); + } +} + +function collectWarnings(options: BuildProviderCommandOptions): WarningMetadata[] { + const features = optionFeatures(options); + const warnings: WarningMetadata[] = unsupportedSessionControlWarnings('copilot', options); + + if (options.jsonSchema) { + warnings.push( + warning( + 'copilot', + 'copilot-jsonschema', + 'Copilot CLI does not support provider-native JSON schema; appending schema instructions to the prompt.' + ) + ); + } + if (options.autoApprove && features.supportsAllowAll === false) { + warnings.push( + warning( + 'copilot', + 'copilot-auto-approve', + 'Copilot CLI does not advertise --allow-all; continuing without the tool auto-approve flag.' + ) + ); + } + if (options.mcpConfig && options.mcpConfig.length > 0 && features.supportsMcpConfig === false) { + warnings.push( + warning( + 'copilot', + 'copilot-mcp-config', + 'Copilot CLI does not advertise --additional-mcp-config; ignoring the requested MCP server config.' + ) + ); + } + return warnings; +} + +function buildCommand(context: string, options: BuildProviderCommandOptions = {}): CommandSpec { + const finalContext = options.jsonSchema + ? appendJsonSchemaPrompt(context, options.jsonSchema) + : context; + const args: string[] = []; + + addOutputArgs(args, options); + addModelArgs(args, options); + addAddDirArgs(args, options); + addAutoApproveArgs(args, options); + addMcpArgs(args, options); + args.push('-p', finalContext); + + return commandSpec({ + binary: 'copilot', + args, + env: {}, + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + warnings: collectWarnings(options), + }); +} + +function resolveModelSpec(level: ModelLevel, overrides?: LevelOverrides): ResolvedModelSpec { + return resolveModelSpecWithConfig({ + mapping: LEVEL_MAPPING, + defaultLevel: 'level2', + level, + overrides, + validateModelId, + }); +} + +function validateModelId(modelId: string | null | undefined): string | null | undefined { + if (modelId === undefined || modelId === null) return modelId; + if (typeof modelId !== 'string') { + throw new Error(`Invalid model "${unknownToMessage(modelId)}" for provider "copilot".`); + } + return modelId; +} + +function classifyError(error: unknown): ErrorClassification { + return classifyBaseProviderError( + error, + [ + /\brate(?:[_ -]?limit| limited)\b/i, + /\b429\b/, + /\boverloaded\b/i, + /\bservice unavailable\b/i, + /\b503\b/, + /\btemporar(?:y|ily)\b/i, + /\btimeout\b/i, + ], + [ + /\bunauthorized\b/i, + /\bforbidden\b/i, + /\bbad credentials\b/i, + /\bauthentication\b/i, + /\binvalid token\b/i, + /\b(GH_TOKEN|GITHUB_TOKEN|COPILOT_GITHUB_TOKEN)\b/, + /\bquota\b/i, + /\b(cancelled|canceled|aborted|interrupted)\b/i, + /\bunknown option\b/i, + ] + ); +} + +export const copilotAdapter: ProviderAdapter = { + id: 'copilot', + displayName: 'Copilot', + binary: 'copilot', + adapterVersion: '1', + credentialEnvKeys: ['COPILOT_GITHUB_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN'], + modelCatalog: MODEL_CATALOG, + levelMapping: LEVEL_MAPPING, + defaultLevel: 'level2', + defaultMaxLevel: 'level3', + defaultMinLevel: 'level1', + detectCliFeatures, + buildCommand, + parseEvent: parseCopilotEvent, + createParserState: createCopilotState, + resolveModelSpec, + validateModelId, + classifyError, +}; diff --git a/src/agent-cli-provider/adapters/gateway.ts b/src/agent-cli-provider/adapters/gateway.ts new file mode 100644 index 00000000..20a47475 --- /dev/null +++ b/src/agent-cli-provider/adapters/gateway.ts @@ -0,0 +1,186 @@ +import path from 'node:path'; +import { + type BuildProviderCommandOptions, + type CommandSpec, + type ErrorClassification, + type GatewayCliFeatures, + type LevelModelSpec, + type LevelOverrides, + type ModelCatalogEntry, + type ModelLevel, + type OutputEvent, + type ProviderAdapter, + type ProviderParserState, + type ResolvedModelSpec, + InvalidProviderModelError, +} from '../types'; +import { classifyBaseProviderError, commandSpec, createParserState, envRedactions } from './common'; +import { resolveGatewayConfiguration, validateGatewaySettings } from '../gateway-tools'; +import { getBoolean, getString, isRecord, tryParseJson } from '../json'; + +const MODEL_CATALOG: Readonly> = {}; + +const LEVEL_MAPPING: Readonly> = { + level1: { rank: 1, model: null }, + level2: { rank: 2, model: null }, + level3: { rank: 3, model: null }, +}; + +export const gatewaySettingsDefaults: Readonly> = Object.freeze({ + baseUrl: null, + apiKey: null, + headers: null, + model: null, + toolPolicy: null, +}); + +export { validateGatewaySettings }; + +function detectCliFeatures(): GatewayCliFeatures { + return { + provider: 'gateway', + supportsBundledRunner: true, + unknown: false, + }; +} + +function buildCommand(context: string, options: BuildProviderCommandOptions = {}): CommandSpec { + const cwd = options.cwd ?? process.cwd(); + const gateway = resolveGatewayConfiguration(options.gateway, 'options.gateway', cwd); + const headerEnv = buildGatewayHeaderEnv(gateway.headers); + const request = { + context, + cwd, + gateway: { + baseUrl: gateway.baseUrl, + model: gateway.model, + toolPolicy: gateway.toolPolicy, + }, + ...(Object.keys(headerEnv.mapping).length === 0 ? {} : { gatewayHeaderEnv: headerEnv.mapping }), + }; + const env = { + ZEROSHOT_GATEWAY_REQUEST: JSON.stringify(request), + ZEROSHOT_GATEWAY_API_KEY: gateway.apiKey, + ...headerEnv.values, + }; + + return commandSpec({ + binary: process.execPath, + args: [gatewayRunnerPath()], + env, + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + redactions: envRedactions(env), + }); +} + +function gatewayRunnerPath(): string { + return path.resolve(__dirname, '..', 'gateway-runner.js'); +} + +function buildGatewayHeaderEnv(headers: Readonly>): { + readonly mapping: Readonly>; + readonly values: Readonly>; +} { + const mapping: Record = {}; + const values: Record = {}; + let index = 0; + for (const [name, value] of Object.entries(headers)) { + const envKey = `ZEROSHOT_GATEWAY_HEADER_${index}`; + index += 1; + mapping[name] = envKey; + values[envKey] = value; + } + return { mapping, values }; +} + +function parseEvent(line: string, _state: ProviderParserState): OutputEvent | null { + const parsed = tryParseJson(line); + if (!isRecord(parsed)) return null; + const type = getString(parsed, 'type'); + if (type === 'text') { + const text = getString(parsed, 'text'); + return text === null ? null : { type: 'text', text }; + } + if (type === 'tool_call') { + if (!Object.prototype.hasOwnProperty.call(parsed, 'toolName')) return null; + return { + type: 'tool_call', + toolName: getString(parsed, 'toolName'), + toolId: getString(parsed, 'toolId'), + input: parsed.input ?? null, + }; + } + if (type === 'tool_result') { + if (!Object.prototype.hasOwnProperty.call(parsed, 'content')) return null; + return { + type: 'tool_result', + toolId: getString(parsed, 'toolId'), + content: parsed.content ?? null, + isError: getBoolean(parsed, 'isError') ?? false, + }; + } + if (type === 'result') { + return { + type: 'result', + success: getBoolean(parsed, 'success') ?? false, + ...(Object.prototype.hasOwnProperty.call(parsed, 'result') ? { result: parsed.result } : {}), + ...(Object.prototype.hasOwnProperty.call(parsed, 'error') ? { error: parsed.error } : {}), + }; + } + return null; +} + +function resolveModelSpec(level: ModelLevel, overrides?: LevelOverrides): ResolvedModelSpec { + const override = overrides?.[level]; + return { + level, + model: override?.model ?? null, + reasoningEffort: override?.reasoningEffort, + }; +} + +function validateModelId(modelId: string | null | undefined): string | null | undefined { + if (modelId === undefined || modelId === null) return modelId; + const normalized = modelId.trim(); + if (normalized) return normalized; + throw new InvalidProviderModelError( + 'Invalid model "" for provider "gateway". Use a non-empty model identifier.' + ); +} + +function classifyError(error: unknown): ErrorClassification { + return classifyBaseProviderError( + error, + [/rate[_ -]?limit/i, /\b429\b/i, /\b5\d{2}\b/, /\btimed out\b/i], + [ + /invalid[_ -]?api[_ -]?key/i, + /\bunauthorized\b/i, + /\bforbidden\b/i, + /\bmodel not found\b/i, + /\bmust be a valid url\b/i, + /\btoolpolicy\b/i, + /\bnon-empty model identifier\b/i, + /\bgateway\.(?:baseUrl|apiKey|model|toolPolicy)\b/i, + ] + ); +} + +export const gatewayAdapter: ProviderAdapter = { + id: 'gateway', + displayName: 'Gateway', + binary: process.execPath, + adapterVersion: '1', + credentialEnvKeys: ['ZEROSHOT_GATEWAY_API_KEY'], + modelCatalog: MODEL_CATALOG, + levelMapping: LEVEL_MAPPING, + defaultLevel: 'level2', + defaultMaxLevel: 'level3', + defaultMinLevel: 'level1', + detectCliFeatures, + buildCommand, + parseEvent, + createParserState: () => createParserState('gateway'), + resolveModelSpec, + validateModelId, + classifyError, +}; diff --git a/src/agent-cli-provider/adapters/gemini.ts b/src/agent-cli-provider/adapters/gemini.ts index 5343dfd0..98953235 100644 --- a/src/agent-cli-provider/adapters/gemini.ts +++ b/src/agent-cli-provider/adapters/gemini.ts @@ -103,7 +103,7 @@ function buildCommand(context: string, options: BuildProviderCommandOptions = {} return commandSpec({ binary: 'gemini', args, - env: {}, + env: { GEMINI_CLI_TRUST_WORKSPACE: 'true' }, ...(options.cwd === undefined ? {} : { cwd: options.cwd }), warnings: collectGeminiWarnings(options), }); @@ -218,7 +218,14 @@ function classifyError(error: unknown): ErrorClassification { /No capacity available/i, /quota.?exceeded/i, ], - [/\bINVALID_ARGUMENT\b/i, /\bPERMISSION_DENIED\b/i, /\bNOT_FOUND\b/i] + [ + /\bINVALID_ARGUMENT\b/i, + /\bPERMISSION_DENIED\b/i, + /\bNOT_FOUND\b/i, + /\bIneligibleTierError\b/i, + /\bUNSUPPORTED_CLIENT\b/i, + /\bno longer supported\b/i, + ] ); } diff --git a/src/agent-cli-provider/adapters/index.ts b/src/agent-cli-provider/adapters/index.ts index d0c88334..13fa082f 100644 --- a/src/agent-cli-provider/adapters/index.ts +++ b/src/agent-cli-provider/adapters/index.ts @@ -1,4 +1,5 @@ import { stripTimestampPrefix } from '../log-prefix'; +import { getProviderRegistryEntry, normalizeProviderName, providerIds } from '../provider-registry'; import type { BuildProviderCommandOptions, CommandSpec, @@ -11,7 +12,6 @@ import type { ProviderId, ResolvedModelSpec, } from '../types'; -import { claudeAdapter } from './claude'; export { NO_MESSAGES_RETURNED, STREAMING_MODE_ERROR, @@ -22,18 +22,6 @@ export { type StreamingModeError, type StructuredOutputRecovery, } from './claude-recovery'; -import { codexAdapter } from './codex'; -import { geminiAdapter } from './gemini'; -import { opencodeAdapter } from './opencode'; - -const ADAPTERS: Readonly> = { - claude: claudeAdapter, - codex: codexAdapter, - gemini: geminiAdapter, - opencode: opencodeAdapter, -}; - -const PROVIDER_IDS: readonly ProviderId[] = ['claude', 'codex', 'gemini', 'opencode']; function isOutputEventArray( event: OutputEvent | readonly OutputEvent[] @@ -41,41 +29,22 @@ function isOutputEventArray( return Array.isArray(event); } -function normalizeProviderName(name: string): ProviderId | string { - const normalized = name.toLowerCase(); - switch (normalized) { - case 'anthropic': - case 'claude': - return 'claude'; - case 'openai': - case 'codex': - return 'codex'; - case 'google': - case 'gemini': - return 'gemini'; - case 'opencode': - return 'opencode'; - default: - return name; - } -} - function adapterForProviderId(provider: ProviderId): ProviderAdapter { - return ADAPTERS[provider]; + return getProviderRegistryEntry(provider).adapter; } function isProviderId(name: string): name is ProviderId { - return name === 'claude' || name === 'codex' || name === 'gemini' || name === 'opencode'; + return (providerIds as readonly string[]).includes(name); } export function getProviderAdapter(name: KnownProviderName | string): ProviderAdapter { const normalized = normalizeProviderName(name || ''); if (isProviderId(normalized)) return adapterForProviderId(normalized); - throw new Error(`Unknown provider: ${name}. Valid: ${PROVIDER_IDS.join(', ')}`); + throw new Error(`Unknown provider: ${name}. Valid: ${providerIds.join(', ')}`); } export function listProviderAdapters(): readonly ProviderId[] { - return PROVIDER_IDS; + return providerIds; } export function buildProviderCommand( diff --git a/src/agent-cli-provider/adapters/opencode.ts b/src/agent-cli-provider/adapters/opencode.ts index 804bafd0..db163eaa 100644 --- a/src/agent-cli-provider/adapters/opencode.ts +++ b/src/agent-cli-provider/adapters/opencode.ts @@ -84,6 +84,7 @@ function detectCliFeatures(helpText?: string | null): OpencodeCliFeatures { supportsJson: unknown ? true : /--format\b/.test(help), supportsModel: unknown ? true : /--model\b/.test(help), supportsVariant: unknown ? true : /--variant\b/.test(help), + supportsDir: unknown ? false : /--dir\b/.test(help), supportsCwd: unknown ? false : /--cwd\b/.test(help), supportsAutoApprove: false, unknown, @@ -107,7 +108,9 @@ function addOpencodeOptionalArgs(args: string[], options: BuildProviderCommandOp args.push('--variant', options.modelSpec.reasoningEffort); } - if (options.cwd && features.supportsCwd) { + if (options.cwd && features.supportsDir) { + args.push('--dir', options.cwd); + } else if (options.cwd && features.supportsCwd) { args.push('--cwd', options.cwd); } } diff --git a/src/agent-cli-provider/adapters/pi.ts b/src/agent-cli-provider/adapters/pi.ts new file mode 100644 index 00000000..45dfc092 --- /dev/null +++ b/src/agent-cli-provider/adapters/pi.ts @@ -0,0 +1,410 @@ +import { appendJsonSchemaPrompt } from '../schema'; +import { contractError } from '../contract-errors'; +import { + getArray, + getBoolean, + getNumber, + getOptionalString, + getRecord, + getString, + isRecord, + tryParseJson, + unknownToMessage, +} from '../json'; +import { + type BuildProviderCommandOptions, + type CommandSpec, + type ErrorClassification, + type LevelModelSpec, + type LevelOverrides, + type ModelCatalogEntry, + type ModelLevel, + type OutputEvent, + type PiCliFeatures, + type ProviderAdapter, + type ProviderParserState, + type ResolvedModelSpec, + type WarningMetadata, +} from '../types'; +import { + classifyBaseProviderError, + commandSpec, + createParserState, + optionFeatures, + resolveModelSpecWithConfig, + warning, +} from './common'; + +const MODEL_CATALOG: Readonly> = {}; + +const LEVEL_MAPPING: Readonly> = { + level1: { rank: 1, model: null }, + level2: { rank: 2, model: null }, + level3: { rank: 3, model: null }, +}; + +const IGNORED_EVENT_TYPES = new Set([ + 'session', + 'agent_start', + 'agent_end', + 'turn_start', + 'queue_update', + 'compaction_start', + 'compaction_end', + 'auto_retry_start', + 'auto_retry_end', +]); + +function detectCliFeatures(helpText?: string | null): PiCliFeatures { + const help = helpText ?? ''; + const unknown = !help; + return { + provider: 'pi', + supportsJsonMode: unknown ? true : /--mode\b/.test(help) && /\bjson\b/.test(help), + supportsModel: unknown ? true : /--model\b/.test(help), + supportsNoSession: unknown ? true : /--no-session\b/.test(help), + supportsNoExtensions: unknown ? true : /--no-extensions\b/.test(help), + supportsNoSkills: unknown ? true : /--no-skills\b/.test(help), + supportsNoPromptTemplates: unknown ? true : /--no-prompt-templates\b/.test(help), + supportsNoContextFiles: unknown ? true : /--no-context-files\b/.test(help), + supportsNoApprove: unknown ? true : /--no-approve\b/.test(help), + unknown, + }; +} + +function addRequiredArgs(args: string[], options: BuildProviderCommandOptions): void { + const features = optionFeatures(options); + if (features.supportsJsonMode !== false) args.push('--mode', 'json'); + if (features.supportsNoSession !== false) args.push('--no-session'); + if (features.supportsNoExtensions !== false) args.push('--no-extensions'); + if (features.supportsNoSkills !== false) args.push('--no-skills'); + if (features.supportsNoPromptTemplates !== false) args.push('--no-prompt-templates'); + if (features.supportsNoContextFiles !== false) args.push('--no-context-files'); + if (features.supportsNoApprove !== false) args.push('--no-approve'); +} + +function addOptionalArgs(args: string[], options: BuildProviderCommandOptions): void { + const features = optionFeatures(options); + if (options.modelSpec?.model && features.supportsModel !== false) { + args.push('--model', options.modelSpec.model); + } +} + +function failClosedUnsupportedSessionControl(options: BuildProviderCommandOptions): void { + const hasResumeSessionId = options.resumeSessionId !== undefined; + if (!hasResumeSessionId && !options.continueSession) return; + const field = hasResumeSessionId ? 'options.resumeSessionId' : 'options.continueSession'; + throw contractError({ + code: 'invalid-field', + field, + exitCode: 2, + message: + 'Pi CLI does not support resume/continue session control; fail closed and start a fresh run instead.', + }); +} + +function collectWarnings(options: BuildProviderCommandOptions): WarningMetadata[] { + const features = optionFeatures(options); + const warnings: WarningMetadata[] = []; + + if (options.jsonSchema) { + warnings.push( + warning( + 'pi', + 'pi-jsonschema', + 'Pi CLI does not support provider-native JSON schema; appending schema instructions to the prompt.' + ) + ); + } + if (features.supportsJsonMode === false) { + warnings.push( + warning('pi', 'pi-json-mode', 'Pi CLI does not advertise --mode json; continuing anyway.') + ); + } + return warnings; +} + +function buildCommand(context: string, options: BuildProviderCommandOptions = {}): CommandSpec { + failClosedUnsupportedSessionControl(options); + const finalContext = options.jsonSchema + ? appendJsonSchemaPrompt(context, options.jsonSchema) + : context; + const args: string[] = []; + + addRequiredArgs(args, options); + addOptionalArgs(args, options); + args.push(finalContext); + + return commandSpec({ + binary: 'pi', + args, + env: {}, + ...(options.cwd === undefined ? {} : { cwd: options.cwd }), + warnings: collectWarnings(options), + }); +} + +function createPiState(): ProviderParserState { + return { + ...createParserState('pi'), + lastAssistantText: '', + lastAssistantThinking: '', + }; +} + +function assistantSnapshot(message: Record): { text: string; thinking: string } { + if (getString(message, 'role') !== 'assistant') return { text: '', thinking: '' }; + + let text = ''; + let thinking = ''; + for (const item of getArray(message, 'content')) { + if (!isRecord(item)) continue; + const type = getString(item, 'type'); + if (type === 'text') { + text += getString(item, 'text') ?? ''; + } else if (type === 'thinking') { + thinking += getString(item, 'thinking') ?? ''; + } + } + + return { text, thinking }; +} + +function snapshotDelta(previous: string, current: string): string | null { + if (!current) return null; + if (previous === current) return null; + if (current.startsWith(previous)) return current.slice(previous.length) || null; + return current; +} + +function emitAssistantSnapshot( + message: Record, + state: ProviderParserState +): readonly OutputEvent[] { + const snapshot = assistantSnapshot(message); + const events: OutputEvent[] = []; + + const textDelta = snapshotDelta(state.lastAssistantText ?? '', snapshot.text); + if (textDelta) events.push({ type: 'text', text: textDelta }); + state.lastAssistantText = snapshot.text; + + const thinkingDelta = snapshotDelta(state.lastAssistantThinking ?? '', snapshot.thinking); + if (thinkingDelta) events.push({ type: 'thinking', text: thinkingDelta }); + state.lastAssistantThinking = snapshot.thinking; + + return events; +} + +function parseAssistantEvent( + event: Record, + state: ProviderParserState +): OutputEvent | null { + const type = getString(event, 'type'); + if (type === 'text_delta') { + const delta = getString(event, 'delta'); + if (!delta) return null; + state.lastAssistantText += delta; + return { type: 'text', text: delta }; + } + if (type === 'thinking_delta') { + const delta = getString(event, 'delta'); + if (!delta) return null; + state.lastAssistantThinking += delta; + return { type: 'thinking', text: delta }; + } + return null; +} + +function parseToolExecutionStart( + event: Record, + state: ProviderParserState +): OutputEvent { + const toolId = getOptionalString(event, 'toolCallId'); + state.lastToolId = toolId; + return { + type: 'tool_call', + toolName: getOptionalString(event, 'toolName'), + toolId, + input: event.args ?? {}, + }; +} + +function parseToolExecutionUpdate( + event: Record, + state: ProviderParserState +): OutputEvent | null { + const toolId = getOptionalString(event, 'toolCallId') ?? state.lastToolId; + if (toolId !== undefined) state.lastToolId = toolId; + if (!Object.prototype.hasOwnProperty.call(event, 'partialResult')) return null; + return { + type: 'tool_result', + toolId, + content: event.partialResult, + isError: false, + }; +} + +function parseToolExecutionEnd( + event: Record, + state: ProviderParserState +): OutputEvent { + const toolId = getOptionalString(event, 'toolCallId') ?? state.lastToolId; + return { + type: 'tool_result', + toolId, + content: event.result ?? '', + isError: getBoolean(event, 'isError') ?? false, + }; +} + +function parseTurnEnd( + event: Record, + state: ProviderParserState +): readonly OutputEvent[] { + const message = getRecord(event, 'message'); + const events: OutputEvent[] = []; + if (message !== null) { + events.push(...emitAssistantSnapshot(message, state)); + } + + const usage = message ? getRecord(message, 'usage') ?? {} : {}; + const stopReason = message ? getString(message, 'stopReason') : null; + const errorMessage = message ? getString(message, 'errorMessage') : null; + const snapshot = message ? assistantSnapshot(message) : { text: '', thinking: '' }; + const success = stopReason !== 'error' && stopReason !== 'aborted' && !errorMessage; + events.push({ + type: 'result', + success, + result: success ? snapshot.text || null : null, + error: success ? null : (errorMessage ?? stopReason ?? 'Pi turn failed'), + inputTokens: getNumber(usage, 'input') ?? 0, + outputTokens: getNumber(usage, 'output') ?? 0, + cacheReadInputTokens: getNumber(usage, 'cacheRead') ?? 0, + cacheCreationInputTokens: getNumber(usage, 'cacheWrite') ?? 0, + cost: getRecord(usage, 'cost') ?? null, + modelUsage: usage, + }); + return events; +} + +function parseMessageEvent( + type: string, + event: Record, + state: ProviderParserState +): readonly OutputEvent[] | OutputEvent | null | undefined { + if (type === 'message_start') { + const message = getRecord(event, 'message'); + if (message !== null && getString(message, 'role') === 'assistant') { + state.lastAssistantText = ''; + state.lastAssistantThinking = ''; + } + return null; + } + + if (type === 'message_update') { + const assistantMessageEvent = getRecord(event, 'assistantMessageEvent'); + if (assistantMessageEvent !== null) { + const assistantEvent = parseAssistantEvent(assistantMessageEvent, state); + if (assistantEvent !== null) return assistantEvent; + } + const message = getRecord(event, 'message'); + return message === null ? null : emitAssistantSnapshot(message, state); + } + + if (type !== 'message_end') return undefined; + const message = getRecord(event, 'message'); + return message === null ? null : emitAssistantSnapshot(message, state); +} + +function parseToolEvent( + type: string, + event: Record, + state: ProviderParserState +): OutputEvent | null | undefined { + if (type === 'tool_execution_start') return parseToolExecutionStart(event, state); + if (type === 'tool_execution_update') return parseToolExecutionUpdate(event, state); + if (type === 'tool_execution_end') return parseToolExecutionEnd(event, state); + return undefined; +} + +function parseEvent( + line: string, + state: ProviderParserState +): readonly OutputEvent[] | OutputEvent | null { + const parsed = tryParseJson(line); + if (!isRecord(parsed)) return null; + + const type = getString(parsed, 'type'); + if (type === null || IGNORED_EVENT_TYPES.has(type)) return null; + + const messageEvent = parseMessageEvent(type, parsed, state); + if (messageEvent !== undefined) return messageEvent; + + const toolEvent = parseToolEvent(type, parsed, state); + if (toolEvent !== undefined) return toolEvent; + + if (type === 'turn_end') return parseTurnEnd(parsed, state); + return null; +} + +function resolveModelSpec(level: ModelLevel, overrides?: LevelOverrides): ResolvedModelSpec { + return resolveModelSpecWithConfig({ + mapping: LEVEL_MAPPING, + defaultLevel: 'level2', + level, + overrides, + validateModelId, + }); +} + +function validateModelId(modelId: string | null | undefined): string | null | undefined { + if (modelId === undefined || modelId === null) return modelId; + if (typeof modelId !== 'string') { + throw new Error(`Invalid model "${unknownToMessage(modelId)}" for provider "pi".`); + } + return modelId; +} + +function classifyError(error: unknown): ErrorClassification { + return classifyBaseProviderError( + error, + [ + /\brate(?:[_ -]?limit| limited)\b/i, + /\bquota\b/i, + /\bresource[_ -]?exhausted\b/i, + /\btemporar(?:y|ily)\b/i, + /\boverloaded\b/i, + /\bservice unavailable\b/i, + ], + [ + /\b(cancelled|canceled|aborted|interrupted)\b/i, + /\brun\s*\/login\b/i, + /\bmissing api key\b/i, + /\bno valid authentication\b/i, + /\bunknown option\b/i, + /\bfailed to load\b/i, + /\bcannot find module\b/i, + /\bno such file or directory\b/i, + ] + ); +} + +export const piAdapter: ProviderAdapter = { + id: 'pi', + displayName: 'Pi', + binary: 'pi', + adapterVersion: '1', + credentialEnvKeys: [], + modelCatalog: MODEL_CATALOG, + levelMapping: LEVEL_MAPPING, + defaultLevel: 'level2', + defaultMaxLevel: 'level3', + defaultMinLevel: 'level1', + detectCliFeatures, + buildCommand, + parseEvent, + createParserState: createPiState, + resolveModelSpec, + validateModelId, + classifyError, +}; diff --git a/src/agent-cli-provider/claude-command.ts b/src/agent-cli-provider/claude-command.ts new file mode 100644 index 00000000..36455746 --- /dev/null +++ b/src/agent-cli-provider/claude-command.ts @@ -0,0 +1,47 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +function settingsFilePath(): string { + return process.env.ZEROSHOT_SETTINGS_FILE || join(homedir(), '.zeroshot', 'settings.json'); +} + +export function readConfiguredClaudeCommand(): string { + if (process.env.ZEROSHOT_CLAUDE_COMMAND?.trim()) { + return process.env.ZEROSHOT_CLAUDE_COMMAND; + } + + const settingsPath = settingsFilePath(); + if (!existsSync(settingsPath)) return 'claude'; + + try { + const settings: unknown = JSON.parse(readFileSync(settingsPath, 'utf8')); + if ( + settings !== null && + typeof settings === 'object' && + 'claudeCommand' in settings && + typeof settings.claudeCommand === 'string' && + settings.claudeCommand.trim() + ) { + return settings.claudeCommand; + } + } catch { + return 'claude'; + } + + return 'claude'; +} + +export function resolveClaudeCommand(): { + readonly command: string; + readonly args: readonly string[]; +} { + const parts = readConfiguredClaudeCommand() + .trim() + .split(/\s+/) + .filter((part) => part.length > 0); + return { + command: parts[0] ?? 'claude', + args: parts.slice(1), + }; +} diff --git a/src/agent-cli-provider/contract-actions.ts b/src/agent-cli-provider/contract-actions.ts index 51b0cdfd..ce577472 100644 --- a/src/agent-cli-provider/contract-actions.ts +++ b/src/agent-cli-provider/contract-actions.ts @@ -15,8 +15,9 @@ import { schemaMode, type RequestData, } from './contract-support'; -import { detectRuntimeProviderCliFeatures } from './single-agent-runtime'; -import { getNumber, getRecord, getString, isRecord, unknownToMessage } from './json'; +import { extractErrorStatus } from './errors'; +import { probeRuntimeProviderCli } from './single-agent-runtime'; +import { getString, isRecord, unknownToMessage } from './json'; import type { ErrorClassification } from './types'; import type { ProcessRunner } from './process-runner'; @@ -42,8 +43,11 @@ function runBuildCommand(request: RequestData): ContractEnvelope { function runProbe(request: RequestData): ContractEnvelope { const adapter = adapterForProvider(request.provider); const helpText = typeof request.raw.helpText === 'string' ? request.raw.helpText : null; + const runtimeProbe = helpText === null ? probeRuntimeProviderCli(adapter.id) : null; const capabilities = - helpText === null ? detectRuntimeProviderCliFeatures(adapter.id) : adapter.detectCliFeatures(helpText); + runtimeProbe === null + ? adapter.detectCliFeatures(helpText) + : runtimeProbe.capabilities; return successEnvelope({ command: request.command ?? 'probe', adapter, @@ -56,6 +60,9 @@ function runProbe(request: RequestData): ContractEnvelope { }, contractVersion: providerExecutableSchemaVersion, adapterVersion: adapter.adapterVersion, + available: runtimeProbe?.available ?? true, + helpText: runtimeProbe?.helpText ?? helpText, + versionText: runtimeProbe?.versionText ?? null, capabilities, credentials: adapter.credentialEnvKeys.map((key) => ({ key, @@ -65,24 +72,14 @@ function runProbe(request: RequestData): ContractEnvelope { }); } -function statusFromError(error: unknown): number | null { - if (!isRecord(error)) return null; - const directStatus = getNumber(error, 'status') ?? getNumber(error, 'statusCode'); - if (directStatus !== null) return directStatus; - - const response = getRecord(error, 'response'); - if (response === null) return null; - return getNumber(response, 'status') ?? getNumber(response, 'statusCode'); -} - function categoryForClassification(classification: ErrorClassification, error: unknown): string { - const status = statusFromError(error); + const status = extractErrorStatus(error); if (status === 401 || status === 403) return 'auth'; if (status === 429) return 'rate-limit'; const message = unknownToMessage(error); if (/auth|api[_ -]?key|unauthorized|forbidden|permission/i.test(message)) return 'auth'; if (/rate|429|quota|resource_exhausted/i.test(message)) return 'rate-limit'; - if (/schema|json|parse|format/i.test(message)) return 'schema'; + if (/schema|json|parse|format|malformed/i.test(message)) return 'schema'; return classification.retryable ? 'retryable' : 'permanent'; } @@ -90,10 +87,10 @@ function errorEvidence(error: unknown): ContractEvidence { const evidence: Record = { message: unknownToMessage(error), }; + const status = extractErrorStatus(error); + if (status !== null) evidence.status = status; if (isRecord(error)) { - const status = statusFromError(error); const code = getString(error, 'code'); - if (status !== null) evidence.status = status; if (code !== null) evidence.code = code; } return evidence; diff --git a/src/agent-cli-provider/contract-invoke.ts b/src/agent-cli-provider/contract-invoke.ts index 273a0926..010b213e 100644 --- a/src/agent-cli-provider/contract-invoke.ts +++ b/src/agent-cli-provider/contract-invoke.ts @@ -1,10 +1,13 @@ import { unlink } from 'node:fs/promises'; +import { buildAcpPrompt } from './adapters/acp'; +import { runAcpStdioPrompt } from './acp-stdio-runner'; import { commandRedactions } from './contract-env'; import { successEnvelope, type ContractEnvelope } from './contract-envelope'; import { buildCommandSpec, optionalNumber, schemaMode, type RequestData } from './contract-support'; import { providerFailureClassification } from './invoke-evidence'; import { parseOutputEvents } from './contract-parse'; import { unknownToMessage } from './json'; +import { getProviderRegistryEntry } from './provider-registry'; import type { CommandSpec } from './types'; import type { ProcessResult, ProcessRunner, ProcessRunnerOptions } from './process-runner'; @@ -51,10 +54,16 @@ export async function runInvoke( request: RequestData, runner: ProcessRunner ): Promise { - const { adapter, commandSpec, options } = buildCommandSpec(request); + const { adapter, commandSpec, context, options } = buildCommandSpec(request); const timeoutMs = optionalNumber(request.raw, 'timeoutMs'); const runnerOptions = timeoutMs === undefined ? {} : { timeoutMs }; - const { result, cleanup } = await runAndCleanup(commandSpec, runner, runnerOptions); + const invokeSpec = getProviderRegistryEntry(adapter.id).invoke; + const invokeRunner = + invokeSpec.lane === 'acp-stdio' + ? (spec: CommandSpec, invokeOptions?: ProcessRunnerOptions): Promise => + runAcpStdioPrompt(adapter.id, spec, buildAcpPrompt(context, options), invokeOptions) + : runner; + const { result, cleanup } = await runAndCleanup(commandSpec, invokeRunner, runnerOptions); const parsed = parseOutputEvents(adapter, { chunk: [result.stdout, result.stderr].join('\n'), sources: [ @@ -62,7 +71,7 @@ export async function runInvoke( { name: 'stderr', value: result.stderr }, ], }); - const classification = providerFailureClassification(adapter, result); + const classification = providerFailureClassification(adapter, result, parsed.events); return successEnvelope({ command: request.command ?? 'invoke', adapter, diff --git a/src/agent-cli-provider/contract-options.ts b/src/agent-cli-provider/contract-options.ts index 8becee43..b5d3f7f7 100644 --- a/src/agent-cli-provider/contract-options.ts +++ b/src/agent-cli-provider/contract-options.ts @@ -1,6 +1,7 @@ import { isRecord } from './json'; import { invalidField, contractError } from './contract-errors'; import { stringRecord } from './contract-env'; +import { normalizeGatewayBuildOptions } from './gateway-tools'; import type { BuildProviderCommandOptions, CliFeatureOverrides, @@ -23,14 +24,46 @@ const CLI_FEATURE_FIELDS = [ 'supportsModel', 'supportsJson', 'supportsOutputSchema', + 'supportsDir', 'supportsCwd', 'supportsConfigOverride', 'supportsSkipGitRepoCheck', 'supportsVariant', + 'supportsJsonMode', + 'supportsNoSession', + 'supportsNoExtensions', + 'supportsNoSkills', + 'supportsNoPromptTemplates', + 'supportsNoContextFiles', + 'supportsNoApprove', + 'supportsJsonOutput', + 'supportsAllowAll', + 'supportsNoAskUser', + 'supportsAddDir', + 'supportsMcpConfig', + 'supportsBundledRunner', + 'supportsAcpStdio', + 'supportsPromptImages', + 'supportsLoadSession', + 'supportsSessionCancel', + 'supportsSessionSetModel', + 'supportsSessionSetMode', + 'supportsRemoteTransport', + 'supportsCustomTransport', + 'supportsPermissionRequests', + 'supportsFsTools', + 'supportsTerminalTools', 'unknown', ] as const; type CliFeatureField = (typeof CLI_FEATURE_FIELDS)[number]; +const FALSE_ONLY_CLI_FEATURE_FIELDS = new Set([ + 'supportsRemoteTransport', + 'supportsCustomTransport', + 'supportsPermissionRequests', + 'supportsFsTools', + 'supportsTerminalTools', +]); export function requestOptions(value: unknown): BuildProviderCommandOptions { if (value === undefined) return {}; @@ -77,6 +110,25 @@ function optionalBooleanValue(value: unknown, field: string): boolean | undefine invalidField(field, `${field} must be a boolean.`); } +function optionalMcpConfig(value: unknown): readonly string[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + invalidField('options.mcpConfig', 'options.mcpConfig must be an array of strings.'); + } + return value.map((item, index) => { + if (typeof item !== 'string') { + invalidField(`options.mcpConfig[${index}]`, `options.mcpConfig[${index}] must be a string.`); + } + if (item.trim().length === 0) { + invalidField( + `options.mcpConfig[${index}]`, + `options.mcpConfig[${index}] must be a non-empty string.` + ); + } + return item; + }); +} + function optionalEnumValue( value: unknown, field: string, @@ -119,12 +171,19 @@ function optionalCliFeatures(value: unknown): CliFeatureOverrides | undefined { invalidField('options.cliFeatures', 'options.cliFeatures must be an object.'); } - const result: Partial> = {}; + const result: Record = {}; for (const field of CLI_FEATURE_FIELDS) { const item = optionalBooleanValue(value[field], `options.cliFeatures.${field}`); - if (item !== undefined) result[field] = item; + if (item === undefined) continue; + if (item && FALSE_ONLY_CLI_FEATURE_FIELDS.has(field)) { + invalidField( + `options.cliFeatures.${field}`, + `options.cliFeatures.${field} must be false when provided.` + ); + } + result[field] = item; } - return result; + return result as CliFeatureOverrides; } function normalizeBuildOptions(value: Record): BuildProviderCommandOptions { @@ -149,11 +208,21 @@ function normalizeBuildOptions(value: Record): BuildProviderCom optionalBooleanValue(value.continueSession, 'options.continueSession') ); addDefined(result, 'cliFeatures', optionalCliFeatures(value.cliFeatures)); + addDefined(result, 'mcpConfig', optionalMcpConfig(value.mcpConfig)); addDefined( result, 'strictSchema', optionalBooleanValue(value.strictSchema, 'options.strictSchema') ); + addDefined( + result, + 'gateway', + normalizeGatewayBuildOptions( + value.gateway, + 'options.gateway', + optionalStringValue(value.cwd, 'options.cwd') ?? process.cwd() + ) + ); if (Object.prototype.hasOwnProperty.call(value, 'authEnv')) { result.authEnv = stringRecord(value.authEnv, 'options.authEnv'); } diff --git a/src/agent-cli-provider/contract-support.ts b/src/agent-cli-provider/contract-support.ts index d2f19a16..14972ee6 100644 --- a/src/agent-cli-provider/contract-support.ts +++ b/src/agent-cli-provider/contract-support.ts @@ -40,10 +40,10 @@ export function adapterForProvider(provider: string | null): ProviderAdapter { } try { return getProviderAdapter(provider); - } catch { + } catch (error) { throw contractError({ code: 'unknown-provider', - message: `Unknown provider: ${provider}.`, + message: error instanceof Error ? error.message : `Unknown provider: ${provider}.`, exitCode: 4, field: 'provider', }); @@ -54,7 +54,17 @@ function mergeCommandSpec( commandSpec: CommandSpec, env: Readonly> ): CommandSpec { - const mergedEnv = { ...commandSpec.env, ...env }; + const overriddenKey = findReservedCommandEnvKey(commandSpec.env, env); + if (overriddenKey !== null) { + throw contractError({ + code: 'forbidden-field', + message: `env.${overriddenKey.requestKey} is not accepted by provider executable requests; provider adapters own ${overriddenKey.commandKey} and other runner control/auth environment variables.`, + exitCode: 2, + field: `env.${overriddenKey.requestKey}`, + }); + } + + const mergedEnv = { ...env, ...commandSpec.env }; return { ...commandSpec, env: mergedEnv, @@ -62,6 +72,21 @@ function mergeCommandSpec( }; } +function findReservedCommandEnvKey( + commandEnv: Readonly>, + requestedEnv: Readonly> +): { readonly requestKey: string; readonly commandKey: string } | null { + const commandEnvKeys = new Map(); + for (const key of Object.keys(commandEnv)) { + commandEnvKeys.set(key.toLowerCase(), key); + } + for (const key of Object.keys(requestedEnv)) { + const commandKey = commandEnvKeys.get(key.toLowerCase()); + if (commandKey !== undefined) return { requestKey: key, commandKey }; + } + return null; +} + function buildOptions(request: RequestData): BuildProviderCommandOptions { const options = requestOptions(request.raw.options); const cwd = optionalString(request.raw, 'cwd'); @@ -73,6 +98,7 @@ export function buildCommandSpec(request: RequestData): { readonly adapter: ProviderAdapter; readonly commandSpec: CommandSpec; readonly options: BuildProviderCommandOptions; + readonly context: string; } { const adapter = adapterForProvider(request.provider); const context = requiredString(request.raw, 'context'); @@ -84,6 +110,7 @@ export function buildCommandSpec(request: RequestData): { }); return { adapter: prepared.adapter, + context, options: prepared.options, commandSpec: mergeCommandSpec(prepared.commandSpec, request.env), }; diff --git a/src/agent-cli-provider/errors.ts b/src/agent-cli-provider/errors.ts index f391f1d3..36726e6a 100644 --- a/src/agent-cli-provider/errors.ts +++ b/src/agent-cli-provider/errors.ts @@ -34,14 +34,24 @@ const BASE_PERMANENT_PATTERNS: readonly RegExp[] = [ /insufficient quota/i, ]; -function getStatus(error: unknown): number | null { +function statusFromMessage(message: string): number | null { + const match = /\b(?:status(?: code)?|http)\s+(?\d{3})\b/i.exec(message); + if (!match?.groups?.status) return null; + const status = Number.parseInt(match.groups.status, 10); + return Number.isInteger(status) ? status : null; +} + +export function extractErrorStatus(error: unknown): number | null { if (!isRecord(error)) return null; const directStatus = getNumber(error, 'status') ?? getNumber(error, 'statusCode'); if (directStatus !== null) return directStatus; const response = getRecord(error, 'response'); - if (response === null) return null; - return getNumber(response, 'status') ?? getNumber(response, 'statusCode'); + const responseStatus = + response === null ? null : getNumber(response, 'status') ?? getNumber(response, 'statusCode'); + if (responseStatus !== null) return responseStatus; + + return statusFromMessage(unknownToMessage(error)); } function getCode(error: unknown): string | null { @@ -69,7 +79,7 @@ export function classifyErrorWithPatterns( retryablePatterns: readonly RegExp[], permanentPatterns: readonly RegExp[] ): ErrorClassification { - const statusClassification = classifyStatus(getStatus(error)); + const statusClassification = classifyStatus(extractErrorStatus(error)); if (statusClassification !== null) return statusClassification; const codeClassification = classifyCode(getCode(error)); diff --git a/src/agent-cli-provider/gateway-client.ts b/src/agent-cli-provider/gateway-client.ts new file mode 100644 index 00000000..760c50d8 --- /dev/null +++ b/src/agent-cli-provider/gateway-client.ts @@ -0,0 +1,170 @@ +import { getArray, getRecord, getString, isRecord, unknownToMessage } from './json'; + +export interface GatewayChatToolDefinition { + readonly type: 'function'; + readonly function: { + readonly name: string; + readonly description: string; + readonly parameters: Record; + }; +} + +export interface GatewayToolCall { + readonly id: string; + readonly name: string; + readonly argumentsText: string; +} + +export interface GatewayChatMessage { + readonly role: 'system' | 'user' | 'assistant' | 'tool'; + readonly content: string; + readonly toolCalls?: readonly GatewayToolCall[]; + readonly toolCallId?: string; +} + +export interface GatewayChatResponse { + readonly text: string; + readonly toolCalls: readonly GatewayToolCall[]; +} + +class GatewayHttpError extends Error { + readonly status: number; + + constructor(status: number, message: string) { + super(message); + this.name = 'GatewayHttpError'; + this.status = status; + } +} + +export async function createGatewayChatCompletion(input: { + readonly baseUrl: string; + readonly apiKey: string; + readonly headers: Readonly>; + readonly model: string; + readonly messages: readonly GatewayChatMessage[]; + readonly tools: readonly GatewayChatToolDefinition[]; +}): Promise { + const response = await fetch(`${input.baseUrl}/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${input.apiKey}`, + ...input.headers, + }, + body: JSON.stringify({ + model: input.model, + messages: input.messages.map((message) => serializeMessage(message)), + tools: input.tools, + tool_choice: 'auto', + temperature: 0, + }), + }); + + const bodyText = await response.text(); + const parsed = tryParseJson(bodyText); + if (!response.ok) { + throw httpError(response.status, parsed ?? bodyText); + } + if (!isRecord(parsed)) { + throw new Error('Gateway returned a non-JSON response.'); + } + + const choice = getArray(parsed, 'choices')[0]; + if (!isRecord(choice)) { + throw new Error('Gateway response did not include choices[0].'); + } + const message = getRecord(choice, 'message'); + if (message === null) { + throw new Error('Gateway response did not include choices[0].message.'); + } + + return { + text: getGatewayMessageText(message), + toolCalls: getGatewayToolCalls(message), + }; +} + +function serializeMessage(message: GatewayChatMessage): Record { + if (message.role === 'assistant' && message.toolCalls && message.toolCalls.length > 0) { + return { + role: 'assistant', + content: message.content, + tool_calls: message.toolCalls.map((toolCall) => ({ + id: toolCall.id, + type: 'function', + function: { + name: toolCall.name, + arguments: toolCall.argumentsText, + }, + })), + }; + } + if (message.role === 'tool') { + return { + role: 'tool', + tool_call_id: message.toolCallId, + content: message.content, + }; + } + return { + role: message.role, + content: message.content, + }; +} + +function getGatewayMessageText(message: Record): string { + const content = message.content; + if (typeof content === 'string') return content; + if (!Array.isArray(content)) return ''; + + let text = ''; + for (const item of content) { + if (!isRecord(item)) continue; + const type = getString(item, 'type'); + if (type === 'text') { + text += getString(item, 'text') ?? ''; + } + } + return text; +} + +function getGatewayToolCalls(message: Record): readonly GatewayToolCall[] { + const result: GatewayToolCall[] = []; + for (const item of getArray(message, 'tool_calls')) { + if (!isRecord(item)) continue; + const id = getString(item, 'id'); + const fn = getRecord(item, 'function'); + const name = fn === null ? null : getString(fn, 'name'); + const argumentsText = fn === null ? null : getString(fn, 'arguments'); + if (!id || !name || argumentsText === null) continue; + result.push({ id, name, argumentsText }); + } + return result; +} + +function httpError(status: number, body: unknown): GatewayHttpError { + return new GatewayHttpError(status, buildGatewayErrorMessage(status, body)); +} + +function buildGatewayErrorMessage(status: number, body: unknown): string { + if (isRecord(body)) { + const nested = getRecord(body, 'error'); + if (nested) { + const message = getString(nested, 'message'); + if (message) return `Gateway request failed with status ${status}: ${message}`; + return `Gateway request failed with status ${status}: ${unknownToMessage(nested)}`; + } + const message = getString(body, 'message'); + if (message) return `Gateway request failed with status ${status}: ${message}`; + } + return `Gateway request failed with status ${status}: ${unknownToMessage(body)}`; +} + +function tryParseJson(value: string): unknown | null { + try { + return JSON.parse(value) as unknown; + } catch { + return null; + } +} diff --git a/src/agent-cli-provider/gateway-runner.ts b/src/agent-cli-provider/gateway-runner.ts new file mode 100644 index 00000000..3b8272c5 --- /dev/null +++ b/src/agent-cli-provider/gateway-runner.ts @@ -0,0 +1,353 @@ +import { + createGatewayChatCompletion, + type GatewayChatMessage, + type GatewayChatToolDefinition, + type GatewayToolCall, +} from './gateway-client'; +import { + type GatewayToolExecutionResult, + executeGatewayToolCall, + resolveGatewayConfiguration, +} from './gateway-tools'; +import { isRecord, parseJson, stringifyJson, unknownToMessage } from './json'; +import type { GatewayBuildOptions, GatewayToolPolicy, OutputEvent, ResolvedGatewayBuildOptions } from './types'; + +const MAX_GATEWAY_TURNS = 12; + +export interface GatewayRunnerRequest { + readonly context: string; + readonly gateway: GatewayBuildOptions; + readonly cwd: string; +} + +const TOOL_DEFINITIONS: readonly GatewayChatToolDefinition[] = [ + { + type: 'function', + function: { + name: 'read_file', + description: 'Read one UTF-8 file within toolPolicy.roots.', + parameters: { + type: 'object', + additionalProperties: false, + properties: { + path: { type: 'string' }, + }, + required: ['path'], + }, + }, + }, + { + type: 'function', + function: { + name: 'apply_patch', + description: + 'Write a full file with content, or replace search text in a UTF-8 file within toolPolicy.roots.', + parameters: { + type: 'object', + additionalProperties: false, + properties: { + path: { type: 'string' }, + content: { type: 'string' }, + search: { type: 'string' }, + replace: { type: 'string' }, + replaceAll: { type: 'boolean' }, + }, + required: ['path'], + }, + }, + }, + { + type: 'function', + function: { + name: 'run_command', + description: 'Run an allowlisted command without a shell.', + parameters: { + type: 'object', + additionalProperties: false, + properties: { + command: { type: 'string' }, + args: { type: 'array', items: { type: 'string' } }, + cwd: { type: 'string' }, + }, + required: ['command'], + }, + }, + }, +] as const; + +export async function runGatewayRequest(request: GatewayRunnerRequest): Promise { + const events: OutputEvent[] = []; + try { + const gateway = resolveGatewayConfiguration(request.gateway, 'gateway', request.cwd); + return await runGatewayLoop(request.context, gateway, request.cwd, events); + } catch (error) { + return [ + ...events, + { + type: 'result', + success: false, + error: unknownToMessage(error), + }, + ]; + } +} + +export function gatewayRunnerRequestFromEnv( + env: NodeJS.ProcessEnv = process.env +): GatewayRunnerRequest { + const raw = env.ZEROSHOT_GATEWAY_REQUEST; + if (!raw) { + throw new Error('ZEROSHOT_GATEWAY_REQUEST is required.'); + } + const parsed = parseJson(raw); + if (!isRecord(parsed)) { + throw new Error('ZEROSHOT_GATEWAY_REQUEST must encode a JSON object.'); + } + const context = parsed.context; + const cwd = parsed.cwd; + if (typeof context !== 'string' || !context.trim()) { + throw new Error('ZEROSHOT_GATEWAY_REQUEST.context must be a non-empty string.'); + } + if (typeof cwd !== 'string' || !cwd.trim()) { + throw new Error('ZEROSHOT_GATEWAY_REQUEST.cwd must be a non-empty string.'); + } + const headers = gatewayHeadersFromEnv(parsed.gatewayHeaderEnv, env); + return { + context, + cwd, + gateway: { + ...(isRecord(parsed.gateway) ? (parsed.gateway as GatewayBuildOptions) : {}), + ...(headers === undefined ? {} : { headers }), + ...(env.ZEROSHOT_GATEWAY_API_KEY === undefined + ? {} + : { apiKey: env.ZEROSHOT_GATEWAY_API_KEY }), + }, + }; +} + +function gatewayHeadersFromEnv( + value: unknown, + env: NodeJS.ProcessEnv +): Readonly> | undefined { + if (!isRecord(value)) return undefined; + const headers: Record = {}; + for (const [name, envKey] of Object.entries(value)) { + if (typeof envKey !== 'string' || !envKey) { + throw new Error('ZEROSHOT_GATEWAY_REQUEST.gatewayHeaderEnv entries must be non-empty strings.'); + } + const headerValue = env[envKey]; + if (typeof headerValue !== 'string') { + throw new Error(`Gateway header env ${envKey} is required.`); + } + headers[name] = headerValue; + } + return headers; +} + +async function runGatewayLoop( + context: string, + gateway: ResolvedGatewayBuildOptions, + cwd: string, + events: OutputEvent[] +): Promise { + const toolState: GatewayToolState = { lastError: null }; + const messages: GatewayChatMessage[] = [ + { + role: 'system', + content: buildSystemPrompt(gateway.toolPolicy), + }, + { + role: 'user', + content: context, + }, + ]; + + for (let turn = 0; turn < MAX_GATEWAY_TURNS; turn += 1) { + const response = await createGatewayChatCompletion({ + baseUrl: gateway.baseUrl, + apiKey: gateway.apiKey, + headers: gateway.headers, + model: gateway.model, + messages, + tools: TOOL_DEFINITIONS, + }); + + if (response.text.trim()) { + events.push({ type: 'text', text: response.text }); + } + + if (response.toolCalls.length === 0) { + events.push(finalGatewayResult(response.text, cwd, toolState)); + return events; + } + + messages.push({ + role: 'assistant', + content: response.text, + toolCalls: response.toolCalls, + }); + + const errorResult = await executeGatewayToolCalls( + response.toolCalls, + gateway.toolPolicy, + events, + messages, + toolState + ); + if (errorResult !== null) { + return [...events, errorResult]; + } + } + + return [ + ...events, + { + type: 'result', + success: false, + error: `Gateway runner exceeded ${MAX_GATEWAY_TURNS} tool turns.`, + }, + ]; +} + +interface GatewayToolState { + lastError: string | null; +} + +function finalGatewayResult(text: string, cwd: string, toolState: GatewayToolState): OutputEvent { + if (toolState.lastError !== null) { + return gatewayToolFailureResult(toolState.lastError); + } + return { + type: 'result', + success: true, + result: { + text, + cwd, + }, + }; +} + +async function executeGatewayToolCalls( + toolCalls: readonly GatewayToolCall[], + toolPolicy: GatewayToolPolicy, + events: OutputEvent[], + messages: GatewayChatMessage[], + toolState: GatewayToolState +): Promise { + for (const toolCall of toolCalls) { + events.push({ + type: 'tool_call', + toolName: toolCall.name, + toolId: toolCall.id, + input: tryParseJson(toolCall.argumentsText) ?? toolCall.argumentsText, + }); + + const toolResult = await executeGatewayTool(toolCall.name, toolCall.argumentsText, toolPolicy); + + events.push({ + type: 'tool_result', + toolId: toolCall.id, + content: toolResult.content, + isError: toolResult.isError, + }); + if (toolResult.isError) { + toolState.lastError = toolResultErrorMessage(toolResult.content); + return gatewayToolFailureResult(toolState.lastError); + } + messages.push({ + role: 'tool', + toolCallId: toolCall.id, + content: stringifyJson(toolResult.content), + }); + } + return null; +} + +function toolResultErrorMessage(content: unknown): string { + if (typeof content === 'string' && content.trim()) return content; + if (isRecord(content)) { + const message = content.message; + if (typeof message === 'string' && message.trim()) return message; + } + return stringifyJson(content); +} + +function gatewayToolFailureResult(message: string): OutputEvent { + return { + type: 'result', + success: false, + error: `Gateway runner observed a tool failure and cannot verify completion: ${message}`, + }; +} + +async function executeGatewayTool( + toolName: string, + argumentsText: string, + toolPolicy: GatewayToolPolicy +): Promise { + let parsedArguments: unknown; + try { + parsedArguments = parseJson(argumentsText); + } catch { + return { + content: { message: `Gateway tool "${toolName}" returned malformed JSON arguments.` }, + isError: true, + }; + } + + try { + return await executeGatewayToolCall(toolName, parsedArguments, toolPolicy); + } catch (error) { + return { + content: { message: unknownToMessage(error) }, + isError: true, + }; + } +} + +function buildSystemPrompt(toolPolicy: GatewayToolPolicy): string { + return [ + 'You are a noninteractive coding agent.', + 'Use tools only when needed.', + 'Never request confirmation.', + `Allowed roots: ${toolPolicy.roots.join(', ')}`, + `Allowlisted commands: ${toolPolicy.commands.join(', ') || '(none)'}`, + 'Return a concise final answer when the task is complete.', + ].join(' '); +} + +function tryParseJson(value: string): unknown | null { + try { + return JSON.parse(value) as unknown; + } catch { + return null; + } +} + +async function main(): Promise { + try { + const request = gatewayRunnerRequestFromEnv(); + const events = await runGatewayRequest(request); + writeEvents(events); + process.exitCode = 0; + } catch (error) { + writeEvents([ + { + type: 'result', + success: false, + error: unknownToMessage(error), + }, + ]); + process.exitCode = 0; + } +} + +function writeEvents(events: readonly OutputEvent[]): void { + for (const event of events) { + process.stdout.write(`${stringifyJson(event)}\n`); + } +} + +if (require.main === module) { + void main(); +} diff --git a/src/agent-cli-provider/gateway-tools.ts b/src/agent-cli-provider/gateway-tools.ts new file mode 100644 index 00000000..bd3a2bcc --- /dev/null +++ b/src/agent-cli-provider/gateway-tools.ts @@ -0,0 +1,616 @@ +import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; +import { contractError, invalidField } from './contract-errors'; +import { getString, isRecord, unknownToMessage } from './json'; +import type { + GatewayBuildOptions, + GatewayToolPolicy, + ResolvedGatewayBuildOptions, +} from './types'; + +const DEFAULT_COMMAND_TIMEOUT_MS = 10_000; + +export class GatewayPolicyError extends Error { + constructor(message: string) { + super(message); + this.name = 'GatewayPolicyError'; + } +} + +export interface GatewayToolExecutionResult { + readonly content: unknown; + readonly isError: boolean; +} + +interface RunCommandRequest { + readonly command: string; + readonly args: readonly string[]; + readonly cwd?: string | undefined; +} + +interface ApplyPatchRequest { + readonly path: string; + readonly content?: string | undefined; + readonly search?: string | undefined; + readonly replace?: string | undefined; + readonly replaceAll: boolean; +} + +export function normalizeGatewayBuildOptions( + value: unknown, + field: string, + cwd: string +): GatewayBuildOptions | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) { + invalidField(field, `${field} must be an object.`); + } + + const result: Record = {}; + const baseUrl = optionalString(value.baseUrl, `${field}.baseUrl`); + const apiKey = optionalString(value.apiKey, `${field}.apiKey`); + const model = optionalNullableString(value.model, `${field}.model`); + const headers = optionalStringRecord(value.headers, `${field}.headers`); + const toolPolicy = optionalGatewayToolPolicy(value.toolPolicy, `${field}.toolPolicy`, cwd); + + if (baseUrl !== undefined) result.baseUrl = baseUrl; + if (typeof apiKey === 'string') result.apiKey = apiKey; + if (model !== undefined) result.model = model; + if (headers !== undefined) result.headers = headers; + if (toolPolicy !== undefined) result.toolPolicy = toolPolicy; + + return result as GatewayBuildOptions; +} + +export function resolveGatewayConfiguration( + value: GatewayBuildOptions | undefined, + field: string, + cwd: string +): ResolvedGatewayBuildOptions { + if (value === undefined) { + throw contractError({ + code: 'invalid-field', + field, + exitCode: 2, + message: `${field} is required for the gateway provider.`, + }); + } + const baseUrl = requiredNonEmptyString(value.baseUrl, `${field}.baseUrl`); + const apiKey = requiredNonEmptyString(value.apiKey, `${field}.apiKey`); + const model = requiredNonEmptyString(value.model, `${field}.model`); + const headers = value.headers ?? {}; + const toolPolicy = requiredGatewayToolPolicy(value.toolPolicy, `${field}.toolPolicy`, cwd); + + assertValidGatewayBaseUrl(baseUrl, `${field}.baseUrl`); + return { + baseUrl: normalizeBaseUrl(baseUrl), + apiKey, + headers, + model, + toolPolicy, + }; +} + +export function validateGatewaySettings(settings: Record): string | null { + try { + optionalString(settings.baseUrl, 'providerSettings.gateway.baseUrl'); + optionalString(settings.apiKey, 'providerSettings.gateway.apiKey'); + optionalNullableString(settings.model, 'providerSettings.gateway.model'); + optionalStringRecord(settings.headers, 'providerSettings.gateway.headers'); + optionalGatewayToolPolicy( + settings.toolPolicy, + 'providerSettings.gateway.toolPolicy', + process.cwd() + ); + return null; + } catch (error) { + return unknownToMessage(error); + } +} + +export function normalizeGatewayToolPolicy( + value: GatewayToolPolicy, + field: string, + cwd: string +): GatewayToolPolicy { + const roots = normalizeGatewayRoots(value.roots, `${field}.roots`, cwd); + const commands = normalizeGatewayCommands(value.commands, `${field}.commands`); + const timeoutMs = optionalFiniteInteger(value.commandTimeoutMs, `${field}.commandTimeoutMs`); + return timeoutMs === undefined + ? { roots, commands } + : { roots, commands, commandTimeoutMs: timeoutMs }; +} + +export async function executeGatewayToolCall( + toolName: string, + input: unknown, + policy: GatewayToolPolicy +): Promise { + switch (toolName) { + case 'read_file': + return { content: await readGatewayFile(input, policy), isError: false }; + case 'apply_patch': + return { content: await applyGatewayPatch(input, policy), isError: false }; + case 'run_command': + return runGatewayCommand(input, policy); + default: + throw new GatewayPolicyError(`Unsupported gateway tool: ${toolName}`); + } +} + +function optionalGatewayToolPolicy( + value: unknown, + field: string, + cwd: string +): GatewayToolPolicy | undefined { + if (value === undefined || value === null) return undefined; + if (!isRecord(value)) { + invalidField(field, `${field} must be an object.`); + } + + return normalizeGatewayToolPolicy( + { + roots: stringArray(value.roots, `${field}.roots`), + commands: stringArray(value.commands, `${field}.commands`), + ...(value.commandTimeoutMs === undefined + ? {} + : { + commandTimeoutMs: requiredFiniteInteger( + value.commandTimeoutMs, + `${field}.commandTimeoutMs` + ), + }), + }, + field, + cwd + ); +} + +function requiredGatewayToolPolicy( + value: GatewayToolPolicy | undefined, + field: string, + cwd: string +): GatewayToolPolicy { + if (value === undefined) { + throw contractError({ + code: 'invalid-field', + field, + exitCode: 2, + message: `${field} is required for the gateway provider.`, + }); + } + return normalizeGatewayToolPolicy(value, field, cwd); +} + +function normalizeGatewayRoots(value: readonly string[], field: string, cwd: string): readonly string[] { + if (value.length === 0) { + invalidField(field, `${field} must contain at least one root path.`); + } + return value.map((root, index) => { + const normalized = root.trim(); + if (!normalized) { + invalidField(`${field}[${index}]`, `${field}[${index}] must be a non-empty string.`); + } + return path.resolve(cwd, normalized); + }); +} + +function normalizeGatewayCommands(value: readonly string[], field: string): readonly string[] { + return value.map((command, index) => { + const normalized = command.trim(); + if (!normalized) { + invalidField(`${field}[${index}]`, `${field}[${index}] must be a non-empty string.`); + } + if (/\s/.test(normalized)) { + invalidField(`${field}[${index}]`, `${field}[${index}] must not contain whitespace.`); + } + return normalized; + }); +} + +async function assertWithinRoots( + targetPath: string, + roots: readonly string[], + field: string, + options: { readonly allowMissingLeaf?: boolean } = {} +): Promise { + const resolvedRoots = await resolveGatewayRoots(roots); + if (resolvedRoots.length === 0) { + throw new GatewayPolicyError('toolPolicy.roots must include at least one root.'); + } + const candidates = getGatewayTargetCandidates(targetPath, resolvedRoots); + const firstExistingMatch = await resolveFirstMatchingGatewayTarget(candidates, resolvedRoots, { + throwOnMissing: !options.allowMissingLeaf, + }); + if (firstExistingMatch !== undefined) return firstExistingMatch; + if (options.allowMissingLeaf) { + const missingLeafMatch = await resolveFirstMatchingGatewayTarget(candidates, resolvedRoots, { + allowMissingLeaf: true, + }); + if (missingLeafMatch !== undefined) return missingLeafMatch; + } + + throw new GatewayPolicyError(`${field} must stay within toolPolicy.roots.`); +} + +interface ResolvedGatewayRoot { + readonly configuredPath: string; + readonly realPath: string; +} + +function resolveGatewayRoots(roots: readonly string[]): Promise { + return Promise.all( + roots.map(async (configuredPath) => ({ + configuredPath, + realPath: await realpath(configuredPath), + })) + ); +} + +function getGatewayTargetCandidates( + targetPath: string, + roots: readonly ResolvedGatewayRoot[] +): readonly string[] { + if (path.isAbsolute(targetPath)) { + return [path.resolve(targetPath)]; + } + return roots.map((root) => path.resolve(root.configuredPath, targetPath)); +} + +async function resolveFirstMatchingGatewayTarget( + candidates: readonly string[], + resolvedRoots: readonly ResolvedGatewayRoot[], + options: { readonly allowMissingLeaf?: boolean; readonly throwOnMissing?: boolean } = {} +): Promise { + let firstMissingError: NodeJS.ErrnoException | undefined; + + for (const candidate of candidates) { + let resolvedTarget: string; + try { + resolvedTarget = options.allowMissingLeaf + ? await resolveGatewayTargetPath(candidate) + : await realpath(candidate); + } catch (error) { + if (isNodeErrorWithCode(error, 'ENOENT')) { + firstMissingError ??= error; + continue; + } + throw error; + } + + if (resolvedRoots.some((root) => isWithinRoot(resolvedTarget, root.realPath))) { + return resolvedTarget; + } + } + + if (firstMissingError !== undefined && options.throwOnMissing) { + throw firstMissingError; + } + return undefined; +} + +async function resolveGatewayTargetPath(candidatePath: string): Promise { + try { + return await realpath(candidatePath); + } catch (error) { + if (!isNodeErrorWithCode(error, 'ENOENT')) throw error; + } + + const missingSegments: string[] = []; + let current = candidatePath; + while (true) { + const parent = path.dirname(current); + if (parent === current) { + throw new GatewayPolicyError(`Unable to resolve ${candidatePath} within toolPolicy.roots.`); + } + missingSegments.unshift(path.basename(current)); + current = parent; + try { + const realCurrent = await realpath(current); + return path.join(realCurrent, ...missingSegments); + } catch (error) { + if (!isNodeErrorWithCode(error, 'ENOENT')) throw error; + } + } +} + +function isWithinRoot(targetPath: string, rootPath: string): boolean { + const relative = path.relative(rootPath, targetPath); + return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative)); +} + +function isNodeErrorWithCode(error: unknown, code: string): error is NodeJS.ErrnoException { + return isRecord(error) && error.code === code; +} + +function readGatewayFileInput(input: unknown): string { + if (!isRecord(input)) { + throw new Error('read_file input must be an object.'); + } + const targetPath = getString(input, 'path'); + if (!targetPath || !targetPath.trim()) { + throw new Error('read_file.path must be a non-empty string.'); + } + return targetPath; +} + +async function readGatewayFile( + input: unknown, + policy: GatewayToolPolicy +): Promise> { + const targetPath = await assertWithinRoots(readGatewayFileInput(input), policy.roots, 'read_file.path'); + const content = await readFile(targetPath, 'utf8'); + return { path: targetPath, content }; +} + +function readApplyPatchInput(input: unknown): ApplyPatchRequest { + if (!isRecord(input)) { + throw new Error('apply_patch input must be an object.'); + } + const targetPath = getString(input, 'path'); + if (!targetPath || !targetPath.trim()) { + throw new Error('apply_patch.path must be a non-empty string.'); + } + const content = optionalRuntimeString(input.content, 'apply_patch.content'); + const search = optionalRuntimeString(input.search, 'apply_patch.search'); + const replace = optionalRuntimeString(input.replace, 'apply_patch.replace'); + const replaceAll = optionalBoolean(input.replaceAll, 'apply_patch.replaceAll') ?? false; + if (content === undefined) { + if (search === undefined || replace === undefined) { + throw new Error('apply_patch requires either content or both search and replace.'); + } + if (search.length === 0) { + throw new Error('apply_patch.search must be a non-empty string.'); + } + } + return { path: targetPath, content, search, replace, replaceAll }; +} + +async function applyGatewayPatch( + input: unknown, + policy: GatewayToolPolicy +): Promise> { + const request = readApplyPatchInput(input); + const targetPath = await assertWithinRoots(request.path, policy.roots, 'apply_patch.path', { + allowMissingLeaf: true, + }); + await mkdir(path.dirname(targetPath), { recursive: true }); + + if (request.content !== undefined) { + await writeFile(targetPath, request.content, 'utf8'); + return { path: targetPath, bytesWritten: Buffer.byteLength(request.content, 'utf8') }; + } + + const current = await readFile(targetPath, 'utf8'); + const search = request.search; + const replace = request.replace; + if (search === undefined || replace === undefined) { + throw new Error('apply_patch requires either content or both search and replace.'); + } + if (!current.includes(search)) { + throw new Error('apply_patch.search did not match the target file.'); + } + const next = request.replaceAll ? current.split(search).join(replace) : current.replace(search, replace); + await writeFile(targetPath, next, 'utf8'); + return { + path: targetPath, + bytesWritten: Buffer.byteLength(next, 'utf8'), + replaced: request.replaceAll ? 'all' : 'first', + }; +} + +function readRunCommandInput(input: unknown): RunCommandRequest { + if (!isRecord(input)) { + throw new Error('run_command input must be an object.'); + } + const command = getString(input, 'command'); + if (!command || !command.trim()) { + throw new Error('run_command.command must be a non-empty string.'); + } + if (/\s/.test(command)) { + throw new GatewayPolicyError('run_command.command must not contain whitespace.'); + } + + const rawArgs = requiredArrayIfPresent(input, 'args', 'run_command.args'); + const args = rawArgs.map((item, index) => { + if (typeof item !== 'string') { + throw new Error(`run_command.args[${index}] must be a string.`); + } + return item; + }); + const cwd = optionalRuntimeString(input.cwd, 'run_command.cwd'); + return { command, args, cwd }; +} + +async function runGatewayCommand( + input: unknown, + policy: GatewayToolPolicy +): Promise { + const request = readRunCommandInput(input); + if (!policy.commands.includes(request.command)) { + throw new GatewayPolicyError( + `run_command.command "${request.command}" is not allowlisted by toolPolicy.commands.` + ); + } + + const timeoutMs = policy.commandTimeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS; + const result = request.cwd + ? await spawnGatewayCommand( + request.command, + request.args, + await assertWithinRoots(request.cwd, policy.roots, 'run_command.cwd'), + timeoutMs + ) + : await spawnGatewayCommand( + request.command, + request.args, + await resolveDefaultGatewayCommandCwd(policy.roots), + timeoutMs + ); + return { + isError: result.exitCode !== 0 || result.signal !== null, + content: result, + }; +} + +async function resolveDefaultGatewayCommandCwd(roots: readonly string[]): Promise { + const resolvedRoots = await resolveGatewayRoots(roots); + if (resolvedRoots.length === 0) { + throw new GatewayPolicyError('toolPolicy.roots must include at least one root.'); + } + const [firstRoot] = resolvedRoots; + if (!firstRoot) { + throw new GatewayPolicyError('toolPolicy.roots must include at least one root.'); + } + return firstRoot.realPath; +} + +function spawnGatewayCommand( + command: string, + args: readonly string[], + cwd: string, + timeoutMs: number +): Promise> { + return new Promise((resolve, reject) => { + const child = spawn(command, [...args], { + cwd, + env: { + PATH: process.env.PATH ?? '', + HOME: process.env.HOME ?? '', + }, + shell: false, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + child.kill('SIGKILL'); + }, timeoutMs); + + child.stdout?.on('data', (chunk: Buffer) => stdout.push(chunk)); + child.stderr?.on('data', (chunk: Buffer) => stderr.push(chunk)); + child.once('error', (error) => { + clearTimeout(timeout); + reject(error); + }); + child.once('close', (exitCode, signal) => { + clearTimeout(timeout); + resolve({ + command, + args, + cwd, + exitCode, + signal, + timedOut, + stdout: Buffer.concat(stdout).toString('utf8'), + stderr: Buffer.concat(stderr).toString('utf8'), + }); + }); + }); +} + +function requiredNonEmptyString(value: string | null | undefined, field: string): string { + if (typeof value === 'string' && value.trim()) return value.trim(); + throw contractError({ + code: 'invalid-field', + field, + exitCode: 2, + message: `${field} must be a non-empty string.`, + }); +} + +function optionalString(value: unknown, field: string): string | undefined { + if (value === undefined || value === null) return undefined; + if (typeof value === 'string') return value; + invalidField(field, `${field} must be a string.`); +} + +function optionalBoolean(value: unknown, field: string): boolean | undefined { + if (value === undefined) return undefined; + if (typeof value === 'boolean') return value; + throw new Error(`${field} must be a boolean.`); +} + +function optionalRuntimeString(value: unknown, field: string): string | undefined { + if (value === undefined) return undefined; + if (typeof value === 'string') return value; + throw new Error(`${field} must be a string.`); +} + +function optionalNullableString(value: unknown, field: string): string | null | undefined { + if (value === undefined) return undefined; + if (value === null) return null; + if (typeof value === 'string') return value; + invalidField(field, `${field} must be a string or null.`); +} + +function optionalStringRecord( + value: unknown, + field: string +): Readonly> | undefined { + if (value === undefined || value === null) return undefined; + if (!isRecord(value)) { + invalidField(field, `${field} must be an object with string values.`); + } + const result: Record = {}; + for (const [key, item] of Object.entries(value)) { + if (typeof item !== 'string') { + invalidField(`${field}.${key}`, `${field}.${key} must be a string.`); + } + result[key] = item; + } + return result; +} + +function stringArray(value: unknown, field: string): readonly string[] { + if (!Array.isArray(value)) { + invalidField(field, `${field} must be an array of strings.`); + } + return value.map((item, index) => { + if (typeof item !== 'string') { + invalidField(`${field}[${index}]`, `${field}[${index}] must be a string.`); + } + return item; + }); +} + +function requiredArrayIfPresent( + record: Record, + key: string, + field: string +): readonly unknown[] { + const value = record[key]; + if (value === undefined) return []; + if (!Array.isArray(value)) { + throw new Error(`${field} must be an array.`); + } + return value; +} + +function optionalFiniteInteger(value: unknown, field: string): number | undefined { + if (value === undefined) return undefined; + return requiredFiniteInteger(value, field); +} + +function requiredFiniteInteger(value: unknown, field: string): number { + if (typeof value === 'number' && Number.isInteger(value) && value > 0) return value; + invalidField(field, `${field} must be a positive integer.`); +} + +function assertValidGatewayBaseUrl(value: string, field: string): void { + try { + const parsed = new URL(value); + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + invalidField(field, `${field} must use http or https.`); + } + } catch { + invalidField(field, `${field} must be a valid URL.`); + } +} + +function normalizeBaseUrl(value: string): string { + return value.replace(/\/+$/, ''); +} diff --git a/src/agent-cli-provider/index.ts b/src/agent-cli-provider/index.ts index d67da1cb..2c6dd19c 100644 --- a/src/agent-cli-provider/index.ts +++ b/src/agent-cli-provider/index.ts @@ -35,20 +35,40 @@ export { export { detectRuntimeProviderCliFeatures, prepareSingleAgentProviderCommand, + probeRuntimeProviderCli, type PreparedSingleAgentProviderCommand, + type RuntimeProviderProbe, type SingleAgentProviderCommandInput, } from './single-agent-runtime'; +export { + findProviderRegistryEntry, + getProviderRegistryEntry, + knownProviderNames, + listProviderRegistryEntries, + normalizeProviderName, + providerAliasMap, + providerAliases, + providerIds, + providerRegistry, + resolveProviderCommand, + supportsProviderCapability, +} from './provider-registry'; export type { AgentCliProviderHelperMetadata, BuildProviderCommandOptions, CleanupMetadata, CliFeatureOverrides, + AcpCliFeatures, ClaudeCliFeatures, CodexCliFeatures, + CopilotCliFeatures, CommandSpec, ErrorClassification, ErrorClassificationKind, + GatewayBuildOptions, + GatewayCliFeatures, + GatewayToolPolicy, GeminiCliFeatures, KnownProviderName, LevelModelSpec, @@ -59,11 +79,19 @@ export type { OpencodeCliFeatures, OutputEvent, OutputFormat, + PiCliFeatures, ProviderAdapter, + ProviderCapabilities, + ProviderCapabilityState, + ProviderCommandSpec, + ProviderDocsMetadata, + ProviderInvokeSpec, ProviderAlias, ProviderCliFeatures, ProviderId, + ProviderRegistryEntry, RedactionMetadata, + ResolvedGatewayBuildOptions, ResolvedModelSpec, ResultEvent, TextEvent, diff --git a/src/agent-cli-provider/invoke-evidence.ts b/src/agent-cli-provider/invoke-evidence.ts index 034b7d49..0d366247 100644 --- a/src/agent-cli-provider/invoke-evidence.ts +++ b/src/agent-cli-provider/invoke-evidence.ts @@ -1,17 +1,38 @@ import { classifyProviderError } from './adapters'; import type { ProcessResult } from './process-runner'; -import type { ErrorClassification, ProviderAdapter } from './types'; +import type { ErrorClassification, OutputEvent, ProviderAdapter, ResultEvent } from './types'; + +function isFailedResultEvent(event: OutputEvent): event is ResultEvent { + return event.type === 'result' && event.success === false; +} + +function classifyParsedFailure( + adapter: ProviderAdapter, + events: readonly OutputEvent[] +): ErrorClassification | null { + const failedResult = [...events].reverse().find(isFailedResultEvent); + if (failedResult === undefined) return null; + return classifyProviderError(adapter.id, { + message: + typeof failedResult.error === 'string' && failedResult.error.trim() + ? failedResult.error + : 'Provider reported a failed result event.', + }); +} export function providerFailureClassification( adapter: ProviderAdapter, - result: ProcessResult + result: ProcessResult, + events: readonly OutputEvent[] = [] ): ErrorClassification | null { if (result.timedOut) { return classifyProviderError(adapter.id, { message: `Provider timed out after ${result.timeoutMs ?? 'unknown'}ms`, }); } - if (result.exitCode === 0 && result.signal === null) return null; + if (result.exitCode === 0 && result.signal === null) { + return classifyParsedFailure(adapter, events); + } return classifyProviderError(adapter.id, { message: result.stderr || result.stdout || `Provider exited with code ${result.exitCode ?? ''}`, }); diff --git a/src/agent-cli-provider/provider-registry.ts b/src/agent-cli-provider/provider-registry.ts new file mode 100644 index 00000000..f9a4cc43 --- /dev/null +++ b/src/agent-cli-provider/provider-registry.ts @@ -0,0 +1,515 @@ +import { createAcpAdapter } from './adapters/acp'; +import { claudeAdapter } from './adapters/claude'; +import { codexAdapter } from './adapters/codex'; +import { copilotAdapter } from './adapters/copilot'; +import { gatewayAdapter, gatewaySettingsDefaults, validateGatewaySettings } from './adapters/gateway'; +import { geminiAdapter } from './adapters/gemini'; +import { opencodeAdapter } from './adapters/opencode'; +import { piAdapter } from './adapters/pi'; +import { resolveClaudeCommand } from './claude-command'; +import type { ModelLevel, ProviderAdapter } from './types'; + +export type ProviderCapabilityState = boolean | 'experimental'; + +export interface ProviderCapabilities { + readonly dockerIsolation: ProviderCapabilityState; + readonly worktreeIsolation: ProviderCapabilityState; + readonly mcpServers: ProviderCapabilityState; + readonly jsonSchema: ProviderCapabilityState; + readonly streamJson: ProviderCapabilityState; + readonly thinkingMode: ProviderCapabilityState; + readonly reasoningEffort: ProviderCapabilityState; +} + +interface FixedProviderCommandSpec { + readonly kind: 'fixed'; + readonly command: string; + readonly args: readonly string[]; +} + +interface ConfiguredClaudeCommandSpec { + readonly kind: 'configured-claude'; +} + +export interface SpawnProviderInvokeSpec { + readonly lane: 'spawn'; +} + +export interface AcpStdioProviderInvokeSpec { + readonly lane: 'acp-stdio'; + readonly transport: 'stdio'; +} + +export type ProviderInvokeSpec = SpawnProviderInvokeSpec | AcpStdioProviderInvokeSpec; + +export type ProviderCommandSpec = FixedProviderCommandSpec | ConfiguredClaudeCommandSpec; + +export interface ProviderDocsMetadata { + readonly label: string; + readonly setupHeading: string; +} + +export interface ProviderDockerMountPreset { + readonly host: string; + readonly container: string; + readonly readonly: boolean; +} + +export interface ProviderDockerMetadata { + readonly mount: ProviderDockerMountPreset; + readonly envPassthrough: readonly string[]; + // False when the mounted dir doesn't hold the secret (auth is via an envPassthrough token). + readonly credentialInMount?: boolean; + // Shell command that installs this provider's CLI inside the Debian-based cluster image, run as + // a docker-cached build layer for the per-provider image variant. Omit for providers already + // baked into the base image (e.g. Claude) or not installable via a single command. + readonly install?: string; +} + +export interface ProviderRegistryEntry { + readonly id: string; + readonly aliases: readonly string[]; + readonly displayName: string; + readonly binary: string; + readonly command: ProviderCommandSpec; + readonly invoke: ProviderInvokeSpec; + readonly installInstructions: string; + readonly authInstructions: string; + readonly credentialPaths: readonly string[]; + readonly credentialEnvKeys: readonly string[]; + readonly settingsFields: readonly string[]; + readonly settingsDefaults?: Readonly>; + readonly settingsValidator?: (settings: Record) => string | null; + readonly availabilityProbe?: 'command' | 'help-or-version'; + readonly capabilities: ProviderCapabilities; + readonly docs: ProviderDocsMetadata; + readonly docker: ProviderDockerMetadata; + readonly defaultLevels: Readonly<{ + readonly min: ModelLevel; + readonly default: ModelLevel; + readonly max: ModelLevel; + }>; + readonly adapter: ProviderAdapter; +} + +const STANDARD_CAPABILITIES: Readonly< + Pick +> = { + dockerIsolation: true, + worktreeIsolation: true, + mcpServers: true, + streamJson: true, + thinkingMode: true, +}; + +const CLAUDE_DOCKER_ENV_PASSTHROUGH = [ + 'ANTHROPIC_API_KEY', + 'AWS_BEARER_TOKEN_BEDROCK', + 'AWS_REGION', + 'CLAUDE_CODE_USE_BEDROCK', +] as const; + +const SPAWN_INVOKE = Object.freeze({ lane: 'spawn' }) as SpawnProviderInvokeSpec; +const ACP_STDIO_INVOKE = Object.freeze({ + lane: 'acp-stdio', + transport: 'stdio', +}) as AcpStdioProviderInvokeSpec; + +const kiroAdapter = createAcpAdapter({ + provider: 'kiro', + displayName: 'Kiro', + binary: 'kiro-cli', + commandArgs: ['acp'], + credentialEnvKeys: ['KIRO_API_KEY'], + supportsPromptImages: true, + supportsLoadSession: false, + supportsSessionCancel: true, + supportsSessionSetModel: false, + supportsSessionSetMode: false, + retryableErrorPatterns: [ + /\brate(?:[ _])?limit\b/i, + /\btemporar(?:y|ily)\b/i, + /\btimeout\b/i, + /\bunavailable\b/i, + ], + permanentErrorPatterns: [ + /\bauth(?:entication)?\b/i, + /\bapi[_ -]?key\b/i, + /\bforbidden\b/i, + /\bunauthorized\b/i, + /\bcancelled\b/i, + /\bmalformed\b/i, + /\bunsupported\b/i, + ], +}); + +export const providerRegistry = [ + { + id: 'claude', + aliases: ['anthropic'], + displayName: 'Claude', + binary: 'claude', + command: { kind: 'configured-claude' }, + invoke: SPAWN_INVOKE, + installInstructions: + 'npm install -g @anthropic-ai/claude-code\nOr (macOS): brew install claude', + authInstructions: 'claude login', + credentialPaths: ['~/.claude'], + credentialEnvKeys: claudeAdapter.credentialEnvKeys, + settingsFields: ['anthropicApiKey', 'bedrockApiKey', 'bedrockRegion'], + capabilities: { + ...STANDARD_CAPABILITIES, + jsonSchema: true, + reasoningEffort: false, + }, + docs: { + label: 'Claude', + setupHeading: 'Claude Setup', + }, + docker: { + mount: { + host: '~/.claude', + container: '$HOME/.claude', + readonly: true, + }, + envPassthrough: CLAUDE_DOCKER_ENV_PASSTHROUGH, + }, + defaultLevels: { + min: claudeAdapter.defaultMinLevel, + default: claudeAdapter.defaultLevel, + max: claudeAdapter.defaultMaxLevel, + }, + adapter: claudeAdapter, + }, + { + id: 'codex', + aliases: ['openai'], + displayName: 'Codex', + binary: 'codex', + command: { kind: 'fixed', command: 'codex', args: ['exec'] }, + invoke: SPAWN_INVOKE, + installInstructions: 'npm install -g @openai/codex', + authInstructions: 'codex login', + credentialPaths: ['~/.config/codex', '~/.codex'], + credentialEnvKeys: codexAdapter.credentialEnvKeys, + settingsFields: [], + capabilities: { + ...STANDARD_CAPABILITIES, + jsonSchema: true, + reasoningEffort: true, + }, + docs: { + label: 'Codex', + setupHeading: 'Codex Setup', + }, + docker: { + mount: { + host: '~/.config/codex', + container: '$HOME/.config/codex', + readonly: true, + }, + install: 'npm install -g @openai/codex', + envPassthrough: [], + }, + defaultLevels: { + min: codexAdapter.defaultMinLevel, + default: codexAdapter.defaultLevel, + max: codexAdapter.defaultMaxLevel, + }, + adapter: codexAdapter, + }, + { + id: 'gateway', + aliases: [], + displayName: 'Gateway', + binary: 'node', + command: { kind: 'fixed', command: 'node', args: [] }, + invoke: SPAWN_INVOKE, + installInstructions: 'Bundled with Zeroshot; no external provider CLI install is required.', + authInstructions: + 'Configure providerSettings.gateway.baseUrl, apiKey, model, and toolPolicy in Zeroshot settings.', + credentialPaths: [], + credentialEnvKeys: gatewayAdapter.credentialEnvKeys, + settingsFields: ['baseUrl', 'apiKey', 'headers', 'model', 'toolPolicy'], + settingsDefaults: gatewaySettingsDefaults, + settingsValidator: validateGatewaySettings, + capabilities: { + ...STANDARD_CAPABILITIES, + mcpServers: false, + jsonSchema: false, + reasoningEffort: false, + }, + docs: { + label: 'Gateway', + setupHeading: 'Gateway Setup', + }, + docker: { + mount: { + host: '~/.zeroshot', + container: '$HOME/.zeroshot', + readonly: true, + }, + envPassthrough: [], + }, + defaultLevels: { + min: gatewayAdapter.defaultMinLevel, + default: gatewayAdapter.defaultLevel, + max: gatewayAdapter.defaultMaxLevel, + }, + adapter: gatewayAdapter, + }, + { + id: 'gemini', + aliases: ['google'], + displayName: 'Gemini', + binary: 'gemini', + command: { kind: 'fixed', command: 'gemini', args: [] }, + invoke: SPAWN_INVOKE, + installInstructions: 'npm install -g @google/gemini-cli', + authInstructions: 'gemini auth login', + credentialPaths: ['~/.config/gcloud', '~/.config/gemini', '~/.gemini'], + credentialEnvKeys: geminiAdapter.credentialEnvKeys, + settingsFields: [], + capabilities: { + ...STANDARD_CAPABILITIES, + jsonSchema: 'experimental', + reasoningEffort: false, + }, + docs: { + label: 'Gemini', + setupHeading: 'Gemini Setup', + }, + docker: { + mount: { + host: '~/.config/gemini', + container: '$HOME/.config/gemini', + readonly: true, + }, + install: 'npm install -g @google/gemini-cli', + envPassthrough: [], + }, + defaultLevels: { + min: geminiAdapter.defaultMinLevel, + default: geminiAdapter.defaultLevel, + max: geminiAdapter.defaultMaxLevel, + }, + adapter: geminiAdapter, + }, + { + id: 'opencode', + aliases: [], + displayName: 'Opencode', + binary: 'opencode', + command: { kind: 'fixed', command: 'opencode', args: ['run'] }, + invoke: SPAWN_INVOKE, + installInstructions: 'See https://opencode.ai for installation instructions.', + authInstructions: 'opencode auth login', + credentialPaths: ['~/.local/share/opencode'], + credentialEnvKeys: opencodeAdapter.credentialEnvKeys, + settingsFields: [], + capabilities: { + ...STANDARD_CAPABILITIES, + jsonSchema: 'experimental', + reasoningEffort: true, + }, + docs: { + label: 'Opencode', + setupHeading: 'Opencode Setup', + }, + docker: { + mount: { + host: '~/.local/share/opencode', + container: '$HOME/.local/share/opencode', + readonly: true, + }, + envPassthrough: [], + }, + defaultLevels: { + min: opencodeAdapter.defaultMinLevel, + default: opencodeAdapter.defaultLevel, + max: opencodeAdapter.defaultMaxLevel, + }, + adapter: opencodeAdapter, + }, + { + id: 'pi', + aliases: [], + displayName: 'Pi', + binary: 'pi', + command: { kind: 'fixed', command: 'pi', args: [] }, + invoke: SPAWN_INVOKE, + installInstructions: + 'npm install -g --ignore-scripts @earendil-works/pi-coding-agent@0.80.3', + authInstructions: 'pi\n/login', + credentialPaths: ['~/.pi'], + credentialEnvKeys: piAdapter.credentialEnvKeys, + settingsFields: [], + availabilityProbe: 'help-or-version', + capabilities: { + ...STANDARD_CAPABILITIES, + mcpServers: false, + jsonSchema: false, + reasoningEffort: false, + }, + docs: { + label: 'Pi', + setupHeading: 'Pi Setup', + }, + docker: { + mount: { + host: '~/.pi', + container: '$HOME/.pi', + readonly: true, + }, + envPassthrough: [], + }, + defaultLevels: { + min: piAdapter.defaultMinLevel, + default: piAdapter.defaultLevel, + max: piAdapter.defaultMaxLevel, + }, + adapter: piAdapter, + }, + { + id: 'kiro', + aliases: [], + displayName: 'Kiro', + binary: 'kiro-cli', + command: { kind: 'fixed', command: 'kiro-cli', args: ['acp'] }, + invoke: ACP_STDIO_INVOKE, + installInstructions: 'See https://kiro.dev/docs/cli/', + authInstructions: 'See https://kiro.dev/docs/cli/authentication/', + credentialPaths: ['~/.kiro'], + credentialEnvKeys: kiroAdapter.credentialEnvKeys, + settingsFields: [], + capabilities: { + ...STANDARD_CAPABILITIES, + mcpServers: false, + jsonSchema: false, + reasoningEffort: false, + }, + docs: { + label: 'Kiro', + setupHeading: 'Kiro Setup', + }, + docker: { + mount: { + host: '~/.kiro', + container: '$HOME/.kiro', + readonly: true, + }, + envPassthrough: ['KIRO_API_KEY'], + }, + defaultLevels: { + min: kiroAdapter.defaultMinLevel, + default: kiroAdapter.defaultLevel, + max: kiroAdapter.defaultMaxLevel, + }, + adapter: kiroAdapter, + }, + { + id: 'copilot', + aliases: [], + displayName: 'Copilot', + binary: 'copilot', + command: { kind: 'fixed', command: 'copilot', args: [] }, + invoke: SPAWN_INVOKE, + installInstructions: 'npm install -g @github/copilot', + // Docker/CI can't use the keychain token; export COPILOT_GITHUB_TOKEN instead. + authInstructions: 'copilot\n/login\n(Docker/CI: export COPILOT_GITHUB_TOKEN=)', + credentialPaths: ['~/.copilot'], + credentialEnvKeys: copilotAdapter.credentialEnvKeys, + settingsFields: [], + availabilityProbe: 'help-or-version', + // MCP servers pass through via the `--additional-mcp-config` CLI flag (see copilot adapter + // addMcpArgs). No native output-schema or reasoning-effort flag. + capabilities: { + ...STANDARD_CAPABILITIES, + mcpServers: true, + jsonSchema: false, + reasoningEffort: false, + }, + docs: { + label: 'Copilot', + setupHeading: 'Copilot Setup', + }, + docker: { + mount: { + host: '~/.copilot', + container: '$HOME/.copilot', + readonly: true, + }, + install: 'npm install -g @github/copilot', + envPassthrough: ['COPILOT_GITHUB_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN'], + credentialInMount: false, // token is in the OS keychain, not ~/.copilot + }, + defaultLevels: { + min: copilotAdapter.defaultMinLevel, + default: copilotAdapter.defaultLevel, + max: copilotAdapter.defaultMaxLevel, + }, + adapter: copilotAdapter, + }, +] as const satisfies readonly ProviderRegistryEntry[]; + +type RegistryProviderId = (typeof providerRegistry)[number]['id']; +type RegistryProviderAlias = (typeof providerRegistry)[number]['aliases'][number]; + +export const providerIds = providerRegistry.map((entry) => entry.id) as readonly RegistryProviderId[]; +export const providerAliases = providerRegistry.flatMap((entry) => entry.aliases) as readonly RegistryProviderAlias[]; +export const knownProviderNames = providerRegistry.flatMap((entry) => [entry.id, ...entry.aliases]) as readonly ( + | RegistryProviderId + | RegistryProviderAlias +)[]; + +export const providerAliasMap: Readonly> = Object.freeze( + providerRegistry.reduce>((result, entry) => { + result[entry.id] = entry.id; + for (const alias of entry.aliases) { + result[alias] = entry.id; + } + return result; + }, {}) +); + +export function normalizeProviderName(name: string): RegistryProviderId | string { + const normalized = name.toLowerCase(); + return providerAliasMap[normalized] ?? name; +} + +export function listProviderRegistryEntries(): readonly ProviderRegistryEntry[] { + return providerRegistry; +} + +export function findProviderRegistryEntry(name: string | null | undefined): ProviderRegistryEntry | undefined { + if (!name) return undefined; + const normalized = normalizeProviderName(name); + return providerRegistry.find((entry) => entry.id === normalized); +} + +export function getProviderRegistryEntry(name: string): ProviderRegistryEntry { + const entry = findProviderRegistryEntry(name); + if (entry) return entry; + throw new Error(`Unknown provider: ${name}. Valid: ${providerIds.join(', ')}`); +} + +export function resolveProviderCommand(name: string): { + readonly command: string; + readonly args: readonly string[]; +} { + const entry = getProviderRegistryEntry(name); + if (entry.command.kind === 'configured-claude') { + return resolveClaudeCommand(); + } + return { + command: entry.command.command, + args: entry.command.args, + }; +} + +export function supportsProviderCapability( + name: string, + capability: keyof ProviderCapabilities +): boolean { + return getProviderRegistryEntry(name).capabilities[capability] === true; +} diff --git a/src/agent-cli-provider/single-agent-runtime.ts b/src/agent-cli-provider/single-agent-runtime.ts index cb92db99..fe0588d9 100644 --- a/src/agent-cli-provider/single-agent-runtime.ts +++ b/src/agent-cli-provider/single-agent-runtime.ts @@ -1,15 +1,23 @@ import { getProviderAdapter } from './adapters'; +import { normalizeGatewayBuildOptions, resolveGatewayConfiguration } from './gateway-tools'; import { isRecord } from './json'; +import { + getProviderRegistryEntry, + resolveProviderCommand, + supportsProviderCapability, +} from './provider-registry'; import type { BuildProviderCommandOptions, CliFeatureOverrides, CommandSpec, + GatewayBuildOptions, LevelOverrides, ModelLevel, ModelSpec, ProviderAdapter, ProviderCliFeatures, ProviderId, + ResolvedGatewayBuildOptions, ReasoningEffort, } from './types'; @@ -23,6 +31,7 @@ interface CommandParts { interface RuntimeProviderSettings { readonly defaultLevel?: ModelLevel; readonly levelOverrides: LevelOverrides; + readonly gateway?: GatewayBuildOptions; } export interface SingleAgentProviderCommandInput { @@ -38,6 +47,13 @@ export interface PreparedSingleAgentProviderCommand { readonly cliFeatures: CliFeatureOverrides; } +export interface RuntimeProviderProbe { + readonly available: boolean; + readonly helpText: string; + readonly versionText: string; + readonly capabilities: ProviderCliFeatures; +} + type MutableModelSpec = { level?: ModelLevel; model?: string | null; @@ -46,24 +62,29 @@ type MutableModelSpec = { const MODEL_LEVELS: readonly ModelLevel[] = ['level1', 'level2', 'level3']; const REASONING_EFFORTS: readonly ReasoningEffort[] = ['low', 'medium', 'high', 'xhigh']; - const settingsModule: unknown = require('../../lib/settings'); const providerDetectionModule: unknown = require('../../lib/provider-detection'); const claudeAuthModule: unknown = require('../../lib/settings/claude-auth'); const loadSettingsFn = moduleFunction(settingsModule, 'loadSettings'); const getClaudeCommandFn = moduleFunction(settingsModule, 'getClaudeCommand'); +const commandExistsFn = moduleFunction(providerDetectionModule, 'commandExists'); const getHelpOutputFn = moduleFunction(providerDetectionModule, 'getHelpOutput'); +const getVersionOutputFn = moduleFunction(providerDetectionModule, 'getVersionOutput'); const resolveClaudeAuthFn = moduleFunction(claudeAuthModule, 'resolveClaudeAuth'); export function prepareSingleAgentProviderCommand( input: SingleAgentProviderCommandInput ): PreparedSingleAgentProviderCommand { + const baseOptions = input.options ?? {}; const settings = loadRuntimeSettings(); const adapter = adapterForRuntimeInput(input.provider, settings); - const providerSettings = runtimeProviderSettings(settings, adapter.id); - const baseOptions = input.options ?? {}; - const cliFeatures = baseOptions.cliFeatures ?? detectRuntimeProviderCliFeatures(adapter.id); + const providerSettings = runtimeProviderSettings( + settings, + adapter.id, + baseOptions.cwd ?? process.cwd() + ); + const cliFeatures = resolveRuntimeCliFeatures(adapter.id, baseOptions.cliFeatures); const authEnv = baseOptions.authEnv ?? resolveRuntimeAuthEnv(adapter.id, settings); const options = buildRuntimeOptions(baseOptions, adapter, providerSettings, cliFeatures, authEnv); return { @@ -75,10 +96,81 @@ export function prepareSingleAgentProviderCommand( } export function detectRuntimeProviderCliFeatures(provider: string): ProviderCliFeatures { + return probeRuntimeProviderCli(provider).capabilities; +} + +function resolveRuntimeCliFeatures( + provider: ProviderId, + overrides: CliFeatureOverrides | undefined +): CliFeatureOverrides { + if (provider === 'gateway') { + return { + ...detectRuntimeProviderCliFeatures(provider), + ...overrides, + supportsBundledRunner: true, + }; + } + if (getProviderRegistryEntry(provider).invoke.lane !== 'acp-stdio') { + return overrides ?? detectRuntimeProviderCliFeatures(provider); + } + + const detected = detectRuntimeProviderCliFeatures(provider); + if (overrides === undefined) return detected; + return mergeAcpFailClosedCliFeatures(detected, overrides); +} + +function mergeAcpFailClosedCliFeatures( + detected: ProviderCliFeatures, + overrides: CliFeatureOverrides +): CliFeatureOverrides { + if (!('supportsAcpStdio' in detected)) return overrides; + return { + ...detected, + ...overrides, + supportsAcpStdio: detected.supportsAcpStdio && overrides.supportsAcpStdio !== false, + supportsPromptImages: + detected.supportsPromptImages && overrides.supportsPromptImages !== false, + supportsLoadSession: detected.supportsLoadSession && overrides.supportsLoadSession !== false, + supportsSessionCancel: + detected.supportsSessionCancel && overrides.supportsSessionCancel !== false, + supportsSessionSetModel: + detected.supportsSessionSetModel && overrides.supportsSessionSetModel !== false, + supportsSessionSetMode: + detected.supportsSessionSetMode && overrides.supportsSessionSetMode !== false, + supportsRemoteTransport: false, + supportsCustomTransport: false, + supportsPermissionRequests: false, + supportsFsTools: false, + supportsTerminalTools: false, + }; +} + +export function probeRuntimeProviderCli(provider: string): RuntimeProviderProbe { const adapter = getProviderAdapter(provider); + if (adapter.id === 'gateway') { + return probeGatewayProvider(adapter); + } const helpCommand = runtimeHelpCommand(adapter.id); - const helpText = stringResult(getHelpOutputFn(helpCommand.command, helpCommand.args)); - return adapter.detectCliFeatures(helpText); + const commandAvailable = booleanResult(commandExistsFn(helpCommand.command)); + if (!commandAvailable) { + return { + available: false, + helpText: '', + versionText: '', + capabilities: adapter.detectCliFeatures(''), + }; + } + + const helpText = stringResult(getHelpOutputFn(helpCommand.command, helpCommand.args)).trim(); + const versionText = stringResult(getVersionOutputFn(helpCommand.command, helpCommand.args)).trim(); + const availabilityProbe = getProviderRegistryEntry(adapter.id).availabilityProbe ?? 'command'; + + return { + available: availabilityProbe === 'help-or-version' ? Boolean(helpText || versionText) : true, + helpText, + versionText, + capabilities: adapter.detectCliFeatures(helpText), + }; } function buildRuntimeOptions( @@ -88,15 +180,63 @@ function buildRuntimeOptions( cliFeatures: CliFeatureOverrides, authEnv: Readonly> ): BuildProviderCommandOptions { + const modelSpec = resolveRuntimeModelSpec(adapter, baseOptions.modelSpec, providerSettings); + const gateway = resolveRuntimeGatewayOptions( + adapter.id, + baseOptions, + providerSettings, + modelSpec + ); const resolved = { ...baseOptions, - modelSpec: resolveRuntimeModelSpec(adapter, baseOptions.modelSpec, providerSettings), + modelSpec, + ...(gateway === undefined ? {} : { gateway }), cliFeatures, }; + if (baseOptions.jsonSchema && !supportsProviderCapability(adapter.id, 'jsonSchema')) { + if (!shouldIncludeAuthEnv(baseOptions, authEnv)) { + return { ...resolved, strictSchema: false }; + } + return { ...resolved, authEnv, strictSchema: false }; + } if (!shouldIncludeAuthEnv(baseOptions, authEnv)) return resolved; return { ...resolved, authEnv }; } +function resolveRuntimeGatewayOptions( + provider: ProviderId, + baseOptions: BuildProviderCommandOptions, + providerSettings: RuntimeProviderSettings, + modelSpec: ModelSpec +): ResolvedGatewayBuildOptions | undefined { + if (provider !== 'gateway') return undefined; + const cwd = baseOptions.cwd ?? process.cwd(); + const settingsGateway = providerSettings.gateway ?? {}; + const requestGateway = baseOptions.gateway ?? {}; + const mergedHeaders = + requestGateway.headers === undefined + ? settingsGateway.headers + : { ...(settingsGateway.headers ?? {}), ...requestGateway.headers }; + const mergedGateway: GatewayBuildOptions = { + ...(requestGateway.baseUrl ?? settingsGateway.baseUrl + ? { baseUrl: requestGateway.baseUrl ?? settingsGateway.baseUrl } + : {}), + ...(requestGateway.apiKey ?? settingsGateway.apiKey + ? { apiKey: requestGateway.apiKey ?? settingsGateway.apiKey } + : {}), + ...(mergedHeaders === undefined ? {} : { headers: mergedHeaders }), + model: requestGateway.model ?? modelSpec.model ?? settingsGateway.model ?? null, + ...(requestGateway.toolPolicy ?? settingsGateway.toolPolicy + ? { toolPolicy: requestGateway.toolPolicy ?? settingsGateway.toolPolicy } + : {}), + }; + return resolveGatewayConfiguration( + mergedGateway, + 'options.gateway', + cwd + ); +} + function shouldIncludeAuthEnv( baseOptions: BuildProviderCommandOptions, authEnv: Readonly> @@ -152,7 +292,8 @@ function adapterForRuntimeInput( function runtimeProviderSettings( settings: Record, - provider: ProviderId + provider: ProviderId, + cwd: string ): RuntimeProviderSettings { const allSettings = optionalRecord(settings.providerSettings, 'settings.providerSettings'); const providerValue = allSettings?.[provider]; @@ -166,19 +307,47 @@ function runtimeProviderSettings( providerSettings.levelOverrides, `settings.providerSettings.${provider}.levelOverrides` ); - if (defaultLevel === undefined) return { levelOverrides }; - return { defaultLevel, levelOverrides }; + const gateway = + provider === 'gateway' + ? normalizeGatewayBuildOptions( + providerSettings, + 'settings.providerSettings.gateway', + cwd + ) + : undefined; + if (defaultLevel === undefined) { + return gateway === undefined ? { levelOverrides } : { levelOverrides, gateway }; + } + return gateway === undefined ? { defaultLevel, levelOverrides } : { defaultLevel, levelOverrides, gateway }; } function runtimeHelpCommand(provider: ProviderId): CommandParts { - if (provider === 'claude') return getClaudeRuntimeCommand(); - if (provider === 'codex') return { command: 'codex', args: ['exec'] }; - if (provider === 'opencode') return { command: 'opencode', args: ['run'] }; - return { command: 'gemini', args: [] }; -} - -function getClaudeRuntimeCommand(): CommandParts { - return commandPartsFromUnknown(getClaudeCommandFn(), 'getClaudeCommand'); + if (provider === 'claude') { + return commandPartsFromUnknown(getClaudeCommandFn(), 'getClaudeCommand'); + } + return resolveProviderCommand(provider); +} + +function probeGatewayProvider(adapter: ProviderAdapter): RuntimeProviderProbe { + const capabilities = adapter.detectCliFeatures(''); + try { + const settings = loadRuntimeSettings(); + const providerSettings = runtimeProviderSettings(settings, 'gateway', process.cwd()); + resolveGatewayConfiguration(providerSettings.gateway, 'settings.providerSettings.gateway', process.cwd()); + return { + available: true, + helpText: 'Bundled gateway runner', + versionText: process.version, + capabilities, + }; + } catch { + return { + available: false, + helpText: 'Bundled gateway runner', + versionText: process.version, + capabilities, + }; + } } function loadRuntimeSettings(): Record { @@ -309,3 +478,8 @@ function stringResult(value: unknown): string { if (typeof value === 'string') return value; throw new Error('Provider help output must be a string.'); } + +function booleanResult(value: unknown): boolean { + if (typeof value === 'boolean') return value; + throw new Error('Provider availability probe must return a boolean.'); +} diff --git a/src/agent-cli-provider/types.ts b/src/agent-cli-provider/types.ts index ef2ca433..cfe13e08 100644 --- a/src/agent-cli-provider/types.ts +++ b/src/agent-cli-provider/types.ts @@ -1,9 +1,32 @@ -export type ProviderId = 'claude' | 'codex' | 'gemini' | 'opencode'; -export type ProviderAlias = 'anthropic' | 'openai' | 'google'; +import type { + providerAliases, + providerIds, + ProviderCapabilities, + ProviderCapabilityState, + ProviderCommandSpec, + ProviderDockerMetadata, + ProviderDockerMountPreset, + ProviderDocsMetadata, + ProviderInvokeSpec, + ProviderRegistryEntry, +} from './provider-registry'; + +export type ProviderId = (typeof providerIds)[number]; +export type ProviderAlias = (typeof providerAliases)[number]; export type KnownProviderName = ProviderId | ProviderAlias; export type ModelLevel = 'level1' | 'level2' | 'level3'; export type ReasoningEffort = 'low' | 'medium' | 'high' | 'xhigh'; export type OutputFormat = 'text' | 'json' | 'stream-json'; +export type { + ProviderCapabilities, + ProviderCapabilityState, + ProviderCommandSpec, + ProviderDockerMetadata, + ProviderDockerMountPreset, + ProviderDocsMetadata, + ProviderInvokeSpec, + ProviderRegistryEntry, +}; export interface AgentCliProviderHelperMetadata { readonly packageName: '@the-open-engine/zeroshot'; @@ -36,6 +59,28 @@ export interface ResolvedModelSpec { export type LevelOverrides = Readonly>>; +export interface GatewayToolPolicy { + readonly roots: readonly string[]; + readonly commands: readonly string[]; + readonly commandTimeoutMs?: number; +} + +export interface GatewayBuildOptions { + readonly baseUrl?: string; + readonly apiKey?: string; + readonly headers?: Readonly>; + readonly model?: string | null; + readonly toolPolicy?: GatewayToolPolicy; +} + +export interface ResolvedGatewayBuildOptions { + readonly baseUrl: string; + readonly apiKey: string; + readonly headers: Readonly>; + readonly model: string; + readonly toolPolicy: GatewayToolPolicy; +} + export interface BaseCliFeatures { readonly provider?: ProviderId; readonly unknown?: boolean; @@ -76,15 +121,62 @@ export interface OpencodeCliFeatures extends BaseCliFeatures { readonly supportsJson: boolean; readonly supportsModel: boolean; readonly supportsVariant: boolean; + readonly supportsDir: boolean; readonly supportsCwd: boolean; readonly supportsAutoApprove: false; } +export interface PiCliFeatures extends BaseCliFeatures { + readonly provider: 'pi'; + readonly supportsJsonMode: boolean; + readonly supportsModel: boolean; + readonly supportsNoSession: boolean; + readonly supportsNoExtensions: boolean; + readonly supportsNoSkills: boolean; + readonly supportsNoPromptTemplates: boolean; + readonly supportsNoContextFiles: boolean; + readonly supportsNoApprove: boolean; +} + +export interface CopilotCliFeatures extends BaseCliFeatures { + readonly provider: 'copilot'; + readonly supportsJsonOutput: boolean; + readonly supportsModel: boolean; + readonly supportsAllowAll: boolean; + readonly supportsNoAskUser: boolean; + readonly supportsAddDir: boolean; + readonly supportsMcpConfig: boolean; +} + +export interface GatewayCliFeatures extends BaseCliFeatures { + readonly provider: 'gateway'; + readonly supportsBundledRunner: true; +} + +export interface AcpCliFeatures extends BaseCliFeatures { + readonly provider: ProviderId; + readonly supportsAcpStdio: boolean; + readonly supportsPromptImages: boolean; + readonly supportsLoadSession: boolean; + readonly supportsSessionCancel: boolean; + readonly supportsSessionSetModel: boolean; + readonly supportsSessionSetMode: boolean; + readonly supportsRemoteTransport: false; + readonly supportsCustomTransport: false; + readonly supportsPermissionRequests: false; + readonly supportsFsTools: false; + readonly supportsTerminalTools: false; +} + export type ProviderCliFeatures = | ClaudeCliFeatures | CodexCliFeatures | GeminiCliFeatures - | OpencodeCliFeatures; + | OpencodeCliFeatures + | PiCliFeatures + | CopilotCliFeatures + | GatewayCliFeatures + | AcpCliFeatures; export interface CliFeatureOverrides { readonly supportsOutputFormat?: boolean; @@ -96,10 +188,35 @@ export interface CliFeatureOverrides { readonly supportsModel?: boolean; readonly supportsJson?: boolean; readonly supportsOutputSchema?: boolean; + readonly supportsDir?: boolean; readonly supportsCwd?: boolean; readonly supportsConfigOverride?: boolean; readonly supportsSkipGitRepoCheck?: boolean; readonly supportsVariant?: boolean; + readonly supportsJsonMode?: boolean; + readonly supportsNoSession?: boolean; + readonly supportsNoExtensions?: boolean; + readonly supportsNoSkills?: boolean; + readonly supportsNoPromptTemplates?: boolean; + readonly supportsNoContextFiles?: boolean; + readonly supportsNoApprove?: boolean; + readonly supportsJsonOutput?: boolean; + readonly supportsAllowAll?: boolean; + readonly supportsNoAskUser?: boolean; + readonly supportsAddDir?: boolean; + readonly supportsMcpConfig?: boolean; + readonly supportsBundledRunner?: boolean; + readonly supportsAcpStdio?: boolean; + readonly supportsPromptImages?: boolean; + readonly supportsLoadSession?: boolean; + readonly supportsSessionCancel?: boolean; + readonly supportsSessionSetModel?: boolean; + readonly supportsSessionSetMode?: boolean; + readonly supportsRemoteTransport?: false; + readonly supportsCustomTransport?: false; + readonly supportsPermissionRequests?: false; + readonly supportsFsTools?: false; + readonly supportsTerminalTools?: false; readonly unknown?: boolean; } @@ -144,6 +261,12 @@ export interface BuildProviderCommandOptions { readonly cliFeatures?: CliFeatureOverrides; readonly authEnv?: Readonly>; readonly strictSchema?: boolean; + readonly gateway?: GatewayBuildOptions; + // MCP server configs forwarded to providers that accept an MCP config CLI flag (currently + // Copilot's `--additional-mcp-config`). Each entry is an inline JSON string (the standard + // `{"mcpServers": {...}}` envelope) or an `@` file reference; adapters emit one flag per + // entry. Providers whose adapter models no MCP flag ignore this field. + readonly mcpConfig?: readonly string[]; } export interface TextEvent { @@ -189,8 +312,26 @@ export type OutputEvent = TextEvent | ThinkingEvent | ToolCallEvent | ToolResult export type ProviderParseResult = OutputEvent | readonly OutputEvent[] | null; export interface ProviderParserState { - provider: ProviderId; + readonly provider: ProviderId; lastToolId: string | null | undefined; + lastAssistantText?: string; + lastAssistantThinking?: string; + messagePhaseById?: Map; + assistantTextByMessageId?: Map; + assistantThinkingByMessageId?: Map; + toolCalls?: Map< + string, + { + name: string | null | undefined; + input: unknown; + } + >; + usage?: { + inputTokens?: number; + outputTokens?: number; + cacheReadInputTokens?: number; + cacheCreationInputTokens?: number; + } | null; } export type ErrorClassificationKind = diff --git a/src/agent/agent-lifecycle.js b/src/agent/agent-lifecycle.js index d7df9422..820101b4 100644 --- a/src/agent/agent-lifecycle.js +++ b/src/agent/agent-lifecycle.js @@ -61,10 +61,6 @@ async function createValidatorIsolation(agent, isolationConfig) { const cluster = agent.cluster || {}; const workDir = agent.config?.cwd || cluster.worktree?.path || cluster.cwd || process.cwd(); - const image = isolationConfig.image; - await IsolationManager.ensureImage(image); - - const manager = new IsolationManager({ image }); const providerName = normalizeProviderName( (agent._resolveProvider && agent._resolveProvider()) || cluster.config?.forceProvider || @@ -72,6 +68,11 @@ async function createValidatorIsolation(agent, isolationConfig) { loadSettings().defaultProvider || 'claude' ); + // Run validators on the provider's image variant (installs its CLI as a Docker-cached layer). + const image = IsolationManager.imageForProvider(providerName, isolationConfig.image); + await IsolationManager.ensureImage(image, true, IsolationManager.providerBuildArgs(providerName)); + + const manager = new IsolationManager({ image }); const isolationClusterId = `${cluster.id}-validators`; const containerId = await manager.createContainer(isolationClusterId, { @@ -200,6 +201,8 @@ function start(agent) { * @returns {Promise} */ async function stop(agent) { + stopLivenessCheck(agent); + if (!agent.running) { return; } diff --git a/src/agent/agent-quality-gate-schema.js b/src/agent/agent-quality-gate-schema.js index 094e70b5..b64e0a85 100644 --- a/src/agent/agent-quality-gate-schema.js +++ b/src/agent/agent-quality-gate-schema.js @@ -41,11 +41,11 @@ function buildQualityGateSchema() { required: ['command', 'exitCode', 'output'], }, completedAt: { - type: ['string', 'number'], + anyOf: [{ type: 'string' }, { type: 'number' }], description: 'When the gate completed, as an ISO string or numeric timestamp.', }, timestamp: { - type: ['string', 'number'], + anyOf: [{ type: 'string' }, { type: 'number' }], description: 'Alternate completion timestamp, as an ISO string or numeric timestamp.', }, stale: { diff --git a/src/agent/agent-task-executor.js b/src/agent/agent-task-executor.js index 91dfd4b3..d05ae5cf 100644 --- a/src/agent/agent-task-executor.js +++ b/src/agent/agent-task-executor.js @@ -14,12 +14,15 @@ const { spawn, spawnSync } = require('child_process'); const path = require('path'); const fs = require('fs'); const os = require('os'); -const { parseProviderChunk } = require('../providers'); +const { parseProviderChunk, getProvider } = require('../providers'); const { getTask } = require('../../task-lib/store.js'); const { loadSettings } = require('../../lib/settings.js'); const { resolveClaudeAuth } = require('../../lib/settings/claude-auth.js'); const { prependWorktreeToolBinToEnv } = require('../worktree-tooling-env.js'); -const { prepareClaudeConfigDir } = require('../worktree-claude-config.js'); +const { + prepareClaudeConfigDir, + resolveRepoMcpConfigPath, +} = require('../worktree-claude-config.js'); const { buildRawLogOnlyMetadata } = require('./context-replay-policy'); function runCommandWithTimeout(command, args, options = {}, callback = null) { @@ -733,9 +736,45 @@ function buildTaskRunArgs({ agent, providerName, modelSpec, runOutputFormat }) { args.push('--json-schema', schema); } + // MCP servers: providers whose CLI accepts an MCP config flag (e.g. Copilot's + // --additional-mcp-config) cannot use the Claude config-dir overlay, so forward the repo's + // `.mcp.json` (the same MCP source Claude consumes) inline via `--mcp-config`. + for (const mcpArg of resolveMcpConfigArgs(agent, providerName)) { + args.push(mcpArg); + } + return args; } +/** + * Build the `--mcp-config` args for a task-run invocation, or [] when they don't apply. + * + * Only providers whose adapter models an MCP config CLI flag receive it — Claude consumes MCP via + * the config-dir `.mcp.json` overlay (see prepareClaudeConfigDir) and needs no flag. The repo + * `.mcp.json` content is inlined (not passed as an @ reference) so the identical value works + * under local, worktree, and Docker isolation without host/container path translation. + */ +function resolveMcpConfigArgs(agent, providerName) { + if (!providerModelsMcpConfigFlag(providerName)) return []; + + const mcpPath = resolveRepoMcpConfigPath({ + cwd: agent.config?.cwd || process.cwd(), + worktreePath: agent.worktree?.path || null, + }); + if (!mcpPath) return []; + + const content = fs.readFileSync(mcpPath, 'utf8').trim(); + if (content.length === 0) return []; + + return ['--mcp-config', content]; +} + +/** True when the provider's adapter models an MCP config CLI flag (currently only Copilot). */ +function providerModelsMcpConfigFlag(providerName) { + const adapter = getProvider(providerName).adapter; + return 'supportsMcpConfig' in adapter.detectCliFeatures(''); +} + function maybeLogStreamJsonNotice(agent, runOutputFormat) { if (agent.config.jsonSchema && runOutputFormat !== 'json' && !agent.quiet) { agent._log( diff --git a/src/config-validator.js b/src/config-validator.js index b53ab5d8..a2d7c86c 100644 --- a/src/config-validator.js +++ b/src/config-validator.js @@ -12,7 +12,11 @@ */ const { loadSettings } = require('../lib/settings'); -const { VALID_PROVIDERS, normalizeProviderName } = require('../lib/provider-names'); +const { + VALID_PROVIDERS, + normalizeProviderName, + providerSupportsCapability, +} = require('../lib/provider-names'); const { getProvider } = require('./providers'); const { CAPABILITIES } = require('./providers/capabilities'); const { GUIDANCE_TOPICS } = require('./guidance-topics'); @@ -2026,6 +2030,10 @@ function validateProviderSettings(provider, providerSettings) { const providerModule = getProvider(provider); const levels = providerModule.getLevelMapping(); const settings = providerSettings || {}; + const providerError = providerModule.validateSettings(settings); + if (providerError) { + throw new Error(providerError); + } const minLevel = settings.minLevel || providerModule.getDefaultMinLevel?.(); const maxLevel = settings.maxLevel || providerModule.getDefaultMaxLevel?.(); @@ -2042,7 +2050,10 @@ function validateProviderSettings(provider, providerSettings) { `Invalid model override (must be non-empty string) for provider "${provider}"` ); } - if (override?.reasoningEffort && !['codex', 'opencode'].includes(provider)) { + if (override?.model) { + providerModule.validateModelId(override.model); + } + if (override?.reasoningEffort && !providerSupportsCapability(provider, 'reasoningEffort')) { throw new Error(`reasoningEffort overrides are only supported for Codex and Opencode`); } if ( @@ -2068,7 +2079,6 @@ function resolveAgentProvider(agent, config, settings, errors) { function buildProviderContext(provider, settings) { const providerModule = getProvider(provider); const levels = providerModule.getLevelMapping(); - const catalog = providerModule.getModelCatalog(); const providerSettings = settings.providerSettings?.[provider] || {}; const minLevel = providerSettings.minLevel; const maxLevel = providerSettings.maxLevel; @@ -2077,7 +2087,6 @@ function buildProviderContext(provider, settings) { return { providerModule, levels, - catalog, providerSettings, minLevel, maxLevel, @@ -2109,10 +2118,12 @@ function validateModelLevelSupport(agent, provider, levels, warnings) { } function validateModelSelection(agent, context, warnings) { - const { provider, catalog, minLevel, maxLevel, rank } = context; + const { provider, providerModule, minLevel, maxLevel, rank } = context; if (agent.model) { - if (!catalog[agent.model]) { + try { + providerModule.validateModelId(agent.model); + } catch { warnings.push( `Agent "${agent.id}" uses model "${agent.model}" which is not valid for ${provider}` ); @@ -2137,7 +2148,7 @@ function validateModelSelection(agent, context, warnings) { } } -function validateModelRulesSupport(agent, provider, catalog, levels, warnings) { +function validateModelRulesSupport(agent, provider, providerModule, levels, warnings) { if (!agent.modelRules || !Array.isArray(agent.modelRules)) return; for (const rule of agent.modelRules) { @@ -2146,16 +2157,20 @@ function validateModelRulesSupport(agent, provider, catalog, levels, warnings) { `Agent "${agent.id}" uses modelLevel "${rule.modelLevel}" in modelRules which is not valid for ${provider}` ); } - if (rule.model && !catalog[rule.model]) { - warnings.push( - `Agent "${agent.id}" uses model "${rule.model}" in modelRules which is not valid for ${provider}` - ); + if (rule.model) { + try { + providerModule.validateModelId(rule.model); + } catch { + warnings.push( + `Agent "${agent.id}" uses model "${rule.model}" in modelRules which is not valid for ${provider}` + ); + } } } } function validateReasoningEffortSupport(agent, provider, warnings) { - if (agent.reasoningEffort && !['codex', 'opencode'].includes(provider)) { + if (agent.reasoningEffort && !providerSupportsCapability(provider, 'reasoningEffort')) { warnings.push(`Agent "${agent.id}" sets reasoningEffort but ${provider} does not support it`); } else if ( agent.reasoningEffort && @@ -2193,10 +2208,13 @@ function validateProviderFeatures(config, settings) { const provider = resolveAgentProvider(agent, config, settings, errors); if (!provider) continue; - const { levels, catalog, minLevel, maxLevel, rank } = buildProviderContext(provider, settings); + const { providerModule, levels, minLevel, maxLevel, rank } = buildProviderContext( + provider, + settings + ); const modelSelectionContext = { provider, - catalog, + providerModule, minLevel, maxLevel, rank, @@ -2205,7 +2223,7 @@ function validateProviderFeatures(config, settings) { validateJsonSchemaSupport(agent, provider, warnings); validateModelLevelSupport(agent, provider, levels, warnings); validateModelSelection(agent, modelSelectionContext, warnings); - validateModelRulesSupport(agent, provider, catalog, levels, warnings); + validateModelRulesSupport(agent, provider, providerModule, levels, warnings); validateReasoningEffortSupport(agent, provider, warnings); } diff --git a/src/isolation-manager.js b/src/isolation-manager.js index 9926213d..7a88df75 100644 --- a/src/isolation-manager.js +++ b/src/isolation-manager.js @@ -16,8 +16,13 @@ const os = require('os'); const fs = require('fs'); const { loadSettings } = require('../lib/settings'); const { CLAUDE_AUTH_ENV_VARS, resolveClaudeAuth } = require('../lib/settings/claude-auth'); -const { normalizeProviderName } = require('../lib/provider-names'); -const { resolveMounts, resolveEnvs, expandEnvPatterns } = require('../lib/docker-config'); +const { normalizeProviderName, getProviderMetadata } = require('../lib/provider-names'); +const { + MOUNT_PRESETS, + resolveMounts, + resolveEnvs, + expandEnvPatterns, +} = require('../lib/docker-config'); const { getProvider } = require('./providers'); const { readRepoSettings } = require('../lib/repo-settings'); const { provisionClaudeCredentials } = require('./claude-credentials'); @@ -98,6 +103,24 @@ function resolveWorktreeSetupTimeoutMs(repoSettings = {}, options = {}) { const DEFAULT_IMAGE = 'zeroshot-cluster-base'; +/** + * Shell command that installs a provider's CLI inside the cluster image, or null when the + * provider is baked into the base image (e.g. Claude) or has no single-command installer. + * Sourced from the provider registry (docker.install) so nothing here is provider-specific. + * @param {string} providerName + * @returns {string|null} + */ +function providerDockerInstall(providerName) { + if (!providerName) return null; + try { + const metadata = getProviderMetadata(providerName); + const install = metadata && metadata.docker && metadata.docker.install; + return typeof install === 'string' && install.trim() ? install.trim() : null; + } catch { + return null; + } +} + class IsolationManager { constructor(options = {}) { this.image = options.image || DEFAULT_IMAGE; @@ -171,7 +194,13 @@ class IsolationManager { clusterConfigDir, }); - const mountedHosts = this._applyCredentialMounts(args, config, settings, containerHome); + const mountedHosts = this._applyCredentialMounts( + args, + config, + settings, + containerHome, + providerName + ); this._warnMissingProviderCredentials(providerName, mountedHosts, config, containerHome); args.push('-w', '/workspace', image, 'tail', '-f', '/dev/null'); @@ -285,13 +314,25 @@ class IsolationManager { return settings.dockerMounts; } - _applyCredentialMounts(args, config, settings, containerHome) { + // Auto-activate the running provider's own credential preset (mount + env) so `--docker` works + // without listing it in dockerMounts. Claude is mounted separately, so skip it. + _withActiveProviderPreset(mountConfig, providerName) { + if (!providerName || providerName === 'claude') return mountConfig; + if (!MOUNT_PRESETS[providerName]) return mountConfig; + if (mountConfig.some((item) => item === providerName)) return mountConfig; + return [...mountConfig, providerName]; + } + + _applyCredentialMounts(args, config, settings, containerHome, providerName) { const mountedHosts = []; if (config.noMounts) { return mountedHosts; } - const mountConfig = this._resolveMountConfig(config, settings); + const mountConfig = this._withActiveProviderPreset( + this._resolveMountConfig(config, settings), + providerName + ); const mounts = resolveMounts(mountConfig, { containerHome }); const claudeContainerPath = path.posix.join(containerHome, '.claude'); @@ -363,16 +404,40 @@ class IsolationManager { return; } + const metadata = getProviderMetadata(providerName); const provider = getProvider(providerName); + + // An env token (e.g. COPILOT_GITHUB_TOKEN) is a complete credential on its own. + const credentialEnvKeys = metadata.credentialEnvKeys || []; + if (credentialEnvKeys.some((key) => process.env[key])) { + return; + } + + // A mount only counts if it carries the secret (credentialInMount !== false). + const credentialInMount = metadata.docker && metadata.docker.credentialInMount === false; const credentialPaths = provider.getCredentialPaths ? provider.getCredentialPaths() : []; const expandedCreds = credentialPaths.map((cred) => expandHomePath(cred)); - const hasCredentialMount = mountedHosts.some((hostPath) => - expandedCreds.some( - (credPath) => pathContains(hostPath, credPath) || pathContains(credPath, hostPath) - ) - ); + const hasCredentialMount = + !credentialInMount && + mountedHosts.some((hostPath) => + expandedCreds.some( + (credPath) => pathContains(hostPath, credPath) || pathContains(credPath, hostPath) + ) + ); + if (hasCredentialMount) { + return; + } + + if (credentialInMount && credentialEnvKeys.length > 0) { + console.warn( + `[IsolationManager] ⚠️ ${provider.displayName} could not find credentials for Docker. ` + + `Its login token is not stored in a mountable file — export one of ` + + `${credentialEnvKeys.join(', ')} before running with --docker.` + ); + return; + } - if (!hasCredentialMount && expandedCreds.length > 0) { + if (expandedCreds.length > 0) { const exampleHost = credentialPaths[0]; const exampleContainer = exampleHost.replace(/^~(?=\/|$)/, containerHome); const mountNote = config.noMounts ? 'Credential mounts are disabled. ' : ''; @@ -1230,13 +1295,40 @@ class IsolationManager { } } + /** + * Resolve the cluster image tag for a provider. Providers baked into the base image (e.g. + * Claude) run on the base image directly; providers with a `docker.install` command get a + * per-provider image variant `-` whose install step is a Docker-cached + * layer (built once, reused thereafter). + * @param {string} providerName + * @param {string} [baseImage] + * @returns {string} + */ + static imageForProvider(providerName, baseImage = DEFAULT_IMAGE) { + if (!providerDockerInstall(providerName)) { + return baseImage; + } + return `${baseImage}-${normalizeProviderName(providerName)}`; + } + + /** + * Docker `--build-arg` values that install a provider's CLI into its image variant, or [] when + * the provider is baked into the base image or has no installer. + * @param {string} providerName + * @returns {string[]} + */ + static providerBuildArgs(providerName) { + const install = providerDockerInstall(providerName); + return install ? [`PROVIDER_INSTALL=${install}`] : []; + } + /** * Build the Docker image with retry logic * @param {string} [image] - Image name to build * @param {number} [maxRetries=3] - Maximum retry attempts * @returns {Promise} */ - static async buildImage(image = DEFAULT_IMAGE, maxRetries = 3) { + static async buildImage(image = DEFAULT_IMAGE, maxRetries = 3, buildArgs = []) { // Repository root is one level up from src/ const repoRoot = path.join(__dirname, '..'); const dockerfilePath = path.join(repoRoot, 'docker', 'zeroshot-cluster', 'Dockerfile'); @@ -1245,6 +1337,12 @@ class IsolationManager { throw new Error(`Dockerfile not found at ${dockerfilePath}`); } + // Each buildArg becomes a `--build-arg KEY=VALUE` pair (e.g. the per-provider install command). + const buildArgFlags = []; + for (const arg of buildArgs) { + buildArgFlags.push('--build-arg', arg); + } + console.log(`[IsolationManager] Building Docker image '${image}'...`); const baseDelay = 3000; // 3 seconds @@ -1253,11 +1351,18 @@ class IsolationManager { try { // CRITICAL: Run from repo root so build context includes package.json and src/ // Use -f flag to specify Dockerfile location - runSync('docker', ['build', '-f', 'docker/zeroshot-cluster/Dockerfile', '-t', image, '.'], { - cwd: repoRoot, - encoding: 'utf8', - stdio: 'inherit', - }); + runSync( + 'docker', + ['build', '-f', 'docker/zeroshot-cluster/Dockerfile', ...buildArgFlags, '-t', image, '.'], + { + cwd: repoRoot, + encoding: 'utf8', + stdio: 'inherit', + // No timeout: image builds legitimately take many minutes (apt, tool downloads, + // provider install). runSync's 30s default would kill every build (ETIMEDOUT). + timeout: 0, + } + ); console.log(`[IsolationManager] ✓ Image '${image}' built successfully`); return; @@ -1284,7 +1389,7 @@ class IsolationManager { * @param {boolean} [autoBuild=true] - Auto-build if missing * @returns {Promise} */ - static async ensureImage(image = DEFAULT_IMAGE, autoBuild = true) { + static async ensureImage(image = DEFAULT_IMAGE, autoBuild = true, buildArgs = []) { if (this.imageExists(image)) { return; } @@ -1297,7 +1402,7 @@ class IsolationManager { } console.log(`[IsolationManager] Image '${image}' not found, building automatically...`); - await this.buildImage(image); + await this.buildImage(image, 3, buildArgs); } /** diff --git a/src/ledger.js b/src/ledger.js index 5afa6b62..32fe9b30 100644 --- a/src/ledger.js +++ b/src/ledger.js @@ -715,6 +715,9 @@ class Ledger extends EventEmitter { * Close the database connection */ close() { + if (this._closed) { + return; + } this._closed = true; // Set flag BEFORE closing to prevent race conditions this.db.close(); } diff --git a/src/message-bus.js b/src/message-bus.js index 15bb2663..5331efda 100644 --- a/src/message-bus.js +++ b/src/message-bus.js @@ -24,12 +24,14 @@ class MessageBus extends EventEmitter { this.setMaxListeners(MAX_LISTENERS); this.ledger = ledger || new Ledger(); this.wsClients = new Set(); + this._closed = false; // Forward ledger events - this.ledger.on('message', (message) => { + this._forwardLedgerMessage = (message) => { this.emit('message', message); this._broadcastToWebSocket(message); - }); + }; + this.ledger.on('message', this._forwardLedgerMessage); } /** @@ -241,6 +243,14 @@ class MessageBus extends EventEmitter { * Close the message bus */ close() { + if (this._closed) { + return; + } + this._closed = true; + + this.ledger.off('message', this._forwardLedgerMessage); + this.removeAllListeners(); + // Close all WebSocket connections for (const ws of this.wsClients) { try { diff --git a/src/orchestrator.js b/src/orchestrator.js index 74951bd1..9c59338f 100644 --- a/src/orchestrator.js +++ b/src/orchestrator.js @@ -47,6 +47,7 @@ const configValidator = require('./config-validator'); const TemplateResolver = require('./template-resolver'); const { loadSettings } = require('../lib/settings'); const { normalizeProviderName } = require('../lib/provider-names'); +const { resolveRunPlan } = require('../lib/run-plan'); const { getProvider } = require('./providers'); const StateSnapshotter = require('./state-snapshotter'); const { resolveClusterRequiredQualityGates } = require('./quality-gates'); @@ -212,14 +213,11 @@ function applyPushBlockedRepairTriggers(config) { } } -function resolveAutoMerge(options) { - return Boolean(options.ship) || Boolean(options.autoMerge); -} - function buildPrOptions(options, requiredQualityGates) { // autoMerge must always be persisted (even when no other PR fields are set) so that // `zeroshot run --pr` (autoMerge=false) vs `--ship` (autoMerge=true) survives resume. - const autoMerge = resolveAutoMerge(options); + // Derived from the canonical run plan — never recomputed from ship/pr here. + const autoMerge = resolveRunPlan(options).autoMerge; return { prBase: options.prBase || null, @@ -577,6 +575,12 @@ class Orchestrator { // missing - a ledger write. Read-only orchestrators must never write, so they // skip it; they only need to read existing snapshots, not produce new ones. if (!this.readonly) { + this._registerClusterSubscriptions({ + messageBus, + clusterId, + isolationManager, + containerId: isolation?.containerId || null, + }); this._startSnapshotter(clusterContext); } this._log(`[Orchestrator] Loaded cluster: ${clusterId} with ${agents.length} agents`); @@ -1134,11 +1138,12 @@ class Orchestrator { const messageBus = new MessageBus(ledger); // Handle isolation mode (Docker container OR git worktree) - const { isolationManager, containerId, worktreeInfo } = await this._initializeIsolation( - options, - config, - clusterId - ); + const { + isolationManager, + containerId, + worktreeInfo, + image: resolvedIsolationImage, + } = await this._initializeIsolation(options, config, clusterId); let requiredQualityGates = resolveClusterRequiredQualityGates(config, options); let commandProofs = resolveClusterCommandProofs(config, options); options.requiredQualityGates = requiredQualityGates; @@ -1184,7 +1189,9 @@ class Orchestrator { ? { enabled: true, containerId, - image: options.isolationImage || 'zeroshot-cluster-base', + // Provider-specific image variant (resolved in _initializeIsolation) so resume + // recreates the container on the same image that has the provider CLI installed. + image: resolvedIsolationImage || options.isolationImage || 'zeroshot-cluster-base', manager: isolationManager, workDir: options.cwd || process.cwd(), // Persisted for resume } @@ -1725,20 +1732,30 @@ class Orchestrator { let isolationManager = null; let containerId = null; let worktreeInfo = null; + let isolationImage = null; if (options.isolation) { if (!IsolationManager.isDockerAvailable()) { throw new Error('Docker is not available. Install Docker to use --docker mode.'); } - const image = options.isolationImage || 'zeroshot-cluster-base'; - await IsolationManager.ensureImage(image); + // Resolve the provider first so the cluster runs on that provider's image variant, which + // installs the provider CLI as a Docker-cached layer. Providers baked into the base image + // (e.g. Claude) resolve back to the base image unchanged. + const providerName = this._resolveClusterProvider(config); + const baseImage = options.isolationImage || 'zeroshot-cluster-base'; + const image = IsolationManager.imageForProvider(providerName, baseImage); + await IsolationManager.ensureImage( + image, + true, + IsolationManager.providerBuildArgs(providerName) + ); + isolationImage = image; isolationManager = new IsolationManager({ image }); this._log(`[Orchestrator] Starting cluster in isolation mode (image: ${image})`); const workDir = options.cwd || process.cwd(); - const providerName = this._resolveClusterProvider(config); containerId = await isolationManager.createContainer(clusterId, { workDir, image, @@ -1766,7 +1783,7 @@ class Orchestrator { this._log(`[Orchestrator] Branch: ${worktreeInfo.branch}`); } - return { isolationManager, containerId, worktreeInfo }; + return { isolationManager, containerId, worktreeInfo, image: isolationImage }; } _applyAutoPrConfig(config, inputData, options) { @@ -1825,7 +1842,7 @@ class Orchestrator { mergeQueue: options.mergeQueue, closeIssue: options.closeIssue, requiredQualityGates: options.requiredQualityGates, - autoMerge: resolveAutoMerge(options), + autoMerge: resolveRunPlan(options).autoMerge, cwd: options.cwd, }); @@ -2316,6 +2333,9 @@ class Orchestrator { * Call before deleting storageDir to prevent ENOENT race conditions during cleanup */ close() { + if (this.closed) { + return; + } this.closed = true; } @@ -4103,7 +4123,7 @@ Continue from where you left off. Review your previous output to understand what // Exported for testing (PR options persistence, e.g. autoMerge for --pr vs --ship). Orchestrator.buildPrOptions = buildPrOptions; -Orchestrator.resolveAutoMerge = resolveAutoMerge; +Orchestrator.resolveRunPlan = resolveRunPlan; module.exports = Orchestrator; module.exports.DuplicateClusterError = DuplicateClusterError; diff --git a/src/preflight.js b/src/preflight.js index f0d257f7..2c18bf1c 100644 --- a/src/preflight.js +++ b/src/preflight.js @@ -19,7 +19,12 @@ const { resolveClaudeAuth, } = require('../lib/settings/claude-auth.js'); const { loadSettings, getClaudeCommand } = require('../lib/settings.js'); -const { normalizeProviderName } = require('../lib/provider-names'); +const { + VALID_PROVIDERS, + getProviderMetadata, + normalizeProviderName, + resolveProviderCommand, +} = require('../lib/provider-names'); const { detectGitContext } = require('../lib/git-remote-utils'); const { readKeychainCredentials } = require('./claude-credentials'); @@ -343,10 +348,21 @@ function buildClaudeCommand(options) { return options.claudeCommand || [command, ...args].join(' '); } +function buildRegistryInstallRecovery(metadata, command) { + return [ + ...metadata.installInstructions.split('\n').filter(Boolean), + `Then run: ${command} --version`, + ]; +} + function validateClaudeProvider(options) { const errors = []; const warnings = []; + const metadata = getProviderMetadata('claude'); const claudeCommand = buildClaudeCommand(options); + const claudeCommandParts = claudeCommand.trim().split(/\s+/); + const usesDefaultClaudeCommand = + claudeCommandParts.length === 1 && claudeCommandParts[0] === metadata.binary; const claude = getClaudeVersion(claudeCommand); if (!claude.installed) { @@ -354,12 +370,8 @@ function validateClaudeProvider(options) { formatError( 'Claude command not available', claude.error, - claudeCommand === 'claude' - ? [ - 'Install Claude CLI: npm install -g @anthropic-ai/claude-code', - 'Or: brew install claude (macOS)', - 'Then run: claude --version', - ] + usesDefaultClaudeCommand + ? buildRegistryInstallRecovery(metadata, metadata.binary) : [ `Command '${claudeCommand}' not found`, 'Check settings: zeroshot settings', @@ -404,41 +416,82 @@ function validateCliProvider(command, title, detail, recovery) { return { errors, warnings: [] }; } -function validateProvider(providerName, options) { - const validatorByProvider = { - claude: () => validateClaudeProvider(options), - codex: () => - validateCliProvider('codex', 'Codex CLI not available', 'Command "codex" not installed', [ - 'Install Codex CLI: npm install -g @openai/codex', - 'Then run: codex --version', - ]), - gemini: () => - validateCliProvider('gemini', 'Gemini CLI not available', 'Command "gemini" not installed', [ - 'Install Gemini CLI: npm install -g @google/gemini-cli', - 'Then run: gemini --version', - ]), - opencode: () => - validateCliProvider( - 'opencode', - 'Opencode CLI not available', - 'Command "opencode" not installed', - ['Install Opencode CLI: see https://opencode.ai', 'Then run: opencode --version'] +function validateGatewayProvider() { + const { getProvider } = require('./providers'); + const provider = getProvider('gateway'); + if (provider.isAvailable()) { + return { errors: [], warnings: [] }; + } + return { + errors: [ + formatError( + 'Gateway provider not configured', + 'providerSettings.gateway must define baseUrl, apiKey, model, and toolPolicy before gateway can run.', + [ + 'Run: zeroshot settings', + 'Set providerSettings.gateway.baseUrl to your OpenAI-compatible endpoint base URL', + 'Set providerSettings.gateway.apiKey and providerSettings.gateway.model', + 'Set providerSettings.gateway.toolPolicy.roots and toolPolicy.commands explicitly', + ] ), + ], + warnings: [], }; +} - const validator = validatorByProvider[providerName]; - if (!validator) { +function validateProvider(providerName, options) { + let metadata; + try { + metadata = getProviderMetadata(providerName); + } catch { return { errors: [ formatError('Unknown provider', `Provider "${providerName}" is not supported`, [ - 'Use claude, codex, gemini, or opencode', + `Use ${VALID_PROVIDERS.join(', ')}`, ]), ], warnings: [], }; } - return validator(); + if (metadata.command.kind === 'configured-claude') { + return validateClaudeProvider(options); + } + if (providerName === 'gateway') { + return validateGatewayProvider(); + } + + return validateRegistryCliProvider(providerName); +} + +function validateRegistryCliProvider(providerName) { + const metadata = getProviderMetadata(providerName); + const { command } = resolveProviderCommand(providerName); + const installSteps = metadata.installInstructions.split('\n').filter(Boolean); + if (!commandExists(command)) { + return validateCliProvider( + command, + `${metadata.displayName} CLI not available`, + `Command "${command}" not installed`, + [...installSteps, `Then run: ${command} --version`] + ); + } + + const { getProvider } = require('./providers'); + if (getProvider(providerName).isAvailable()) { + return { errors: [], warnings: [] }; + } + + return { + errors: [ + formatError( + `${metadata.displayName} CLI not available`, + `Command "${command}" is installed but did not produce usable --help/--version output`, + [...installSteps, `Then run: ${command} --version`] + ), + ], + warnings: [], + }; } function validateGhRequirement() { diff --git a/src/providers/capabilities.js b/src/providers/capabilities.js index ceb8fb1f..c4052fad 100644 --- a/src/providers/capabilities.js +++ b/src/providers/capabilities.js @@ -1,48 +1,21 @@ -const { normalizeProviderName } = require('../../lib/provider-names'); +const { + PROVIDER_CAPABILITIES, + normalizeProviderName, + providerSupportsCapability, +} = require('../../lib/provider-names'); -const CAPABILITIES = { - claude: { - dockerIsolation: true, - worktreeIsolation: true, - mcpServers: true, - jsonSchema: true, - streamJson: true, - thinkingMode: true, - reasoningEffort: false, - }, - codex: { - dockerIsolation: true, - worktreeIsolation: true, - mcpServers: true, - jsonSchema: true, - streamJson: true, - thinkingMode: true, - reasoningEffort: true, - }, - gemini: { - dockerIsolation: true, - worktreeIsolation: true, - mcpServers: true, - jsonSchema: 'experimental', - streamJson: true, - thinkingMode: true, - reasoningEffort: false, - }, - opencode: { - dockerIsolation: true, - worktreeIsolation: true, - mcpServers: true, - jsonSchema: 'experimental', - streamJson: true, - thinkingMode: true, - reasoningEffort: true, - }, -}; +const CAPABILITIES = Object.freeze( + Object.fromEntries( + Object.entries(PROVIDER_CAPABILITIES).map(([provider, capabilities]) => [ + provider, + Object.freeze({ ...capabilities }), + ]) + ) +); function checkCapability(provider, capability) { - const caps = CAPABILITIES[normalizeProviderName(provider)]; - if (!caps) return false; - return caps[capability] === true; + if (!provider) return false; + return providerSupportsCapability(provider, capability); } function warnIfExperimental(provider, capability) { diff --git a/src/providers/index.js b/src/providers/index.js index 73734d33..40c4a488 100644 --- a/src/providers/index.js +++ b/src/providers/index.js @@ -1,61 +1,16 @@ const BaseProvider = require('./base-provider'); -const { getClaudeCommand } = require('../../lib/settings'); -const { normalizeProviderName } = require('../../lib/provider-names'); +const { + getProviderMetadata, + normalizeProviderName, + resolveProviderCommand, +} = require('../../lib/provider-names'); const { commandExists, getCommandPath } = require('../../lib/provider-detection'); const helper = require('../../lib/agent-cli-provider'); -const PROVIDER_METADATA = { - claude: { - displayName: 'Claude', - cliCommand: 'claude', - helpArgs: () => getClaudeCommand().args, - commandParts: () => getClaudeCommand(), - installInstructions: - 'npm install -g @anthropic-ai/claude-code\nOr (macOS): brew install claude', - authInstructions: 'claude login', - credentialPaths: ['~/.claude'], - settingsFields: ['anthropicApiKey', 'bedrockApiKey', 'bedrockRegion'], - }, - codex: { - displayName: 'Codex', - cliCommand: 'codex', - helpArgs: () => ['exec'], - commandParts: () => ({ command: 'codex', args: [] }), - installInstructions: 'npm install -g @openai/codex', - authInstructions: 'codex login', - credentialPaths: ['~/.config/codex', '~/.codex'], - settingsFields: [], - }, - gemini: { - displayName: 'Gemini', - cliCommand: 'gemini', - helpArgs: () => [], - commandParts: () => ({ command: 'gemini', args: [] }), - installInstructions: 'npm install -g @google/gemini-cli', - authInstructions: 'gemini auth login', - credentialPaths: ['~/.config/gcloud', '~/.config/gemini', '~/.gemini'], - settingsFields: [], - }, - opencode: { - displayName: 'Opencode', - cliCommand: 'opencode', - helpArgs: () => ['run'], - commandParts: () => ({ command: 'opencode', args: [] }), - installInstructions: 'See https://opencode.ai for installation instructions.', - authInstructions: 'opencode auth login', - credentialPaths: ['~/.local/share/opencode'], - settingsFields: [], - }, -}; - const warned = new Set(); function metadataForProvider(name) { - const metadata = PROVIDER_METADATA[name]; - if (!metadata) { - throw new Error(`Unknown provider: ${name}. Valid: ${listProviders().join(', ')}`); - } - return metadata; + return getProviderMetadata(name); } class RuntimeProvider extends BaseProvider { @@ -65,7 +20,7 @@ class RuntimeProvider extends BaseProvider { super({ name: normalized, displayName: metadata.displayName, - cliCommand: metadata.cliCommand, + cliCommand: metadata.binary, }); this._metadata = metadata; this._adapter = helper.getProviderAdapter(normalized); @@ -78,12 +33,19 @@ class RuntimeProvider extends BaseProvider { } isAvailable() { - const { command } = this._metadata.commandParts(); - return commandExists(command); + if (this._usesBundledRunner()) { + return helper.probeRuntimeProviderCli(this.name).available; + } + const { command } = resolveProviderCommand(this.name); + if (!commandExists(command)) return false; + return helper.probeRuntimeProviderCli(this.name).available; } getCliPath() { - const { command } = this._metadata.commandParts(); + if (this._usesBundledRunner()) { + return this._adapter.binary || process.execPath; + } + const { command } = resolveProviderCommand(this.name); return getCommandPath(command) || command; } @@ -105,6 +67,10 @@ class RuntimeProvider extends BaseProvider { return this._metadata.credentialPaths; } + _usesBundledRunner() { + return this._adapter.detectCliFeatures('').supportsBundledRunner === true; + } + buildCommand(context, options = {}) { const prepared = helper.prepareSingleAgentProviderCommand({ provider: this.name, @@ -169,30 +135,42 @@ class RuntimeProvider extends BaseProvider { getDefaultSettings() { const settings = super.getDefaultSettings(); - if (this.name !== 'claude') return settings; - return { - ...settings, - anthropicApiKey: null, - bedrockApiKey: null, - bedrockRegion: null, - }; + if (this._metadata.settingsDefaults) { + return { + ...settings, + ...this._metadata.settingsDefaults, + }; + } + if (this._metadata.settingsFields.length === 0) return settings; + return this._metadata.settingsFields.reduce( + (result, field) => ({ + ...result, + [field]: null, + }), + { ...settings } + ); } validateSettings(settings) { const baseError = super.validateSettings(settings); if (baseError) return baseError; + if (this._metadata.settingsValidator) { + const providerError = this._metadata.settingsValidator(settings); + if (providerError) return providerError; + } else { + for (const field of this._metadata.settingsFields) { + if ( + settings[field] !== undefined && + settings[field] !== null && + typeof settings[field] !== 'string' + ) { + return `providerSettings.${this.name}.${field} must be a string or null`; + } + } + } if (this.name !== 'claude') return null; const { isValidAnthropicKey, ANTHROPIC_KEY_PREFIX } = require('../../lib/settings/claude-auth'); - for (const field of this._metadata.settingsFields) { - if ( - settings[field] !== undefined && - settings[field] !== null && - typeof settings[field] !== 'string' - ) { - return `providerSettings.claude.${field} must be a string or null`; - } - } if (settings.anthropicApiKey && !isValidAnthropicKey(settings.anthropicApiKey)) { return `providerSettings.claude.anthropicApiKey must start with ${ANTHROPIC_KEY_PREFIX}`; } @@ -206,7 +184,9 @@ class RuntimeProvider extends BaseProvider { function getProvider(name) { const normalized = normalizeProviderName(name || ''); - if (!PROVIDER_METADATA[normalized]) { + try { + metadataForProvider(normalized); + } catch { throw new Error(`Unknown provider: ${name}. Valid: ${listProviders().join(', ')}`); } return new RuntimeProvider(normalized); diff --git a/src/worktree-claude-config.js b/src/worktree-claude-config.js index da420462..009d1654 100644 --- a/src/worktree-claude-config.js +++ b/src/worktree-claude-config.js @@ -119,6 +119,23 @@ function prepareClaudeConfigDir(options = {}) { return overlayDir; } +/** + * Resolve the repo's `.mcp.json` path (the same MCP-server source Claude consumes via + * prepareClaudeConfigDir) for a given worktree/cwd, or null if none exists. Reused by providers + * that consume MCP servers through a CLI flag instead of the Claude config-dir overlay (e.g. + * Copilot's `--additional-mcp-config`), so both providers share one MCP config surface. + */ +function resolveRepoMcpConfigPath(options = {}) { + const worktreeRoot = resolveWorktreeRoot(options.worktreePath || options.cwd); + if (!worktreeRoot) { + return null; + } + + const mcpPath = path.join(worktreeRoot, CLAUDE_DIRNAME, MCP_BASENAME); + return fs.existsSync(mcpPath) ? mcpPath : null; +} + module.exports = { prepareClaudeConfigDir, + resolveRepoMcpConfigPath, }; diff --git a/task-lib/commands/run.js b/task-lib/commands/run.js index 9b139d18..9aff6365 100644 --- a/task-lib/commands/run.js +++ b/task-lib/commands/run.js @@ -38,6 +38,7 @@ export async function runTask(prompt, options = {}) { continue: options.continue, outputFormat, jsonSchema, + mcpConfig: options.mcpConfig, silentJsonOutput, }); diff --git a/task-lib/provider-helper-runtime.js b/task-lib/provider-helper-runtime.js index 449a9d25..fb22adee 100644 --- a/task-lib/provider-helper-runtime.js +++ b/task-lib/provider-helper-runtime.js @@ -19,11 +19,22 @@ export const { classifyProviderError, detectProviderFatalError, detectProviderStreamingModeError, + findProviderRegistryEntry, getProviderAdapter, + getProviderRegistryEntry, + knownProviderNames, listProviderAdapters, + listProviderRegistryEntries, + normalizeProviderName, parseProviderChunk, prepareSingleAgentProviderCommand, recoverProviderStructuredOutput, + resolveProviderCommand, resolveModelSpec, + supportsProviderCapability, + providerAliasMap, + providerAliases, + providerIds, + providerRegistry, supportsProviderStructuredOutputRecovery, } = helper; diff --git a/task-lib/runner.js b/task-lib/runner.js index fb525b78..12808a57 100644 --- a/task-lib/runner.js +++ b/task-lib/runner.js @@ -80,11 +80,18 @@ function buildProviderOptions(options, outputFormat, jsonSchema, cwd) { cwd, autoApprove: true, ...modelSpecOption(options), + ...mcpConfigOption(options), ...(options.resume ? { resumeSessionId: options.resume } : {}), ...(options.continue ? { continueSession: true } : {}), }; } +function mcpConfigOption(options) { + const entries = options.mcpConfig; + if (!Array.isArray(entries) || entries.length === 0) return {}; + return { mcpConfig: entries }; +} + function modelSpecOption(options) { const modelSpec = resolveRequestedModelSpec(options); return modelSpec === undefined ? {} : { modelSpec }; diff --git a/tests/agent-cli-provider/architecture.test.js b/tests/agent-cli-provider/architecture.test.js index 18e9f39f..24c767d3 100644 --- a/tests/agent-cli-provider/architecture.test.js +++ b/tests/agent-cli-provider/architecture.test.js @@ -24,8 +24,12 @@ function relative(file) { const providerPolicyNames = [ 'claude', 'codex', + 'gateway', 'gemini', 'opencode', + 'pi', + 'kiro', + 'copilot', 'anthropic', 'openai', 'google', @@ -76,6 +80,7 @@ function containsForbiddenProviderPolicy(source) { test('live runtime paths only import the built provider helper through approved facades', () => { const allowedHelperImports = new Set([ + 'lib/provider-names.js', 'src/providers/index.js', 'task-lib/provider-helper-runtime.js', ]); @@ -129,11 +134,17 @@ test('deleted duplicate provider implementation files stay deleted', () => { assert.deepEqual(existing, []); }); -test('provider id and alias switch policy stays inside adapter registry and type declarations', () => { +test('provider id and alias switch policy stays inside provider registry and type declarations', () => { const helperFiles = walk(path.join(repoRoot, 'src', 'agent-cli-provider')) .filter((file) => file.endsWith('.ts')) .filter((file) => !relative(file).startsWith('src/agent-cli-provider/adapters/')) - .filter((file) => relative(file) !== 'src/agent-cli-provider/types.ts'); + .filter( + (file) => + ![ + 'src/agent-cli-provider/provider-registry.ts', + 'src/agent-cli-provider/types.ts', + ].includes(relative(file)) + ); const offenders = helperFiles .filter((file) => containsForbiddenProviderPolicy(fs.readFileSync(file, 'utf8'))) @@ -142,6 +153,47 @@ test('provider id and alias switch policy stays inside adapter registry and type assert.deepEqual(offenders, []); }); +test('preflight provider validation dispatches through registry metadata', () => { + const source = fs.readFileSync(path.join(repoRoot, 'src', 'preflight.js'), 'utf8'); + + assert.match(source, /getProviderMetadata\(providerName\)/); + assert.match(source, /metadata\.command\.kind === 'configured-claude'/); + assert.doesNotMatch( + source, + /validatorByProvider\s*=\s*\{[\s\S]*['"]codex['"]:[\s\S]*['"]gemini['"]:[\s\S]*['"]opencode['"]:/m + ); +}); + +test('stream log parser builds provider parsers from runtime provider list', () => { + const source = fs.readFileSync(path.join(repoRoot, 'lib', 'stream-json-parser.js'), 'utf8'); + + assert.match(source, /listProviders\(\)\.map\(\(name\) => getProvider\(name\)\)/); + assert.doesNotMatch(source, /providerOrder\s*=\s*\[/); + assert.match(source, /const providerParsers = createProviderParsers\(\)/); +}); + +test('docker preset surfaces derive provider entries from registry', () => { + const dockerSource = fs.readFileSync(path.join(repoRoot, 'lib', 'docker-config.js'), 'utf8'); + const cliSource = fs.readFileSync(path.join(repoRoot, 'cli', 'index.js'), 'utf8'); + + assert.match(dockerSource, /listProviderMetadata\(\)/); + assert.doesNotMatch( + dockerSource, + /claude:\s*\{[\s\S]*codex:\s*\{[\s\S]*gemini:\s*\{[\s\S]*opencode:\s*\{[\s\S]*pi:\s*\{/m + ); + assert.match(cliSource, /Object\.keys\(MOUNT_PRESETS\)\.join\(', '\)/); +}); + +test('CLI provider TUI entrypoints derive from registry provider list', () => { + const cliSource = fs.readFileSync(path.join(repoRoot, 'cli', 'index.js'), 'utf8'); + + assert.match(cliSource, /for \(const providerName of VALID_PROVIDERS\)/); + assert.doesNotMatch(cliSource, /registerTuiEntrypoint\(['"]codex['"]/); + assert.doesNotMatch(cliSource, /registerTuiEntrypoint\(['"]claude['"]/); + assert.doesNotMatch(cliSource, /registerTuiEntrypoint\(['"]gemini['"]/); + assert.doesNotMatch(cliSource, /registerTuiEntrypoint\(['"]opencode['"]/); +}); + test('task runner passes provider args to watcher once', () => { const source = fs.readFileSync(path.join(repoRoot, 'task-lib', 'runner.js'), 'utf8'); diff --git a/tests/agent-cli-provider/executable-contract-build.test.js b/tests/agent-cli-provider/executable-contract-build.test.js index 73496551..7f897bdb 100644 --- a/tests/agent-cli-provider/executable-contract-build.test.js +++ b/tests/agent-cli-provider/executable-contract-build.test.js @@ -6,6 +6,9 @@ const path = require('node:path'); const { assertNoSecret, fakeCodexScript, + fakeCopilotScript, + fakeKiroScript, + fakePiScript, runExecutable, withFakeProviderCli, withTempEnv, @@ -185,6 +188,382 @@ process.exit(17); ); }); +test('build-command returns ACP stdio command specs without prompt argv coupling', () => { + withFakeProviderCli( + 'kiro-cli', + fakeKiroScript(` +if (process.argv.includes('--help')) { + process.stdout.write('Usage: kiro-cli acp\\n'); + process.exit(0); +} +process.stderr.write('build-command should not execute kiro-cli acp'); +process.exit(17); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'kiro', + context: 'Reply with OK', + options: { + cwd: '/tmp/kiro-worktree', + }, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.commandSpec.binary, 'kiro-cli'); + assert.deepEqual(response.envelope.result.commandSpec.args, ['acp']); + assert.equal(response.envelope.result.commandSpec.cwd, '/tmp/kiro-worktree'); + assert.equal(response.envelope.result.commandSpec.args.includes('Reply with OK'), false); + } + ); +}); + +test('build-command returns bundled gateway runner specs with redacted config env', () => { + const secret = 'gateway-secret-token'; + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'gateway', + context: 'Edit the target file.', + options: { + cwd: '/tmp/gateway-project', + gateway: { + baseUrl: 'http://127.0.0.1:4000', + apiKey: secret, + headers: { + 'X-API-Key': 'custom-header-secret-42', + }, + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: ['node'], + }, + }, + }, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.provider, 'gateway'); + assert.equal(response.envelope.result.commandSpec.binary, process.execPath); + assert.match(response.envelope.result.commandSpec.args[0], /gateway-runner\.js$/); + assert.equal( + response.envelope.result.commandSpec.env.ZEROSHOT_GATEWAY_REQUEST.includes(secret), + false + ); + assert.equal( + response.envelope.result.commandSpec.env.ZEROSHOT_GATEWAY_REQUEST.includes( + 'custom-header-secret-42' + ), + false + ); + assertNoSecret(response.envelope, secret); + assertNoSecret(response.envelope, 'custom-header-secret-42'); +}); + +test('build-command rejects caller env that collides with gateway runner control vars', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'gateway', + context: 'Edit the target file.', + env: { + ZEROSHOT_GATEWAY_API_KEY: 'attacker-key', + }, + options: { + gateway: { + baseUrl: 'http://127.0.0.1:4000', + apiKey: 'gateway-secret-token', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: ['node'], + }, + }, + }, + }); + + assert.equal(response.exitCode, 2); + assert.equal(response.envelope.ok, false); + assert.equal(response.envelope.error.code, 'forbidden-field'); + assert.equal(response.envelope.error.field, 'env.ZEROSHOT_GATEWAY_API_KEY'); + assert.match(response.envelope.error.message, /provider adapters own ZEROSHOT_GATEWAY_API_KEY/i); +}); + +test('build-command resolves gateway settings tool roots against options.cwd', () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-settings-')); + const settingsFile = path.join(tempDir, 'settings.json'); + const worktree = path.join(tempDir, 'worktree'); + fs.mkdirSync(worktree, { recursive: true }); + fs.writeFileSync( + settingsFile, + JSON.stringify( + { + providerSettings: { + gateway: { + baseUrl: 'http://127.0.0.1:4000', + apiKey: 'gateway-secret-token', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: ['node'], + }, + }, + }, + }, + null, + 2 + ), + 'utf8' + ); + + try { + withTempEnv({ ZEROSHOT_SETTINGS_FILE: settingsFile }, () => { + const prepared = require('../../lib/agent-cli-provider').prepareSingleAgentProviderCommand({ + context: 'Edit the target file.', + provider: 'gateway', + options: { + cwd: worktree, + }, + }); + + const request = JSON.parse(prepared.commandSpec.env.ZEROSHOT_GATEWAY_REQUEST); + assert.deepEqual(request.gateway.toolPolicy.roots, [worktree]); + }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('build-command fails closed when ACP stdio support is not advertised', () => { + withFakeProviderCli( + 'kiro-cli', + fakeKiroScript(` +if (process.argv.includes('--help')) { + process.stdout.write('Usage: kiro-cli --version\\n'); + process.exit(0); +} +process.stderr.write('build-command should not execute kiro-cli acp'); +process.exit(17); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'kiro', + context: 'Reply with OK', + }); + + assert.equal(response.exitCode, 2); + assert.equal(response.envelope.ok, false); + assert.equal(response.envelope.error.code, 'invalid-field'); + assert.equal(response.envelope.error.field, 'options.cliFeatures.supportsAcpStdio'); + assert.match(response.envelope.error.message, /does not advertise ACP stdio support/i); + } + ); +}); + +test('build-command ignores caller ACP support overrides when runtime probe rejects ACP stdio', () => { + withFakeProviderCli( + 'kiro-cli', + fakeKiroScript(` +if (process.argv.includes('--help')) { + process.stdout.write('Usage: kiro-cli --version\\n'); + process.exit(0); +} +process.stderr.write('build-command should not execute kiro-cli acp'); +process.exit(17); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'kiro', + context: 'Reply with OK', + options: { + cliFeatures: { + supportsAcpStdio: true, + }, + }, + }); + + assert.equal(response.exitCode, 2); + assert.equal(response.envelope.ok, false); + assert.equal(response.envelope.error.code, 'invalid-field'); + assert.equal(response.envelope.error.field, 'options.cliFeatures.supportsAcpStdio'); + assert.match(response.envelope.error.message, /does not advertise ACP stdio support/i); + } + ); +}); + +test('build-command uses Pi JSON mode with discovery disabled and schema prompt fallback', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'pi', + context: 'Return JSON.', + options: { + outputFormat: 'json', + cwd: '/tmp/worktree', + jsonSchema: { type: 'object', properties: { ok: { type: 'boolean' } } }, + modelSpec: { model: 'openai/gpt-5.5' }, + cliFeatures: { + supportsJsonMode: true, + supportsNoSession: true, + supportsNoExtensions: true, + supportsNoSkills: true, + supportsNoPromptTemplates: true, + supportsNoContextFiles: true, + supportsNoApprove: true, + supportsModel: true, + }, + }, + }); + + const { commandSpec } = response.envelope.result; + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.schemaMode, 'prompt'); + assert.equal(commandSpec.binary, 'pi'); + assert.equal(commandSpec.cwd, '/tmp/worktree'); + assert.deepEqual(commandSpec.args.slice(0, 11), [ + '--mode', + 'json', + '--no-session', + '--no-extensions', + '--no-skills', + '--no-prompt-templates', + '--no-context-files', + '--no-approve', + '--model', + 'openai/gpt-5.5', + commandSpec.args.at(-1), + ]); + assert.ok(commandSpec.args.at(-1).includes('## OUTPUT FORMAT (CRITICAL - REQUIRED)')); + assert.ok(response.envelope.warnings.some((warning) => warning.code === 'pi-jsonschema')); +}); + +test('build-command rejects Pi resume/continue session control requests', () => { + const resumed = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'pi', + context: 'Return JSON.', + options: { + resumeSessionId: 'ignored-session', + cliFeatures: { + supportsJsonMode: true, + }, + }, + }); + + assert.equal(resumed.exitCode, 2); + assert.equal(resumed.envelope.ok, false); + assert.equal(resumed.envelope.error.code, 'invalid-field'); + assert.equal(resumed.envelope.error.field, 'options.resumeSessionId'); + + const emptyResumed = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'pi', + context: 'Return JSON.', + options: { + resumeSessionId: '', + cliFeatures: { + supportsJsonMode: true, + }, + }, + }); + + assert.equal(emptyResumed.exitCode, 2); + assert.equal(emptyResumed.envelope.ok, false); + assert.equal(emptyResumed.envelope.error.code, 'invalid-field'); + assert.equal(emptyResumed.envelope.error.field, 'options.resumeSessionId'); + + const continued = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'pi', + context: 'Return JSON.', + options: { + continueSession: true, + cliFeatures: { + supportsJsonMode: true, + }, + }, + }); + + assert.equal(continued.exitCode, 2); + assert.equal(continued.envelope.ok, false); + assert.equal(continued.envelope.error.code, 'invalid-field'); + assert.equal(continued.envelope.error.field, 'options.continueSession'); +}); + +test('build-command ignores undefined Pi resumeSessionId values', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'pi', + context: 'Return JSON.', + options: { + resumeSessionId: undefined, + cliFeatures: { + supportsJsonMode: true, + }, + }, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.commandSpec.binary, 'pi'); + assert.equal(response.envelope.result.commandSpec.args.at(-1), 'Return JSON.'); +}); + +test('build-command keeps Pi JSON-mode args when only version probe returns output', () => { + withFakeProviderCli( + 'pi', + fakePiScript(` +if (process.argv.includes('--help')) { + process.exit(0); +} +if (process.argv.includes('--version')) { + process.stdout.write('0.80.3\\n'); + process.exit(0); +} +process.stderr.write('unknown option -h\\n'); +process.exit(1); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'pi', + context: 'Return JSON.', + options: { + outputFormat: 'json', + }, + }); + + const args = response.envelope.result.commandSpec.args; + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.deepEqual(args.slice(0, 8), [ + '--mode', + 'json', + '--no-session', + '--no-extensions', + '--no-skills', + '--no-prompt-templates', + '--no-context-files', + '--no-approve', + ]); + assert.equal(args.at(-1), 'Return JSON.'); + } + ); +}); + test('build-command resolves Codex settings default level and model overrides', () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-provider-settings-')); const settingsFile = path.join(tempDir, 'settings.json'); @@ -292,3 +671,243 @@ process.exit(17); } ); }); + +test('probe requires Pi help or version output when helpText is not supplied', () => { + withFakeProviderCli( + 'pi', + fakePiScript(` +if (process.argv.includes('--help')) { + process.stdout.write('Usage: pi --mode json --no-session --no-extensions --no-skills --no-prompt-templates --no-context-files --no-approve --model\\n'); + process.exit(0); +} +if (process.argv.includes('--version')) { + process.stdout.write('0.80.3\\n'); + process.exit(0); +} +process.exit(17); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'probe', + provider: 'pi', + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.available, true); + assert.equal(response.envelope.result.provider.id, 'pi'); + assert.equal(response.envelope.result.capabilities.supportsJsonMode, true); + assert.equal(response.envelope.result.capabilities.supportsNoApprove, true); + assert.equal(response.envelope.result.versionText, '0.80.3'); + } + ); +}); + +test('probe exposes ACP CLI capabilities for Kiro', () => { + withFakeProviderCli( + 'kiro-cli', + fakeKiroScript(` +if (process.argv.includes('--help')) { + process.stdout.write('Usage: kiro-cli acp\\n'); + process.exit(0); +} +process.exit(17); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'probe', + provider: 'kiro', + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.available, true); + assert.equal(response.envelope.result.provider.id, 'kiro'); + assert.equal(response.envelope.result.capabilities.supportsAcpStdio, true); + assert.equal(response.envelope.result.capabilities.supportsPermissionRequests, false); + assert.equal(response.envelope.result.capabilities.supportsTerminalTools, false); + } + ); +}); + +test('build-command builds Copilot autonomous argv with schema prompt fallback', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'copilot', + context: 'Return JSON.', + options: { + outputFormat: 'json', + cwd: '/tmp/worktree', + autoApprove: true, + jsonSchema: { type: 'object', properties: { ok: { type: 'boolean' } } }, + modelSpec: { model: 'gpt-5.2' }, + cliFeatures: { + supportsJsonOutput: true, + supportsModel: true, + supportsAllowAll: true, + supportsNoAskUser: true, + supportsAddDir: true, + }, + }, + }); + + const { commandSpec } = response.envelope.result; + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.schemaMode, 'prompt'); + assert.equal(commandSpec.binary, 'copilot'); + assert.equal(commandSpec.cwd, '/tmp/worktree'); + assert.deepEqual(commandSpec.args, [ + '--output-format', + 'json', + '--model', + 'gpt-5.2', + '--add-dir', + '/tmp/worktree', + '--allow-all', + '--no-ask-user', + '-p', + commandSpec.args.at(-1), + ]); + assert.equal(commandSpec.args.at(-2), '-p'); + assert.ok(commandSpec.args.at(-1).includes('## OUTPUT FORMAT (CRITICAL - REQUIRED)')); + assert.ok(response.envelope.warnings.some((warning) => warning.code === 'copilot-jsonschema')); +}); + +test('build-command omits Copilot approval flags when autoApprove is not requested', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'copilot', + context: 'Return JSON.', + options: { + outputFormat: 'json', + cliFeatures: { + supportsJsonOutput: true, + supportsModel: true, + supportsAllowAll: true, + supportsNoAskUser: true, + supportsAddDir: true, + }, + }, + }); + + const { commandSpec } = response.envelope.result; + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(commandSpec.args.includes('--allow-all'), false); + assert.equal(commandSpec.args.includes('--no-ask-user'), false); + assert.equal(commandSpec.args.at(-2), '-p'); + assert.equal(commandSpec.args.at(-1), 'Return JSON.'); +}); + +test('build-command keeps Copilot JSON output args when only version probe returns output', () => { + withFakeProviderCli( + 'copilot', + fakeCopilotScript(` +if (process.argv.includes('--help')) { + process.exit(0); +} +if (process.argv.includes('--version')) { + process.stdout.write('1.0.0\\n'); + process.exit(0); +} +process.stderr.write('unknown option -h\\n'); +process.exit(1); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'copilot', + context: 'Return JSON.', + options: { + outputFormat: 'json', + }, + }); + + const args = response.envelope.result.commandSpec.args; + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.deepEqual(args.slice(0, 2), ['--output-format', 'json']); + assert.equal(args.at(-2), '-p'); + assert.equal(args.at(-1), 'Return JSON.'); + } + ); +}); + +test('build-command emits one Copilot --additional-mcp-config flag per mcpConfig entry', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'copilot', + context: 'Do work.', + options: { + autoApprove: true, + mcpConfig: ['{"mcpServers":{"a":{"command":"a-bin"}}}', '@/tmp/servers.json'], + cliFeatures: { + supportsAllowAll: true, + supportsNoAskUser: true, + supportsMcpConfig: true, + }, + }, + }); + + const { commandSpec } = response.envelope.result; + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.deepEqual(commandSpec.args, [ + '--allow-all', + '--no-ask-user', + '--additional-mcp-config', + '{"mcpServers":{"a":{"command":"a-bin"}}}', + '--additional-mcp-config', + '@/tmp/servers.json', + '-p', + 'Do work.', + ]); + assert.equal( + response.envelope.warnings.some((warning) => warning.code === 'copilot-mcp-config'), + false + ); +}); + +test('build-command gates Copilot MCP config on feature detection and warns when unsupported', () => { + withFakeProviderCli( + 'copilot', + fakeCopilotScript(` +if (process.argv.includes('--help')) { + process.stdout.write('Usage: copilot -p --output-format json --model --allow-all --no-ask-user --add-dir \\n'); + process.exit(0); +} +if (process.argv.includes('--version')) { + process.stdout.write('1.0.0\\n'); + process.exit(0); +} +process.exit(0); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'build-command', + provider: 'copilot', + context: 'Do work.', + options: { + mcpConfig: ['{"mcpServers":{"a":{"command":"a-bin"}}}'], + }, + }); + + const { commandSpec } = response.envelope.result; + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(commandSpec.args.includes('--additional-mcp-config'), false); + assert.ok( + response.envelope.warnings.some((warning) => warning.code === 'copilot-mcp-config'), + 'expected a copilot-mcp-config warning when the CLI lacks --additional-mcp-config' + ); + } + ); +}); diff --git a/tests/agent-cli-provider/executable-contract-env-safety.test.js b/tests/agent-cli-provider/executable-contract-env-safety.test.js index b2e4737b..6c84a0c9 100644 --- a/tests/agent-cli-provider/executable-contract-env-safety.test.js +++ b/tests/agent-cli-provider/executable-contract-env-safety.test.js @@ -145,3 +145,43 @@ test('invoke rejects caller-supplied process-control authEnv before runner execu assert.equal(runnerCalled, false); assertNoSecret(response.envelope, '--require /tmp/pwn.cjs'); }); + +test('invoke rejects caller-supplied gateway runner env overrides before runner execution', async () => { + let runnerCalled = false; + const response = await runProviderExecutable( + { + schemaVersion: 1, + command: 'invoke', + provider: 'gateway', + context: 'hi', + env: { + zeroshot_gateway_request: '{"context":"pwn"}', + }, + options: { + gateway: { + baseUrl: 'http://127.0.0.1:11434', + apiKey: 'gateway-secret-token', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: ['node'], + }, + }, + }, + }, + { + runner: () => { + runnerCalled = true; + return runnerResult({ stdout: 'MALICIOUS-GATEWAY-RAN' }); + }, + } + ); + + assert.equal(response.exitCode, 2); + assert.equal(response.envelope.ok, false); + assert.equal(response.envelope.command, 'invoke'); + assert.equal(response.envelope.provider, 'gateway'); + assert.equal(response.envelope.error.code, 'forbidden-field'); + assert.equal(response.envelope.error.field, 'env.zeroshot_gateway_request'); + assert.equal(runnerCalled, false); +}); diff --git a/tests/agent-cli-provider/executable-contract-helpers.cjs b/tests/agent-cli-provider/executable-contract-helpers.cjs index 05e0a1e3..2ec9e839 100644 --- a/tests/agent-cli-provider/executable-contract-helpers.cjs +++ b/tests/agent-cli-provider/executable-contract-helpers.cjs @@ -103,6 +103,18 @@ function fakeCodexScript(body) { return `#!/usr/bin/env node\n${body}\n`; } +function fakePiScript(body) { + return `#!/usr/bin/env node\n${body}\n`; +} + +function fakeKiroScript(body) { + return `#!/usr/bin/env node\n${body}\n`; +} + +function fakeCopilotScript(body) { + return `#!/usr/bin/env node\n${body}\n`; +} + function invokeCodexSchemaRequest(overrides = {}) { return { schemaVersion: 1, @@ -119,6 +131,9 @@ module.exports = { codexSchemaOptions, invokeCodexSchemaRequest, fakeCodexScript, + fakeCopilotScript, + fakeKiroScript, + fakePiScript, repoRoot, runExecutable, runProviderExecutable, diff --git a/tests/agent-cli-provider/executable-contract-invoke.test.js b/tests/agent-cli-provider/executable-contract-invoke.test.js index c808c81c..d491e0b0 100644 --- a/tests/agent-cli-provider/executable-contract-invoke.test.js +++ b/tests/agent-cli-provider/executable-contract-invoke.test.js @@ -1,14 +1,20 @@ const assert = require('node:assert/strict'); const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); const { test } = require('node:test'); const { assertNoSecret, fakeCodexScript, + fakeCopilotScript, + fakeKiroScript, + fakePiScript, invokeCodexSchemaRequest, runExecutable, runProviderExecutable, runnerResult, withFakeProviderCli, + withTempEnv, } = require('./executable-contract-helpers.cjs'); test('invoke returns redacted terminal evidence, parsed events, status, timing, and cleanup', async () => { @@ -54,6 +60,115 @@ test('invoke returns redacted terminal evidence, parsed events, status, timing, assertNoSecret(response.envelope, secret); }); +test('invoke parses bundled gateway runner events', async () => { + let runnerCommand = null; + const secret = 'gateway-secret'; + const response = await runProviderExecutable( + { + schemaVersion: 1, + command: 'invoke', + provider: 'gateway', + context: 'Edit note.txt', + options: { + cwd: '/tmp/gateway-project', + gateway: { + baseUrl: 'http://127.0.0.1:11434', + apiKey: secret, + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: ['node'], + }, + }, + }, + }, + { + runner: (commandSpec) => { + runnerCommand = commandSpec; + return runnerResult({ + stdout: [ + JSON.stringify({ type: 'text', text: 'editing' }), + JSON.stringify({ + type: 'tool_call', + toolName: 'read_file', + toolId: 'tool-1', + input: { path: 'note.txt' }, + }), + JSON.stringify({ + type: 'tool_result', + toolId: 'tool-1', + content: { path: 'note.txt', content: 'before' }, + isError: false, + }), + JSON.stringify({ type: 'result', success: true, result: { text: 'done' } }), + ].join('\n'), + }); + }, + } + ); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(runnerCommand.binary, process.execPath); + assert.match(runnerCommand.args[0], /gateway-runner\.js$/); + assert.equal(runnerCommand.env.ZEROSHOT_GATEWAY_API_KEY, secret); + assert.equal(runnerCommand.env.ZEROSHOT_GATEWAY_REQUEST.includes(secret), false); + assert.deepEqual( + response.envelope.result.events.map((event) => event.type), + ['text', 'tool_call', 'tool_result', 'result'] + ); +}); + +test('invoke redacts gateway api keys leaked by the runner', async () => { + const secret = 'gateway-plain-secret'; + const headerSecret = 'custom-header-secret-42'; + const response = await runProviderExecutable( + { + schemaVersion: 1, + command: 'invoke', + provider: 'gateway', + context: 'Reply with ok.', + options: { + gateway: { + baseUrl: 'http://127.0.0.1:11434', + apiKey: secret, + headers: { + 'X-API-Key': headerSecret, + }, + model: 'openrouter/test-model', + toolPolicy: { + roots: [process.cwd()], + commands: [], + }, + }, + }, + }, + { + runner: (commandSpec) => { + const request = JSON.parse(commandSpec.env.ZEROSHOT_GATEWAY_REQUEST); + const headerEnvKey = request.gatewayHeaderEnv?.['X-API-Key']; + const leakedHeaderSecret = + (typeof headerEnvKey === 'string' && commandSpec.env[headerEnvKey]) || + request.gateway?.headers?.['X-API-Key'] || + ''; + return runnerResult({ + stdout: `leak ${secret} ${leakedHeaderSecret}`, + stderr: `auth failed ${secret} ${leakedHeaderSecret}`, + }); + }, + } + ); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.evidence.stdout.includes(secret), false); + assert.equal(response.envelope.result.evidence.stderr.includes(secret), false); + assert.equal(response.envelope.result.evidence.stdout.includes(headerSecret), false); + assert.equal(response.envelope.result.evidence.stderr.includes(headerSecret), false); + assertNoSecret(response.envelope, secret); + assertNoSecret(response.envelope, headerSecret); +}); + test('invoke removes schema cleanup files when runner rejects', async () => { let cleanupPath = null; const response = await runProviderExecutable(invokeCodexSchemaRequest(), { @@ -213,3 +328,464 @@ process.stdin.resume(); } ); }); + +test('invoke runs ACP stdio providers through the shared headless lane', () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-kiro-worktree-')); + + try { + withFakeProviderCli( + 'kiro-cli', + fakeKiroScript(` +const readline = require('node:readline'); +if (process.argv.includes('--help')) { + process.stdout.write('Usage: kiro-cli acp\\n'); + process.exit(0); +} +const rl = readline.createInterface({ input: process.stdin }); +rl.on('line', (line) => { + const message = JSON.parse(line); + if (message.method === 'initialize') { + process.stdout.write(JSON.stringify({ + jsonrpc: '2.0', + id: message.id, + result: { protocolVersion: 1 }, + }) + '\\n'); + return; + } + if (message.method === 'session/new') { + process.stdout.write(JSON.stringify({ + jsonrpc: '2.0', + id: message.id, + result: { sessionId: 'kiro-session-1' }, + }) + '\\n'); + return; + } + if (message.method === 'session/prompt') { + process.stdout.write(JSON.stringify({ + jsonrpc: '2.0', + method: 'session/update', + params: { + sessionId: 'kiro-session-1', + update: { + sessionUpdate: 'tool_call', + toolCallId: 'tool-1', + title: 'bash', + rawInput: { command: 'pwd' }, + }, + }, + }) + '\\n'); + process.stdout.write(JSON.stringify({ + jsonrpc: '2.0', + method: 'session/update', + params: { + sessionId: 'kiro-session-1', + update: { + sessionUpdate: 'tool_call_update', + toolCallId: 'tool-1', + status: 'completed', + rawOutput: '/tmp/kiro', + }, + }, + }) + '\\n'); + process.stdout.write(JSON.stringify({ + jsonrpc: '2.0', + method: 'session/update', + params: { + sessionId: 'kiro-session-1', + update: { + sessionUpdate: 'agent_message_chunk', + messageId: 'msg-1', + content: { type: 'text', text: 'Kiro invoke OK' }, + }, + }, + }) + '\\n'); + process.stdout.write(JSON.stringify({ + jsonrpc: '2.0', + id: message.id, + result: { + stopReason: 'end_turn', + usage: { + inputTokens: 5, + outputTokens: 3, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + }, + }, + }) + '\\n'); + } +}); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'invoke', + provider: 'kiro', + context: 'Reply with Kiro invoke OK', + options: { + cwd: tempDir, + }, + timeoutMs: 300, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.commandSpec.binary, 'kiro-cli'); + assert.deepEqual(response.envelope.result.commandSpec.args, ['acp']); + assert.deepEqual(response.envelope.result.events, [ + { type: 'tool_call', toolName: 'bash', toolId: 'tool-1', input: { command: 'pwd' } }, + { type: 'tool_result', toolId: 'tool-1', content: '/tmp/kiro', isError: false }, + { type: 'text', text: 'Kiro invoke OK' }, + { + type: 'result', + success: true, + result: 'Kiro invoke OK', + error: null, + inputTokens: 5, + outputTokens: 3, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + cost: null, + modelUsage: { + inputTokens: 5, + outputTokens: 3, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + }, + }, + ]); + } + ); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('invoke fails closed on ACP permission callbacks', () => { + withFakeProviderCli( + 'kiro-cli', + fakeKiroScript(` +const readline = require('node:readline'); +if (process.argv.includes('--help')) { + process.stdout.write('Usage: kiro-cli acp\\n'); + process.exit(0); +} +const rl = readline.createInterface({ input: process.stdin }); +rl.on('line', (line) => { + const message = JSON.parse(line); + if (message.method === 'initialize') { + process.stdout.write(JSON.stringify({ + jsonrpc: '2.0', + id: message.id, + result: { protocolVersion: 1 }, + }) + '\\n'); + return; + } + if (message.method === 'session/new') { + process.stdout.write(JSON.stringify({ + jsonrpc: '2.0', + id: message.id, + result: { sessionId: 'kiro-session-1' }, + }) + '\\n'); + return; + } + if (message.method === 'session/prompt') { + process.stdout.write(JSON.stringify({ + jsonrpc: '2.0', + id: 77, + method: 'session/request_permission', + params: { sessionId: 'kiro-session-1' }, + }) + '\\n'); + setInterval(() => {}, 1000); + } +}); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'invoke', + provider: 'kiro', + context: 'trigger permission callback', + timeoutMs: 300, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.timedOut, false); + assert.equal(response.envelope.result.classification.retryable, false); + assert.match( + response.envelope.result.evidence.stderr, + /kiro ACP stdio fail-closed: unsupported session\/request_permission callback/i + ); + } + ); +}); + +test('invoke fails closed on malformed ACP stdout JSON', () => { + withFakeProviderCli( + 'kiro-cli', + fakeKiroScript(` +const readline = require('node:readline'); +if (process.argv.includes('--help')) { + process.stdout.write('Usage: kiro-cli acp\\n'); + process.exit(0); +} +const rl = readline.createInterface({ input: process.stdin }); +rl.on('line', (line) => { + const message = JSON.parse(line); + if (message.method === 'initialize') { + process.stdout.write('{not json}\\n'); + setInterval(() => {}, 1000); + } +}); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'invoke', + provider: 'kiro', + context: 'Reply with OK', + timeoutMs: 300, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.timedOut, false); + assert.equal(response.envelope.result.classification.retryable, false); + assert.match( + response.envelope.result.evidence.stderr, + /kiro ACP stdio fail-closed: malformed ACP stdout JSON/i + ); + } + ); +}); + +test('invoke fails closed when ACP stdio support is not advertised', () => { + withFakeProviderCli( + 'kiro-cli', + fakeKiroScript(` +if (process.argv.includes('--help')) { + process.stdout.write('Usage: kiro-cli --version\\n'); + process.exit(0); +} +process.stderr.write('invoke should not execute kiro-cli acp'); +process.exit(17); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'invoke', + provider: 'kiro', + context: 'Reply with OK', + timeoutMs: 300, + }); + + assert.equal(response.exitCode, 2); + assert.equal(response.envelope.ok, false); + assert.equal(response.envelope.error.code, 'invalid-field'); + assert.equal(response.envelope.error.field, 'options.cliFeatures.supportsAcpStdio'); + assert.match(response.envelope.error.message, /does not advertise ACP stdio support/i); + } + ); +}); + +test('invoke ignores caller ACP support overrides when runtime probe rejects ACP stdio', () => { + withFakeProviderCli( + 'kiro-cli', + fakeKiroScript(` +if (process.argv.includes('--help')) { + process.stdout.write('Usage: kiro-cli --version\\n'); + process.exit(0); +} +process.stderr.write('invoke should not execute kiro-cli acp'); +process.exit(17); +`), + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'invoke', + provider: 'kiro', + context: 'Reply with OK', + timeoutMs: 300, + options: { + cliFeatures: { + supportsAcpStdio: true, + }, + }, + }); + + assert.equal(response.exitCode, 2); + assert.equal(response.envelope.ok, false); + assert.equal(response.envelope.error.code, 'invalid-field'); + assert.equal(response.envelope.error.field, 'options.cliFeatures.supportsAcpStdio'); + assert.match(response.envelope.error.message, /does not advertise ACP stdio support/i); + } + ); +}); + +test('invoke runs Pi in the requested worktree cwd and normalizes streamed JSONL', () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-pi-worktree-')); + const fixturePath = path.join(__dirname, '..', 'fixtures', 'pi', 'tool.jsonl'); + + try { + withFakeProviderCli( + 'pi', + fakePiScript(` +const fs = require('node:fs'); +if (process.argv.includes('--help')) { + process.stdout.write('Usage: pi --mode json --no-session --no-extensions --no-skills --no-prompt-templates --no-context-files --no-approve --model\\n'); + process.exit(0); +} +if (process.argv.includes('--version')) { +process.stdout.write('0.80.3\\n'); + process.exit(0); +} +if (fs.realpathSync(process.cwd()) !== process.env.PI_EXPECT_CWD) { + process.stderr.write(\`cwd mismatch: \${process.cwd()}\`); + process.exit(19); +} +process.stdout.write(fs.readFileSync(process.env.PI_FIXTURE, 'utf8')); +`), + () => + withTempEnv( + { + PI_EXPECT_CWD: fs.realpathSync(tempDir), + PI_FIXTURE: fixturePath, + }, + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'invoke', + provider: 'pi', + context: 'Run one tool.', + options: { + cwd: tempDir, + outputFormat: 'json', + modelSpec: { model: 'openai/gpt-5.5' }, + }, + timeoutMs: 300, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.exitCode, 0); + assert.equal(response.envelope.result.commandSpec.cwd, tempDir); + assert.deepEqual(response.envelope.result.commandSpec.args.slice(0, 10), [ + '--mode', + 'json', + '--no-session', + '--no-extensions', + '--no-skills', + '--no-prompt-templates', + '--no-context-files', + '--no-approve', + '--model', + 'openai/gpt-5.5', + ]); + assert.equal(response.envelope.result.events[0].type, 'tool_call'); + assert.equal(response.envelope.result.events.at(-1).type, 'result'); + } + ) + ); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('invoke classifies Pi in-band turn_end failures even when the process exits 0', async () => { + const fixturePath = path.join(__dirname, '..', 'fixtures', 'pi', 'auth-failure.jsonl'); + const response = await runProviderExecutable( + { + schemaVersion: 1, + command: 'invoke', + provider: 'pi', + context: 'Authenticate.', + options: { + outputFormat: 'json', + }, + }, + { + runner: () => + runnerResult({ + stdout: fs.readFileSync(fixturePath, 'utf8'), + exitCode: 0, + signal: null, + }), + } + ); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.events.at(-1).type, 'result'); + assert.equal(response.envelope.result.events.at(-1).error, 'authentication required: run /login'); + assert.equal(response.envelope.result.classification.retryable, false); + assert.equal(response.envelope.result.classification.kind, 'permanent-pattern'); +}); + +test('invoke runs Copilot in the requested worktree cwd and normalizes streamed JSONL', () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-copilot-worktree-')); + const fixturePath = path.join(__dirname, '..', 'fixtures', 'copilot', 'tool.jsonl'); + + try { + withFakeProviderCli( + 'copilot', + fakeCopilotScript(` +const fs = require('node:fs'); +if (process.argv.includes('--help')) { + process.stdout.write('Usage: copilot -p --output-format json --model --allow-all --no-ask-user --add-dir \\n'); + process.exit(0); +} +if (process.argv.includes('--version')) { + process.stdout.write('1.0.0\\n'); + process.exit(0); +} +if (fs.realpathSync(process.cwd()) !== process.env.COPILOT_EXPECT_CWD) { + process.stderr.write(\`cwd mismatch: \${process.cwd()}\`); + process.exit(19); +} +process.stdout.write(fs.readFileSync(process.env.COPILOT_FIXTURE, 'utf8')); +`), + () => + withTempEnv( + { + COPILOT_EXPECT_CWD: fs.realpathSync(tempDir), + COPILOT_FIXTURE: fixturePath, + }, + () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'invoke', + provider: 'copilot', + context: 'Run one tool.', + options: { + cwd: tempDir, + outputFormat: 'json', + }, + timeoutMs: 300, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.exitCode, 0); + assert.equal(response.envelope.result.commandSpec.cwd, tempDir); + const args = response.envelope.result.commandSpec.args; + const addDirIndex = args.indexOf('--add-dir'); + assert.notEqual(addDirIndex, -1); + assert.equal(args[addDirIndex + 1], tempDir); + const events = response.envelope.result.events; + assert.ok( + events.some((event) => event.type === 'tool_call'), + 'expected a tool_call event' + ); + assert.ok( + events.some((event) => event.type === 'tool_result'), + 'expected a tool_result event' + ); + assert.equal(events.at(-1).type, 'result'); + assert.equal(events.at(-1).success, true); + } + ) + ); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); diff --git a/tests/agent-cli-provider/executable-contract-option-validation.test.js b/tests/agent-cli-provider/executable-contract-option-validation.test.js index 93ca0f58..26d2c06d 100644 --- a/tests/agent-cli-provider/executable-contract-option-validation.test.js +++ b/tests/agent-cli-provider/executable-contract-option-validation.test.js @@ -30,6 +30,11 @@ const invalidNestedOptionCases = [ options: { cliFeatures: { supportsCwd: 'true' } }, field: 'options.cliFeatures.supportsCwd', }, + { + name: 'cliFeatures supportsDir boolean', + options: { cliFeatures: { supportsDir: 'true' } }, + field: 'options.cliFeatures.supportsDir', + }, { name: 'modelSpec object', options: { modelSpec: 'level2' }, field: 'options.modelSpec' }, { name: 'modelSpec level enum', @@ -46,6 +51,46 @@ const invalidNestedOptionCases = [ options: { modelSpec: { reasoningEffort: 'extreme' } }, field: 'options.modelSpec.reasoningEffort', }, + { + name: 'mcpConfig array', + options: { mcpConfig: '{"mcpServers":{}}' }, + field: 'options.mcpConfig', + }, + { + name: 'mcpConfig entry type', + options: { mcpConfig: [123] }, + field: 'options.mcpConfig[0]', + }, + { + name: 'mcpConfig entry empty', + options: { mcpConfig: [' '] }, + field: 'options.mcpConfig[0]', + }, + { + name: 'gateway object', + options: { gateway: 'http://localhost:11434' }, + field: 'options.gateway', + }, + { + name: 'gateway headers string values', + options: { gateway: { headers: { Authorization: 123 } } }, + field: 'options.gateway.headers.Authorization', + }, + { + name: 'gateway tool policy object', + options: { gateway: { toolPolicy: 'all-access' } }, + field: 'options.gateway.toolPolicy', + }, + { + name: 'gateway tool policy roots array', + options: { gateway: { toolPolicy: { roots: 'src', commands: [] } } }, + field: 'options.gateway.toolPolicy.roots', + }, + { + name: 'gateway tool policy command timeout', + options: { gateway: { toolPolicy: { roots: ['src'], commands: [], commandTimeoutMs: 0 } } }, + field: 'options.gateway.toolPolicy.commandTimeoutMs', + }, ]; test('build-command rejects invalid nested options before command spec creation', () => { diff --git a/tests/agent-cli-provider/executable-contract-output.test.js b/tests/agent-cli-provider/executable-contract-output.test.js index cab5774f..002e561e 100644 --- a/tests/agent-cli-provider/executable-contract-output.test.js +++ b/tests/agent-cli-provider/executable-contract-output.test.js @@ -1,4 +1,6 @@ const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); const { test } = require('node:test'); const { assertNoSecret, @@ -6,6 +8,18 @@ const { runProviderExecutable, } = require('./executable-contract-helpers.cjs'); +function piFixture(name) { + return fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'pi', name), 'utf8'); +} + +function kiroFixture(name) { + return fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'kiro', name), 'utf8'); +} + +function copilotFixture(name) { + return fs.readFileSync(path.join(__dirname, '..', 'fixtures', 'copilot', name), 'utf8'); +} + test('parse-output returns normalized events and parser diagnostics', () => { const stdout = [ JSON.stringify({ @@ -119,6 +133,24 @@ test('classify-error returns machine-readable category and redacted evidence', ( assertNoSecret(response.envelope, 'sk-secret'); }); +test('classify-error treats Gemini unsupported-client auth failures as permanent', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'classify-error', + provider: 'gemini', + error: { + message: + 'IneligibleTierError: This client is no longer supported for Gemini Code Assist for individuals. reasonCode: UNSUPPORTED_CLIENT', + status: 1, + }, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.classification.retryable, false); + assert.equal(response.envelope.result.classification.kind, 'permanent-pattern'); +}); + test('classify-error redacts secret-looking evidence without matching env value', () => { const secret = 'sk-live-123456'; const response = runExecutable({ @@ -197,6 +229,34 @@ test('classify-error uses nested response status categories and evidence', () => } }); +test('classify-error recovers HTTP status from message-only gateway errors', () => { + for (const [message, expected] of [ + [ + 'Gateway request failed with status 404: No such model', + { kind: 'status-permanent', category: 'permanent', status: 404, retryable: false }, + ], + [ + 'Gateway request failed with status 500: Upstream exploded', + { kind: 'status-retryable', category: 'retryable', status: 500, retryable: true }, + ], + ]) { + const response = runExecutable({ + schemaVersion: 1, + command: 'classify-error', + provider: 'gateway', + error: { message }, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.evidence.status, expected.status); + assert.equal(response.envelope.result.evidence.status, expected.status); + assert.equal(response.envelope.result.classification.kind, expected.kind); + assert.equal(response.envelope.result.classification.retryable, expected.retryable); + assert.equal(response.envelope.result.category, expected.category); + } +}); + test('parse-output reports parser-error diagnostics when provider parser throws', async () => { const adapters = require('../../lib/agent-cli-provider/adapters'); const originalParseProviderChunk = adapters.parseProviderChunk; @@ -222,3 +282,526 @@ test('parse-output reports parser-error diagnostics when provider parser throws' adapters.parseProviderChunk = originalParseProviderChunk; } }); + +test('parse-output normalizes Pi JSON fixtures without duplicate snapshot text', () => { + const textResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'pi', + stdout: piFixture('text.jsonl'), + }); + + assert.equal(textResponse.exitCode, 0); + assert.equal(textResponse.envelope.ok, true); + assert.deepEqual(textResponse.envelope.result.events, [ + { type: 'text', text: 'Hello from Pi' }, + { + type: 'result', + success: true, + result: 'Hello from Pi', + error: null, + inputTokens: 7, + outputTokens: 3, + cacheReadInputTokens: 1, + cacheCreationInputTokens: 0, + cost: null, + modelUsage: { input: 7, output: 3, cacheRead: 1, cacheWrite: 0 }, + }, + ]); + + const toolResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'pi', + stdout: piFixture('tool.jsonl'), + }); + + assert.equal(toolResponse.exitCode, 0); + assert.equal(toolResponse.envelope.ok, true); + assert.deepEqual(toolResponse.envelope.result.events, [ + { type: 'tool_call', toolName: 'bash', toolId: 'tool-1', input: { command: 'pwd' } }, + { type: 'tool_result', toolId: 'tool-1', content: 'line1', isError: false }, + { type: 'tool_result', toolId: 'tool-1', content: 'line1\n/tmp/pi', isError: false }, + { type: 'text', text: 'done' }, + { + type: 'result', + success: true, + result: 'done', + error: null, + inputTokens: 9, + outputTokens: 4, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + cost: null, + modelUsage: { input: 9, output: 4, cacheRead: 0, cacheWrite: 0 }, + }, + ]); +}); + +test('parse-output handles Pi failure and empty fixtures', () => { + for (const [name, expectedError] of [ + ['command-failure.jsonl', 'Unknown option --bogus'], + ['auth-failure.jsonl', 'authentication required: run /login'], + ['rate-limit.jsonl', 'rate limit exceeded; retry later'], + ['cancelled.jsonl', 'cancelled by user'], + ]) { + const response = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'pi', + stdout: piFixture(name), + }); + + const lastEvent = response.envelope.result.events.at(-1); + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(lastEvent.type, 'result'); + assert.equal(lastEvent.success, false); + assert.equal(lastEvent.error, expectedError); + } + + const emptyResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'pi', + stdout: piFixture('empty.jsonl'), + }); + + assert.equal(emptyResponse.exitCode, 0); + assert.equal(emptyResponse.envelope.ok, true); + assert.deepEqual(emptyResponse.envelope.result.events, []); +}); + +test('parse-output normalizes Kiro ACP fixtures and diagnostics', () => { + const textResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'kiro', + stdout: kiroFixture('text.jsonl'), + }); + + assert.equal(textResponse.exitCode, 0); + assert.equal(textResponse.envelope.ok, true); + assert.deepEqual(textResponse.envelope.result.events, [ + { type: 'text', text: 'Hello' }, + { type: 'text', text: ' from Kiro' }, + { + type: 'result', + success: true, + result: 'Hello from Kiro', + error: null, + inputTokens: 7, + outputTokens: 3, + cacheReadInputTokens: 1, + cacheCreationInputTokens: 0, + cost: null, + modelUsage: { + inputTokens: 7, + outputTokens: 3, + cacheReadInputTokens: 1, + cacheCreationInputTokens: 0, + }, + }, + ]); + + const toolResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'kiro', + stdout: kiroFixture('tool.jsonl'), + }); + + assert.equal(toolResponse.exitCode, 0); + assert.equal(toolResponse.envelope.ok, true); + assert.deepEqual(toolResponse.envelope.result.events, [ + { type: 'tool_call', toolName: 'bash', toolId: 'tool-1', input: { command: 'pwd' } }, + { type: 'tool_result', toolId: 'tool-1', content: 'line1', isError: false }, + { type: 'tool_result', toolId: 'tool-1', content: 'line1\n/tmp/kiro', isError: false }, + { type: 'text', text: 'done' }, + { + type: 'result', + success: true, + result: 'done', + error: null, + inputTokens: 9, + outputTokens: 4, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + cost: null, + modelUsage: { + inputTokens: 9, + outputTokens: 4, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + }, + }, + ]); + + for (const [name, expectedError] of [ + ['auth-failure.jsonl', 'authentication required: run kiro auth login'], + ['cancelled.jsonl', 'cancelled by user'], + ]) { + const response = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'kiro', + stdout: kiroFixture(name), + }); + + const lastEvent = response.envelope.result.events.at(-1); + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(lastEvent.type, 'result'); + assert.equal(lastEvent.success, false); + assert.equal(lastEvent.error, expectedError); + } + + const malformedResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'kiro', + stdout: kiroFixture('malformed.jsonl'), + }); + + assert.equal(malformedResponse.exitCode, 0); + assert.equal(malformedResponse.envelope.ok, true); + assert.deepEqual(malformedResponse.envelope.result.events, [{ type: 'text', text: 'ok' }]); + assert.equal(malformedResponse.envelope.result.diagnostics[0].kind, 'parse-error'); + + const emptyResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'kiro', + stdout: kiroFixture('empty.jsonl'), + }); + + assert.equal(emptyResponse.exitCode, 0); + assert.equal(emptyResponse.envelope.ok, true); + assert.deepEqual(emptyResponse.envelope.result.events, []); +}); + +test('parse-output handles spec-compliant ACP content blocks and thought chunks', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'kiro', + stdout: kiroFixture('thought.jsonl'), + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.deepEqual(response.envelope.result.events, [ + { type: 'thinking', text: 'Need a plan' }, + { type: 'thinking', text: ' first' }, + { type: 'text', text: 'Done' }, + { + type: 'result', + success: true, + result: 'Done', + error: null, + cost: null, + modelUsage: null, + }, + ]); +}); + +test('parse-output accumulates ACP delta chunks into the terminal result', () => { + const stdout = [ + JSON.stringify({ + jsonrpc: '2.0', + method: 'session/update', + params: { + sessionId: 'kiro-session-1', + update: { + sessionUpdate: 'agent_message_chunk', + messageId: 'msg-delta-1', + content: { type: 'text', text: 'Hello' }, + }, + }, + }), + JSON.stringify({ + jsonrpc: '2.0', + method: 'session/update', + params: { + sessionId: 'kiro-session-1', + update: { + sessionUpdate: 'agent_message_chunk', + messageId: 'msg-delta-1', + content: { type: 'text', text: ' world' }, + }, + }, + }), + JSON.stringify({ + jsonrpc: '2.0', + result: { + stopReason: 'end_turn', + }, + }), + ].join('\n'); + + const response = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'kiro', + stdout, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.deepEqual(response.envelope.result.events, [ + { type: 'text', text: 'Hello' }, + { type: 'text', text: ' world' }, + { + type: 'result', + success: true, + result: 'Hello world', + error: null, + cost: null, + modelUsage: null, + }, + ]); +}); + +test('classify-error maps Pi auth, rate-limit, cancellation, and command failures', () => { + for (const [message, expectedCategory, expectedRetryable] of [ + ['authentication required: run /login', 'auth', false], + ['rate limit exceeded; retry later', 'rate-limit', true], + ['cancelled by user', 'permanent', false], + ['Unknown option --bogus', 'permanent', false], + ]) { + const response = runExecutable({ + schemaVersion: 1, + command: 'classify-error', + provider: 'pi', + error: { message }, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.category, expectedCategory); + assert.equal(response.envelope.result.classification.retryable, expectedRetryable); + } +}); + +test('classify-error maps Kiro auth, retryable, cancellation, and malformed ACP failures', () => { + for (const [message, expectedCategory, expectedRetryable] of [ + ['authentication required: run kiro auth login', 'auth', false], + ['rate limit exceeded; retry later', 'rate-limit', true], + ['cancelled by user', 'permanent', false], + ['malformed ACP response', 'schema', false], + ]) { + const response = runExecutable({ + schemaVersion: 1, + command: 'classify-error', + provider: 'kiro', + error: { message }, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.category, expectedCategory); + assert.equal(response.envelope.result.classification.retryable, expectedRetryable); + } +}); + +test('parse-output normalizes Copilot text and tool JSONL fixtures', () => { + const textResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'copilot', + stdout: copilotFixture('text.jsonl'), + }); + + assert.equal(textResponse.exitCode, 0); + assert.equal(textResponse.envelope.ok, true); + assert.deepEqual(textResponse.envelope.result.events, [ + { type: 'text', text: 'pong' }, + { + type: 'result', + success: true, + result: 'pong', + error: null, + inputTokens: 0, + outputTokens: 21, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + }, + ]); + + const toolResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'copilot', + stdout: copilotFixture('tool.jsonl'), + }); + + assert.equal(toolResponse.exitCode, 0); + assert.equal(toolResponse.envelope.ok, true); + assert.deepEqual(toolResponse.envelope.result.events, [ + // `phase:"commentary"` narration is surfaced as thinking, not as the answer text. + { type: 'thinking', text: 'Creating hello.txt now.' }, + { + type: 'tool_call', + toolName: 'apply_patch', + toolId: 'call_1', + input: '*** Begin Patch\n*** Add File: hello.txt\n+hi there\n*** End Patch\n', + }, + { + type: 'tool_result', + toolId: 'call_1', + content: 'Added 1 file(s): hello.txt', + isError: false, + }, + { type: 'text', text: 'Done. It printed: hi there' }, + { + type: 'result', + success: true, + result: 'Done. It printed: hi there', + error: null, + inputTokens: 0, + outputTokens: 223, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + }, + ]); +}); + +test('parse-output passes object-form Copilot tool arguments through unchanged', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'copilot', + stdout: copilotFixture('tool-object-args.jsonl'), + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + const events = response.envelope.result.events; + // Real copilot tool arguments are string-form for some tools (apply_patch) and object-form for + // others (bash); the parser must forward both without reshaping. + assert.deepEqual( + events.find((event) => event.type === 'tool_call'), + { + type: 'tool_call', + toolName: 'bash', + toolId: 'call_9', + input: { command: 'cat hello.txt' }, + } + ); + assert.deepEqual( + events.find((event) => event.type === 'tool_result'), + { + type: 'tool_result', + toolId: 'call_9', + content: 'hi there\n', + isError: false, + } + ); +}); + +test('parse-output ignores unknown Copilot event types (fail-open)', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'copilot', + stdout: copilotFixture('unknown-event.jsonl'), + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.deepEqual(response.envelope.result.events, [ + { type: 'text', text: 'partial' }, + { + type: 'result', + success: true, + result: 'partial', + error: null, + inputTokens: 0, + outputTokens: 3, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + }, + ]); +}); + +test('parse-output normalizes Copilot reasoning as a thinking event', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'copilot', + stdout: copilotFixture('reasoning.jsonl'), + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.deepEqual(response.envelope.result.events, [ + { type: 'thinking', text: 'Let me think about this.' }, + { type: 'text', text: 'answer' }, + { + type: 'result', + success: true, + result: 'answer', + error: null, + inputTokens: 0, + outputTokens: 5, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + }, + ]); +}); + +test('parse-output reports a non-zero Copilot exitCode as a failed result', () => { + const response = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'copilot', + stdout: copilotFixture('failure.jsonl'), + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + // Success is derived from the terminal `result` event's top-level exitCode, NOT from any message + // text — a non-zero exit must surface as a failed result so restart/resume logic can react. + assert.deepEqual(response.envelope.result.events.at(-1), { + type: 'result', + success: false, + result: null, + error: 'Copilot exited with code 1', + inputTokens: 0, + outputTokens: 9, + cacheReadInputTokens: 0, + cacheCreationInputTokens: 0, + }); + + const emptyResponse = runExecutable({ + schemaVersion: 1, + command: 'parse-output', + provider: 'copilot', + stdout: copilotFixture('empty.jsonl'), + }); + + assert.equal(emptyResponse.exitCode, 0); + assert.equal(emptyResponse.envelope.ok, true); + assert.deepEqual(emptyResponse.envelope.result.events, []); +}); + +test('classify-error maps Copilot auth, rate-limit, quota, and command failures', () => { + for (const [message, expectedCategory, expectedRetryable] of [ + ['authentication failed: set GITHUB_TOKEN or run /login', 'auth', false], + ['rate limit exceeded; retry later', 'rate-limit', true], + ['insufficient quota', 'rate-limit', false], + ['Unknown option --bogus', 'permanent', false], + ]) { + const response = runExecutable({ + schemaVersion: 1, + command: 'classify-error', + provider: 'copilot', + error: { message }, + }); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.category, expectedCategory); + assert.equal(response.envelope.result.classification.retryable, expectedRetryable); + } +}); diff --git a/tests/agent-cli-provider/executable-contract-request-errors.test.js b/tests/agent-cli-provider/executable-contract-request-errors.test.js index 6b76eedc..d2c24235 100644 --- a/tests/agent-cli-provider/executable-contract-request-errors.test.js +++ b/tests/agent-cli-provider/executable-contract-request-errors.test.js @@ -55,6 +55,10 @@ test('unknown provider returns structured provider error envelope', () => { assert.equal(response.exitCode, 4); assert.equal(response.envelope.ok, false); assert.equal(response.envelope.error.code, 'unknown-provider'); + assert.equal( + response.envelope.error.message, + 'Unknown provider: unknown. Valid: claude, codex, gateway, gemini, opencode, pi, kiro, copilot' + ); }); test('unknown provider redacts env-matched provider in error envelope', () => { diff --git a/tests/agent-cli-provider/gateway-runner.test.js b/tests/agent-cli-provider/gateway-runner.test.js new file mode 100644 index 00000000..2cacf45c --- /dev/null +++ b/tests/agent-cli-provider/gateway-runner.test.js @@ -0,0 +1,839 @@ +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); +const { test } = require('node:test'); + +const helper = require('../../lib/agent-cli-provider'); +const { runGatewayRequest } = require('../../lib/agent-cli-provider/gateway-runner'); +const { executeGatewayToolCall } = require('../../lib/agent-cli-provider/gateway-tools'); +const { + assertNoSecret, + runProviderExecutable, +} = require('./executable-contract-helpers.cjs'); + +async function withGatewayServer(handler, fn) { + const server = http.createServer(async (req, res) => { + const body = await readRequestBody(req); + const json = body ? JSON.parse(body) : null; + await handler(req, res, json); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + const baseUrl = `http://127.0.0.1:${address.port}`; + + try { + return await fn(baseUrl); + } finally { + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + } +} + +function readRequestBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + req.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))); + req.on('error', reject); + }); +} + +function jsonResponse(res, statusCode, body) { + res.writeHead(statusCode, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(body)); +} + +test('gateway invoke completes deterministic edit task', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-runner-')); + const targetFile = path.join(tempDir, 'note.txt'); + fs.writeFileSync(targetFile, 'before\n', 'utf8'); + + try { + let requestCount = 0; + const response = await withGatewayServer((_req, res) => { + requestCount += 1; + if (requestCount === 1) { + jsonResponse(res, 200, { + choices: [ + { + message: { + content: 'Reading the target file.', + tool_calls: [ + { + id: 'tool-read', + type: 'function', + function: { + name: 'read_file', + arguments: JSON.stringify({ path: 'note.txt' }), + }, + }, + ], + }, + }, + ], + }); + return; + } + if (requestCount === 2) { + jsonResponse(res, 200, { + choices: [ + { + message: { + content: 'Applying the edit.', + tool_calls: [ + { + id: 'tool-write', + type: 'function', + function: { + name: 'apply_patch', + arguments: JSON.stringify({ + path: 'note.txt', + search: 'before\n', + replace: 'after\n', + }), + }, + }, + ], + }, + }, + ], + }); + return; + } + jsonResponse(res, 200, { + choices: [ + { + message: { + content: 'Task complete.', + }, + }, + ], + }); + }, (baseUrl) => + runProviderExecutable( + JSON.stringify({ + schemaVersion: 1, + command: 'invoke', + provider: 'gateway', + context: 'Update note.txt from before to after.', + options: { + cwd: tempDir, + gateway: { + baseUrl, + apiKey: 'gateway-test-key', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: ['node'], + }, + }, + }, + timeoutMs: 2_000, + }) + ) + ); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(fs.readFileSync(targetFile, 'utf8'), 'after\n'); + assert.deepEqual( + response.envelope.result.events.map((event) => event.type), + ['text', 'tool_call', 'tool_result', 'text', 'tool_call', 'tool_result', 'text', 'result'] + ); + assert.equal(response.envelope.result.events.at(-1).success, true); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway rejects invalid config as permanent failures', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-invalid-')); + + try { + for (const gateway of [ + { + baseUrl: 'not-a-url', + apiKey: 'secret', + model: 'test-model', + toolPolicy: { roots: ['.'], commands: [] }, + }, + { + baseUrl: 'http://127.0.0.1:1', + apiKey: 'secret', + model: '', + toolPolicy: { roots: ['.'], commands: [] }, + }, + ]) { + const events = await runGatewayRequest({ + context: 'noop', + cwd: tempDir, + gateway, + }); + const result = events.at(-1); + assert.equal(result.type, 'result'); + assert.equal(result.success, false); + assert.equal( + helper.classifyProviderError('gateway', { message: result.error }).retryable, + false + ); + } + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway redacts auth failures', async () => { + const secret = 'gateway-auth-secret'; + const response = await withGatewayServer( + (_req, res) => { + jsonResponse(res, 401, { + error: { + message: 'Unauthorized', + }, + }); + }, + (baseUrl) => + runProviderExecutable( + JSON.stringify({ + schemaVersion: 1, + command: 'invoke', + provider: 'gateway', + context: 'Reply with ok.', + options: { + gateway: { + baseUrl, + apiKey: secret, + model: 'openrouter/test-model', + toolPolicy: { + roots: [process.cwd()], + commands: [], + }, + }, + }, + }) + ) + ); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.classification.retryable, false); + assertNoSecret(response.envelope, secret); +}); + +test('gateway classifies remote invalid-model 404 failures as permanent', async () => { + const events = await withGatewayServer( + (_req, res) => { + jsonResponse(res, 404, { + error: { + message: 'No such model', + }, + }); + }, + (baseUrl) => + runGatewayRequest({ + context: 'Reply with ok.', + cwd: process.cwd(), + gateway: { + baseUrl, + apiKey: 'gateway-test-key', + model: 'openrouter/missing-model', + toolPolicy: { + roots: [process.cwd()], + commands: [], + }, + }, + }) + ); + + const result = events.at(-1); + assert.equal(result.type, 'result'); + assert.equal(result.success, false); + assert.match(result.error, /status 404: No such model/); + assert.equal( + helper.classifyProviderError('gateway', { message: result.error }).retryable, + false + ); +}); + +test('gateway requires explicit tool policy before invoke', async () => { + let runnerCalled = false; + const response = await helper.runProviderExecutable( + JSON.stringify({ + schemaVersion: 1, + command: 'invoke', + provider: 'gateway', + context: 'noop', + options: { + gateway: { + baseUrl: 'http://127.0.0.1:4000', + apiKey: 'secret', + model: 'test-model', + }, + }, + }), + { + runner: () => { + runnerCalled = true; + return { + stdout: '', + stderr: '', + exitCode: 0, + signal: null, + durationMs: 1, + }; + }, + } + ); + + assert.equal(response.exitCode, 2); + assert.equal(response.envelope.ok, false); + assert.equal(response.envelope.error.field, 'options.gateway.toolPolicy'); + assert.equal(runnerCalled, false); +}); + +test('gateway blocks disallowed command before execution', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-policy-')); + const marker = path.join(tempDir, 'marker'); + + try { + let requestCount = 0; + const events = await withGatewayServer( + (_req, res) => { + requestCount += 1; + if (requestCount === 1) { + jsonResponse(res, 200, { + choices: [ + { + message: { + content: 'Try a blocked command.', + tool_calls: [ + { + id: 'tool-run', + type: 'function', + function: { + name: 'run_command', + arguments: JSON.stringify({ + command: 'node', + args: ['-e', `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'x')`], + cwd: '.', + }), + }, + }, + ], + }, + }, + ], + }); + return; + } + throw new Error('gateway should stop after the rejected command'); + }, + (baseUrl) => + runGatewayRequest({ + context: 'noop', + cwd: tempDir, + gateway: { + baseUrl, + apiKey: 'secret', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: [], + }, + }, + }) + ); + + const result = events.at(-1); + assert.equal(fs.existsSync(marker), false); + assert.equal(requestCount, 1); + assert.deepEqual( + events.map((event) => event.type), + ['text', 'tool_call', 'tool_result', 'result'] + ); + assert.equal(events.find((event) => event.type === 'tool_result')?.isError, true); + assert.equal(result.type, 'result'); + assert.equal(result.success, false); + assert.equal( + helper.classifyProviderError('gateway', { message: result.error }).retryable, + false + ); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway stops remaining tool calls in a batch after the first tool error', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-batch-stop-')); + const marker = path.join(tempDir, 'marker.txt'); + + try { + let requestCount = 0; + const events = await withGatewayServer((_req, res) => { + requestCount += 1; + if (requestCount === 1) { + jsonResponse(res, 200, { + choices: [ + { + message: { + content: 'Run the blocked command, then patch the file.', + tool_calls: [ + { + id: 'tool-run', + type: 'function', + function: { + name: 'run_command', + arguments: JSON.stringify({ + command: 'node', + args: ['-e', `require('node:fs').writeFileSync(${JSON.stringify(marker)}, 'x')`], + cwd: '.', + }), + }, + }, + { + id: 'tool-write', + type: 'function', + function: { + name: 'apply_patch', + arguments: JSON.stringify({ + path: 'marker.txt', + content: 'should-not-exist\n', + }), + }, + }, + ], + }, + }, + ], + }); + return; + } + throw new Error('gateway should stop after the first tool error in a batch'); + }, (baseUrl) => + runGatewayRequest({ + context: 'noop', + cwd: tempDir, + gateway: { + baseUrl, + apiKey: 'secret', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: [], + }, + }, + }) + ); + + assert.equal(requestCount, 1); + assert.equal(fs.existsSync(marker), false); + assert.deepEqual( + events.map((event) => event.type), + ['text', 'tool_call', 'tool_result', 'result'] + ); + const toolCalls = events.filter((event) => event.type === 'tool_call'); + assert.equal(toolCalls.length, 1); + assert.equal(toolCalls[0].toolId, 'tool-run'); + assert.equal(events.at(-1).success, false); + assert.match(events.at(-1).error, /cannot verify completion/); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway preserves prior normalized events when a later gateway request fails', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-midrun-failure-')); + const targetFile = path.join(tempDir, 'note.txt'); + fs.writeFileSync(targetFile, 'before\n', 'utf8'); + + try { + let requestCount = 0; + const events = await withGatewayServer((_req, res) => { + requestCount += 1; + if (requestCount === 1) { + jsonResponse(res, 200, { + choices: [ + { + message: { + content: 'Read the file first.', + tool_calls: [ + { + id: 'tool-read', + type: 'function', + function: { + name: 'read_file', + arguments: JSON.stringify({ path: 'note.txt' }), + }, + }, + ], + }, + }, + ], + }); + return; + } + res.writeHead(500, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: { message: 'boom' } })); + }, (baseUrl) => + runGatewayRequest({ + context: 'noop', + cwd: tempDir, + gateway: { + baseUrl, + apiKey: 'secret', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: [], + }, + }, + }) + ); + + assert.deepEqual( + events.map((event) => event.type), + ['text', 'tool_call', 'tool_result', 'result'] + ); + assert.equal(events.at(-1).success, false); + assert.match(events.at(-1).error, /status 500: boom/); + assert.equal(fs.readFileSync(targetFile, 'utf8'), 'before\n'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway emits tool_result for malformed tool arguments before failing the run', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-malformed-tool-')); + + try { + let requestCount = 0; + const events = await withGatewayServer((_req, res) => { + requestCount += 1; + if (requestCount === 1) { + jsonResponse(res, 200, { + choices: [ + { + message: { + content: 'Run the malformed tool call.', + tool_calls: [ + { + id: 'tool-read', + type: 'function', + function: { + name: 'read_file', + arguments: '{"path":', + }, + }, + ], + }, + }, + ], + }); + return; + } + throw new Error('gateway should stop after malformed tool arguments'); + }, (baseUrl) => + runGatewayRequest({ + context: 'noop', + cwd: tempDir, + gateway: { + baseUrl, + apiKey: 'secret', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: [], + }, + }, + }) + ); + + assert.deepEqual( + events.map((event) => event.type), + ['text', 'tool_call', 'tool_result', 'result'] + ); + assert.equal(requestCount, 1); + assert.equal(events.find((event) => event.type === 'tool_result')?.isError, true); + assert.match( + events.find((event) => event.type === 'tool_result').content.message, + /malformed JSON arguments/ + ); + assert.equal(events.at(-1).success, false); + assert.match(events.at(-1).error, /malformed JSON arguments/); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway returns failure result after tool errors even if the model stops', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-tool-error-')); + + try { + let requestCount = 0; + const events = await withGatewayServer((_req, res) => { + requestCount += 1; + if (requestCount === 1) { + jsonResponse(res, 200, { + choices: [ + { + message: { + content: 'Reading a missing file.', + tool_calls: [ + { + id: 'tool-read', + type: 'function', + function: { + name: 'read_file', + arguments: JSON.stringify({ path: 'missing.txt' }), + }, + }, + ], + }, + }, + ], + }); + return; + } + jsonResponse(res, 200, { + choices: [ + { + message: { + content: 'Done.', + }, + }, + ], + }); + }, (baseUrl) => + runGatewayRequest({ + context: 'noop', + cwd: tempDir, + gateway: { + baseUrl, + apiKey: 'secret', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: [], + }, + }, + }) + ); + + assert.equal(events.find((event) => event.type === 'tool_result')?.isError, true); + const result = events.at(-1); + assert.equal(result.type, 'result'); + assert.equal(result.success, false); + assert.match(result.error, /tool failure/i); + assert.match(result.error, /missing\.txt/); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway apply_patch rejects malformed fields before editing files', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-apply-patch-')); + const targetFile = path.join(tempDir, 'note.txt'); + fs.writeFileSync(targetFile, 'x\nx\n', 'utf8'); + + try { + await assert.rejects( + executeGatewayToolCall( + 'apply_patch', + { path: 'note.txt', content: 123 }, + { roots: [tempDir], commands: [] } + ), + /apply_patch\.content must be a string/ + ); + assert.equal(fs.readFileSync(targetFile, 'utf8'), 'x\nx\n'); + + await assert.rejects( + executeGatewayToolCall( + 'apply_patch', + { path: 'note.txt', search: 'x', replace: 'y', replaceAll: 'false' }, + { roots: [tempDir], commands: [] } + ), + /apply_patch\.replaceAll must be a boolean/ + ); + assert.equal(fs.readFileSync(targetFile, 'utf8'), 'x\nx\n'); + + await assert.rejects( + executeGatewayToolCall( + 'apply_patch', + { path: 'note.txt', search: '', replace: 'y' }, + { roots: [tempDir], commands: [] } + ), + /apply_patch\.search must be a non-empty string/ + ); + assert.equal(fs.readFileSync(targetFile, 'utf8'), 'x\nx\n'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway rejects symlink escapes for file writes and command cwd', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-symlink-')); + const rootDir = path.join(tempDir, 'root'); + const outsideDir = path.join(tempDir, 'outside'); + const outsideFile = path.join(outsideDir, 'outside.txt'); + const outsideWrite = path.join(outsideDir, 'written.txt'); + const outsideMarker = path.join(outsideDir, 'marker.txt'); + fs.mkdirSync(rootDir, { recursive: true }); + fs.mkdirSync(outsideDir, { recursive: true }); + fs.writeFileSync(outsideFile, 'outside\n', 'utf8'); + fs.symlinkSync(outsideFile, path.join(rootDir, 'link.txt')); + fs.symlinkSync(outsideDir, path.join(rootDir, 'linkdir')); + + try { + await assert.rejects( + executeGatewayToolCall( + 'read_file', + { path: 'link.txt' }, + { roots: [rootDir], commands: [] } + ), + /toolPolicy\.roots/ + ); + + await assert.rejects( + executeGatewayToolCall( + 'apply_patch', + { path: 'linkdir/written.txt', content: 'escaped\n' }, + { roots: [rootDir], commands: [] } + ), + /toolPolicy\.roots/ + ); + + await assert.rejects( + executeGatewayToolCall( + 'run_command', + { + command: process.execPath, + args: [ + '-e', + `require('node:fs').writeFileSync(${JSON.stringify(outsideMarker)}, 'x', 'utf8')`, + ], + cwd: 'linkdir', + }, + { roots: [rootDir], commands: [process.execPath] } + ), + /toolPolicy\.roots/ + ); + + assert.equal(fs.existsSync(outsideWrite), false); + assert.equal(fs.existsSync(outsideMarker), false); + assert.equal(fs.readFileSync(outsideFile, 'utf8'), 'outside\n'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway resolves relative tool paths against later allowlisted roots when needed', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-multi-root-')); + const firstRoot = path.join(tempDir, 'first'); + const secondRoot = path.join(tempDir, 'second'); + const secondSubdir = path.join(secondRoot, 'subdir'); + const targetFile = path.join(secondRoot, 'file.txt'); + const markerFile = path.join(secondSubdir, 'marker.txt'); + fs.mkdirSync(firstRoot, { recursive: true }); + fs.mkdirSync(secondSubdir, { recursive: true }); + fs.writeFileSync(targetFile, 'before\n', 'utf8'); + + try { + const readResult = await executeGatewayToolCall( + 'read_file', + { path: 'file.txt' }, + { roots: [firstRoot, secondRoot], commands: [] } + ); + assert.equal(readResult.isError, false); + assert.equal(fs.realpathSync(readResult.content.path), fs.realpathSync(targetFile)); + assert.equal(readResult.content.content, 'before\n'); + + const patchResult = await executeGatewayToolCall( + 'apply_patch', + { path: 'file.txt', search: 'before\n', replace: 'after\n' }, + { roots: [firstRoot, secondRoot], commands: [] } + ); + assert.equal(patchResult.isError, false); + assert.equal(fs.readFileSync(targetFile, 'utf8'), 'after\n'); + assert.equal(fs.realpathSync(patchResult.content.path), fs.realpathSync(targetFile)); + + const commandResult = await executeGatewayToolCall( + 'run_command', + { + command: process.execPath, + args: ['-e', "require('node:fs').writeFileSync('marker.txt', 'ok\\n', 'utf8')"], + cwd: 'subdir', + }, + { roots: [firstRoot, secondRoot], commands: [process.execPath] } + ); + assert.equal(commandResult.isError, false); + assert.equal(fs.realpathSync(commandResult.content.cwd), fs.realpathSync(secondSubdir)); + assert.equal(fs.readFileSync(markerFile, 'utf8'), 'ok\n'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway run_command without cwd executes once in the first allowlisted root', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-multi-root-command-')); + const firstRoot = path.join(tempDir, 'first'); + const secondRoot = path.join(tempDir, 'second'); + const sharedLog = path.join(tempDir, 'shared.log'); + const targetFile = path.join(secondRoot, 'note.txt'); + fs.mkdirSync(firstRoot, { recursive: true }); + fs.mkdirSync(secondRoot, { recursive: true }); + fs.writeFileSync(targetFile, 'from-second-root\n', 'utf8'); + + try { + const commandResult = await executeGatewayToolCall( + 'run_command', + { + command: process.execPath, + args: [ + '-e', + `const fs=require('node:fs');fs.appendFileSync(${JSON.stringify(sharedLog)},'x');process.stdout.write(fs.readFileSync('note.txt','utf8'))`, + ], + }, + { roots: [firstRoot, secondRoot], commands: [process.execPath] } + ); + assert.equal(commandResult.isError, true); + assert.equal(fs.realpathSync(commandResult.content.cwd), fs.realpathSync(firstRoot)); + assert.match(commandResult.content.stderr, /note\.txt/); + assert.equal(fs.readFileSync(sharedLog, 'utf8'), 'x'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +test('gateway run_command rejects malformed args before execution', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-run-command-')); + const targetFile = path.join(tempDir, 'ran.txt'); + + try { + await assert.rejects( + executeGatewayToolCall( + 'run_command', + { command: 'node', args: '-e', cwd: '.' }, + { roots: [tempDir], commands: ['node'] } + ), + /run_command\.args must be an array/ + ); + + await assert.rejects( + executeGatewayToolCall( + 'run_command', + { + command: process.execPath, + args: ['-e', `require('fs').writeFileSync(${JSON.stringify(targetFile)},'ok')`], + cwd: 123, + }, + { roots: [tempDir], commands: [process.execPath] } + ), + /run_command\.cwd must be a string/ + ); + assert.equal(fs.existsSync(targetFile), false); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); diff --git a/tests/agent-cli-provider/parity.test.js b/tests/agent-cli-provider/parity.test.js index 22b8cbb8..4852a330 100644 --- a/tests/agent-cli-provider/parity.test.js +++ b/tests/agent-cli-provider/parity.test.js @@ -1,10 +1,18 @@ const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const { afterEach, test } = require('node:test'); const helper = require('../../lib/agent-cli-provider'); +const { ENV_PRESETS, MOUNT_PRESETS } = require('../../lib/docker-config'); +const { validateSetting } = require('../../lib/settings'); +const { + KNOWN_PROVIDER_NAMES, + VALID_PROVIDERS, + normalizeProviderName, +} = require('../../lib/provider-names'); const runtimeProviders = require('../../src/providers'); const createdTempFiles = new Set(); @@ -115,11 +123,138 @@ test('runtime Opencode command facade delegates to helper', () => { cliFeatures: { supportsJson: true, supportsVariant: true, + supportsDir: true, supportsCwd: true, }, }); }); +test('runtime Pi command facade delegates to helper', () => { + assertRuntimeCommandParity('pi', 'pi context', { + outputFormat: 'json', + jsonSchema: { type: 'object', properties: { ok: { type: 'boolean' } } }, + cwd: '/tmp/project', + modelSpec: { level: 'level2', model: 'openai/gpt-5.5' }, + cliFeatures: { + supportsJsonMode: true, + supportsNoSession: true, + supportsNoExtensions: true, + supportsNoSkills: true, + supportsNoPromptTemplates: true, + supportsNoContextFiles: true, + supportsNoApprove: true, + supportsModel: true, + }, + }); +}); + +test('runtime Copilot command facade delegates to helper', () => { + assertRuntimeCommandParity('copilot', 'copilot context', { + outputFormat: 'json', + autoApprove: true, + jsonSchema: { type: 'object', properties: { ok: { type: 'boolean' } } }, + cwd: '/tmp/project', + modelSpec: { level: 'level2', model: 'gpt-5.2' }, + cliFeatures: { + supportsJsonOutput: true, + supportsModel: true, + supportsAllowAll: true, + supportsNoAskUser: true, + supportsAddDir: true, + }, + }); +}); + +test('gateway availability and cli path use the bundled node runtime, not PATH lookup', async () => { + const settingsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-provider-')); + const settingsFile = path.join(settingsDir, 'settings.json'); + const originalPath = process.env.PATH; + const originalSettingsFile = process.env.ZEROSHOT_SETTINGS_FILE; + + fs.writeFileSync( + settingsFile, + JSON.stringify( + { + defaultProvider: 'gateway', + providerSettings: { + gateway: { + baseUrl: 'http://127.0.0.1:11434/v1', + apiKey: 'gateway-key', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: ['node'], + }, + }, + }, + }, + null, + 2 + ) + ); + + process.env.ZEROSHOT_SETTINGS_FILE = settingsFile; + process.env.PATH = '/nonexistent'; + + try { + const detected = await runtimeProviders.detectProviders(); + assert.equal(detected.gateway.available, true); + assert.equal(runtimeProviders.getProvider('gateway').getCliPath(), process.execPath); + } finally { + process.env.PATH = originalPath; + if (originalSettingsFile === undefined) { + delete process.env.ZEROSHOT_SETTINGS_FILE; + } else { + process.env.ZEROSHOT_SETTINGS_FILE = originalSettingsFile; + } + fs.rmSync(settingsDir, { recursive: true, force: true }); + } +}); + +test('gateway provider discovery fails closed on malformed gateway settings', () => { + const settingsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-gateway-provider-invalid-')); + const settingsFile = path.join(settingsDir, 'settings.json'); + + fs.writeFileSync( + settingsFile, + JSON.stringify( + { + defaultProvider: 'gateway', + providerSettings: { + gateway: { + toolPolicy: 'bad', + }, + }, + }, + null, + 2 + ) + ); + + try { + const child = spawnSync( + process.execPath, + [ + '-e', + "require('./src/providers').detectProviders().then((result) => process.stdout.write(JSON.stringify(result.gateway)))", + ], + { + cwd: path.join(__dirname, '..', '..'), + env: { + ...process.env, + ZEROSHOT_SETTINGS_FILE: settingsFile, + }, + encoding: 'utf8', + } + ); + + assert.equal(child.status, 0, child.stderr); + assert.deepEqual(JSON.parse(child.stdout), { available: false }); + } finally { + fs.rmSync(settingsDir, { recursive: true, force: true }); + } +}); + test('Codex helper exposes strict schema cleanup metadata through runtime facade', () => { const actual = runtimeProviders.getProvider('codex').buildCommand('schema context', { outputFormat: 'json', @@ -155,6 +290,14 @@ test('model resolution and invalid-model permanence match helper', () => { current.resolveModelSpec('level2', { level2: { model: '' } }) ); + if (provider === 'pi' || provider === 'copilot' || provider === 'gateway') { + assert.deepEqual( + helper.resolveModelSpec(provider, 'level2', { level2: { model: 'invalid' } }), + current.resolveModelSpec('level2', { level2: { model: 'invalid' } }) + ); + continue; + } + assert.throws( () => helper.resolveModelSpec(provider, 'level2', { level2: { model: 'invalid' } }), { permanent: true } @@ -195,6 +338,19 @@ test('parser output from runtime facade matches helper fixtures', () => { for (const [provider, files] of [ ['codex', ['text.jsonl', 'tool.jsonl']], ['gemini', ['text.jsonl', 'tool.jsonl']], + [ + 'kiro', + [ + 'text.jsonl', + 'tool.jsonl', + 'auth-failure.jsonl', + 'cancelled.jsonl', + 'empty.jsonl', + 'malformed.jsonl', + ], + ], + ['pi', ['text.jsonl', 'tool.jsonl', 'command-failure.jsonl']], + ['copilot', ['text.jsonl', 'tool.jsonl', 'unknown-event.jsonl']], ]) { for (const file of files) { const chunk = fixture(provider, file); @@ -285,7 +441,134 @@ test('feature probing is deterministic from injected help text', () => { assert.equal( helper .getProviderAdapter('opencode') - .detectCliFeatures('opencode run --format --model --variant').supportsCwd, + .detectCliFeatures('opencode run --format --model --variant --dir --cwd').supportsDir, + true + ); + assert.equal( + helper.getProviderAdapter('opencode').detectCliFeatures('opencode run --format').supportsCwd, false ); + assert.equal( + helper + .getProviderAdapter('pi') + .detectCliFeatures( + 'pi --mode json --no-session --no-extensions --no-skills --no-prompt-templates --no-context-files --no-approve --model' + ).supportsNoApprove, + true + ); + assert.equal( + helper + .getProviderAdapter('copilot') + .detectCliFeatures( + 'copilot -p --output-format json --model --allow-all --no-ask-user --add-dir' + ).supportsAllowAll, + true + ); + assert.deepEqual(helper.getProviderAdapter('kiro').detectCliFeatures('kiro-cli acp --help'), { + provider: 'kiro', + supportsAcpStdio: true, + supportsPromptImages: true, + supportsLoadSession: false, + supportsSessionCancel: true, + supportsSessionSetModel: false, + supportsSessionSetMode: false, + supportsRemoteTransport: false, + supportsCustomTransport: false, + supportsPermissionRequests: false, + supportsFsTools: false, + supportsTerminalTools: false, + unknown: false, + }); + assert.deepEqual(helper.getProviderAdapter('gateway').detectCliFeatures(''), { + provider: 'gateway', + supportsBundledRunner: true, + unknown: false, + }); +}); + +test('provider registry stays in parity across helper runtime settings and probe contract', async () => { + assert.deepEqual(helper.listProviderAdapters(), VALID_PROVIDERS); + assert.deepEqual(runtimeProviders.listProviders(), VALID_PROVIDERS); + assert.deepEqual( + helper.listProviderRegistryEntries().map((entry) => entry.id), + VALID_PROVIDERS + ); + assert.deepEqual( + KNOWN_PROVIDER_NAMES.map((name) => normalizeProviderName(name)), + KNOWN_PROVIDER_NAMES.map((name) => helper.normalizeProviderName(name)) + ); + + for (const provider of VALID_PROVIDERS) { + const metadata = helper.getProviderRegistryEntry(provider); + const runtime = runtimeProviders.getProvider(provider); + assert.equal(runtime.displayName, metadata.displayName); + assert.deepEqual(runtime.getCredentialPaths(), metadata.credentialPaths); + assert.deepEqual(runtime.getSettingsFields().slice(4), metadata.settingsFields); + + const response = await helper.runProviderExecutable( + JSON.stringify({ + schemaVersion: 1, + command: 'probe', + provider, + helpText: '', + }), + { + runner: async () => { + await Promise.resolve(); + return { + stdout: '', + stderr: '', + exitCode: 0, + signal: null, + durationMs: 1, + }; + }, + } + ); + + assert.equal(response.exitCode, 0); + assert.equal(response.envelope.ok, true); + assert.equal(response.envelope.result.provider.id, provider); + assert.equal(response.envelope.result.provider.displayName, metadata.displayName); + assert.deepEqual( + response.envelope.result.credentials.map((credential) => credential.key), + metadata.credentialEnvKeys + ); + } + + assert.equal(validateSetting('defaultProvider', 'openai'), null); + assert.equal( + validateSetting('defaultProvider', 'invalid-provider'), + `Invalid provider: invalid-provider. Valid providers: ${VALID_PROVIDERS.join(', ')}` + ); + assert.equal( + validateSetting('providerSettings', { + openai: { defaultLevel: 'level2', levelOverrides: {} }, + }), + null + ); + assert.equal( + validateSetting('providerSettings', { + gateway: { + defaultLevel: 'level2', + levelOverrides: {}, + baseUrl: 'http://127.0.0.1:11434', + apiKey: 'gateway-key', + model: 'openrouter/test-model', + toolPolicy: { roots: ['.'], commands: ['node'] }, + }, + }), + null + ); + assert.equal( + validateSetting('providerSettings', { + 'invalid-provider': { defaultLevel: 'level2', levelOverrides: {} }, + }), + `Unknown provider in providerSettings: invalid-provider. Valid providers: ${VALID_PROVIDERS.join(', ')}` + ); + + for (const metadata of helper.listProviderRegistryEntries()) { + assert.deepEqual(MOUNT_PRESETS[metadata.id], metadata.docker.mount); + assert.deepEqual(ENV_PRESETS[metadata.id], metadata.docker.envPassthrough); + } }); diff --git a/tests/agent-cli-provider/providers-command-parity.test.js b/tests/agent-cli-provider/providers-command-parity.test.js new file mode 100644 index 00000000..4ef1b677 --- /dev/null +++ b/tests/agent-cli-provider/providers-command-parity.test.js @@ -0,0 +1,146 @@ +const assert = require('node:assert/strict'); +const readline = require('node:readline'); +const { afterEach, test } = require('node:test'); + +const helper = require('../../lib/agent-cli-provider'); + +const providersModulePath = require.resolve('../../src/providers'); +const settingsModulePath = require.resolve('../../lib/settings'); +const commandModulePath = require.resolve('../../cli/commands/providers'); + +const originalProvidersModule = require(providersModulePath); +const originalSettingsModule = require(settingsModulePath); +const originalCreateInterface = readline.createInterface; +const originalConsoleLog = console.log; + +afterEach(() => { + require.cache[providersModulePath].exports = originalProvidersModule; + require.cache[settingsModulePath].exports = originalSettingsModule; + delete require.cache[commandModulePath]; + readline.createInterface = originalCreateInterface; + console.log = originalConsoleLog; +}); + +function loadCommands({ detected, providerFactory, settings, saveSettings = () => {} }) { + require.cache[providersModulePath].exports = { + ...originalProvidersModule, + detectProviders: async () => { + await Promise.resolve(); + return detected; + }, + getProvider: providerFactory, + }; + require.cache[settingsModulePath].exports = { + ...originalSettingsModule, + loadSettings: () => settings, + saveSettings, + }; + delete require.cache[commandModulePath]; + return require(commandModulePath); +} + +test('providers command renders rows from registry-backed runtime metadata', async () => { + const settings = { + defaultProvider: 'gemini', + providerSettings: { + claude: { defaultLevel: 'level1', levelOverrides: {} }, + codex: { defaultLevel: 'level3', levelOverrides: {} }, + gemini: { defaultLevel: 'level2', levelOverrides: {} }, + opencode: { defaultLevel: 'level2', levelOverrides: {} }, + pi: { defaultLevel: 'level2', levelOverrides: {} }, + copilot: { defaultLevel: 'level2', levelOverrides: {} }, + }, + }; + const detected = Object.fromEntries( + helper + .listProviderRegistryEntries() + .map((entry, index) => [entry.id, { available: index % 2 === 0 }]) + ); + const logs = []; + console.log = (...args) => logs.push(args.join(' ')); + + const { providersCommand } = loadCommands({ + detected, + settings, + providerFactory: (name) => { + const metadata = helper.getProviderRegistryEntry(name); + return { + displayName: metadata.displayName, + getDefaultLevel: () => metadata.defaultLevels.default, + resolveModelSpec: (level, overrides) => helper.resolveModelSpec(name, level, overrides), + getCliPath: async () => { + await Promise.resolve(); + return `/opt/${name}`; + }, + }; + }, + }); + + await providersCommand(); + + assert.equal(logs[0], '\nProvider Status Default Level Model CLI Path'); + assert.equal(logs[1], '─'.repeat(70)); + + const expectedRows = helper.listProviderRegistryEntries().map((metadata) => { + const providerSettings = settings.providerSettings[metadata.id] || {}; + const defaultLevel = providerSettings.defaultLevel || metadata.defaultLevels.default; + const modelLabel = helper.resolveModelSpec(metadata.id, defaultLevel).model || '-'; + const statusIcon = detected[metadata.id].available ? '✓ found' : '✗ not found'; + const cliPath = detected[metadata.id].available ? `/opt/${metadata.id}` : '-'; + const isDefault = settings.defaultProvider === metadata.id ? ' (default)' : ''; + return `${metadata.displayName.padEnd(12)} ${statusIcon.padEnd(12)} ${defaultLevel.padEnd( + 14 + )} ${modelLabel.padEnd(16)} ${cliPath}${isDefault}`; + }); + + assert.deepEqual(logs.slice(2, 2 + expectedRows.length), expectedRows); +}); + +test('provider setup prints install and auth instructions from registry metadata', async () => { + const provider = 'opencode'; + const metadata = helper.getProviderRegistryEntry(provider); + const logs = []; + console.log = (...args) => logs.push(args.join(' ')); + readline.createInterface = () => ({ + question(_prompt, callback) { + callback(''); + }, + close() {}, + }); + + const settings = { + defaultProvider: 'claude', + providerSettings: {}, + }; + const saved = []; + const { setupCommand } = loadCommands({ + detected: {}, + settings, + saveSettings: (next) => saved.push(next), + providerFactory: () => ({ + displayName: metadata.displayName, + cliCommand: metadata.binary, + isAvailable: async () => { + await Promise.resolve(); + return true; + }, + getAuthInstructions: () => metadata.authInstructions, + getInstallInstructions: () => metadata.installInstructions, + getLevelMapping: () => ({ + level1: { rank: 1, model: null, reasoningEffort: 'low' }, + level2: { rank: 2, model: null, reasoningEffort: 'medium' }, + level3: { rank: 3, model: null, reasoningEffort: 'high' }, + }), + getModelCatalog: () => ({}), + }), + }); + + await setupCommand([provider]); + + assert.ok(logs.includes(`\n${metadata.displayName} Setup\n`)); + assert.ok(logs.includes(`✓ ${metadata.binary} CLI found`)); + assert.ok(logs.includes('\nAuth is user-managed; run the CLI login flow if needed:')); + assert.ok(logs.includes(metadata.authInstructions)); + assert.equal(saved.length, 1); + assert.equal(saved[0].providerSettings[provider].defaultLevel, 'level3'); +}); diff --git a/tests/agent-cli-provider/strict-lane.test.ts b/tests/agent-cli-provider/strict-lane.test.ts index 059b454f..fcf725a0 100644 --- a/tests/agent-cli-provider/strict-lane.test.ts +++ b/tests/agent-cli-provider/strict-lane.test.ts @@ -31,13 +31,30 @@ test('provider helper metadata documents package and build output', (): void => test('provider helper public API exposes typed provider adapters', (): void => { const providerIds: readonly ProviderId[] = listProviderAdapters(); - assert.deepEqual(providerIds, ['claude', 'codex', 'gemini', 'opencode']); + assert.deepEqual(providerIds, [ + 'claude', + 'codex', + 'gateway', + 'gemini', + 'opencode', + 'pi', + 'kiro', + 'copilot', + ]); const adapter = getProviderAdapter('openai'); assert.equal(adapter.id, 'codex'); assert.equal(adapter.displayName, 'Codex'); assert.equal(typeof adapter.adapterVersion, 'string'); assert.ok(adapter.credentialEnvKeys.length >= 0); + + const kiroAdapter = getProviderAdapter('kiro'); + assert.equal(kiroAdapter.id, 'kiro'); + assert.equal(kiroAdapter.binary, 'kiro-cli'); + + const gatewayAdapter = getProviderAdapter('gateway'); + assert.equal(gatewayAdapter.id, 'gateway'); + assert.equal(gatewayAdapter.binary, process.execPath); }); test('command specs preserve cleanup and warnings as typed metadata', (): void => { diff --git a/tests/e2e/cluster-lifecycle.test.js b/tests/e2e/cluster-lifecycle.test.js new file mode 100644 index 00000000..dd4ba099 --- /dev/null +++ b/tests/e2e/cluster-lifecycle.test.js @@ -0,0 +1,116 @@ +/** + * Tier 1 e2e: daemon-mode run plus the observability commands (status/list/ + * logs) an operator would actually use, ported from the now-deleted + * tests/integration/e2e-framework.test.ts (which could never run - it + * imported @playwright/test, an uninstalled dependency, and asserted a + * hardcoded stale version string). + */ + +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { + setupE2ERepo, + cleanupE2ERepo, + runZeroshot, + readCluster, + worktreePath, + scenarioPath, +} = require('./helpers/e2e-harness'); + +const CONFIG_PATH = path.join(__dirname, 'fixtures', 'single-worker-config.json'); + +function pollUntil(predicate, timeoutMs, intervalMs = 300) { + const start = Date.now(); + return new Promise((resolve, reject) => { + const tick = () => { + const value = predicate(); + if (value) return resolve(value); + if (Date.now() - start > timeoutMs) { + return reject(new Error(`Condition not met within ${timeoutMs}ms`)); + } + setTimeout(tick, intervalMs); + }; + tick(); + }); +} + +describe('e2e: cluster lifecycle (daemon mode + observability commands)', function () { + this.timeout(60000); + + let env; + + beforeEach(() => { + env = setupE2ERepo(); + }); + + afterEach(() => { + cleanupE2ERepo(env); + }); + + it('runs detached and is observable via status/list/logs until completion', async function () { + const issueDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-issue-')); + const issuePath = path.join(issueDir, 'feature.md'); + fs.writeFileSync(issuePath, '# Add feature\n\nDo X.\n'); + + // NOTE: unlike foreground `run`, the `-d` parent branch always calls + // generateName('cluster') itself (cli/index.js:shouldRunDetached path) - + // it does not honor a pre-set ZEROSHOT_CLUSTER_ID the way foreground/resume + // do, so the id must be parsed from stdout ("Started "). + const startResult = runZeroshot( + env, + ['run', issuePath, '-d', '--worktree', '--config', CONFIG_PATH], + { + FAKE_AGENT_SCENARIO: scenarioPath('single-worker-success'), + timeout: 15000, + } + ); + assert.strictEqual( + startResult.status, + 0, + `zeroshot run -d exited ${startResult.status}\nSTDOUT:\n${startResult.stdout}\nSTDERR:\n${startResult.stderr}` + ); + const clusterIdMatch = /Started (\S+)/.exec(startResult.stdout); + assert.ok(clusterIdMatch, `expected "Started " in stdout, got:\n${startResult.stdout}`); + const clusterId = clusterIdMatch[1]; + + await pollUntil(() => { + const cluster = readCluster(env, clusterId); + return cluster && cluster.state !== 'initializing' ? cluster : null; + }, 15000); + + const listResult = runZeroshot(env, ['list', '--json']); + assert.strictEqual(listResult.status, 0, listResult.stderr); + const listData = JSON.parse(listResult.stdout); + const clusters = Array.isArray(listData) ? listData : listData.clusters; + assert.ok( + clusters.some((c) => c.id === clusterId), + `expected ${clusterId} in list --json output: ${listResult.stdout}` + ); + + await pollUntil(() => { + const cluster = readCluster(env, clusterId); + return cluster?.state === 'stopped' || cluster?.state === 'killed' ? cluster : null; + }, 30000); + + const statusResult = runZeroshot(env, ['status', clusterId, '--json']); + assert.strictEqual(statusResult.status, 0, statusResult.stderr); + const statusData = JSON.parse(statusResult.stdout); + assert.strictEqual(statusData.id, clusterId); + assert.ok(['stopped', 'killed'].includes(statusData.state), JSON.stringify(statusData)); + + const logsResult = runZeroshot(env, ['logs', clusterId, '-n', '200']); + assert.strictEqual(logsResult.status, 0, logsResult.stderr); + assert.ok( + logsResult.stdout.includes('Implementing the requested feature'), + `expected fake-agent message text in logs, got:\n${logsResult.stdout}` + ); + + const writtenFile = path.join(worktreePath(env, clusterId), 'output.txt'); + assert.ok(fs.existsSync(writtenFile)); + + fs.rmSync(issueDir, { recursive: true, force: true }); + }); +}); diff --git a/tests/e2e/copilot-provider.test.js b/tests/e2e/copilot-provider.test.js new file mode 100644 index 00000000..013c3108 --- /dev/null +++ b/tests/e2e/copilot-provider.test.js @@ -0,0 +1,167 @@ +/** + * Tier 1 e2e for the Copilot provider engine: the full pipeline through the real + * `zeroshot` binary with a fake `copilot` CLI (on PATH) standing in for the model. + * + * Unlike the claude-based e2e (which shims the binary via ZEROSHOT_CLAUDE_COMMAND), a + * generic registry provider is resolved from PATH by binary name. The harness prepends an + * isolated bin dir to PATH, so installing an executable named `copilot` there exercises for + * real: CLI parsing -> `--provider copilot` override -> registry resolution -> preflight + * availability probe -> subprocess spawn -> Copilot `--output-format json` JSONL parsing (the + * new copilot adapter/parser) -> hook-driven completion -> worktree-isolated file write. + * + * Fully offline: no Copilot API calls, no credentials, no network. + */ + +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('node:child_process'); + +const { + setupE2ERepo, + cleanupE2ERepo, + runZeroshot, + worktreePath, + waitForClusterState, + readLedgerMessages, + gitStatusPorcelain, +} = require('./helpers/e2e-harness'); + +const CONFIG_PATH = path.join(__dirname, 'fixtures', 'copilot-worker-config.json'); +const FAKE_COPILOT = path.join(__dirname, 'fixtures', 'fake-copilot.js'); + +/** Install an executable literally named `copilot` into the harness bin dir (already on PATH). */ +function installFakeCopilot(binDir) { + const shim = path.join(binDir, 'copilot'); + fs.writeFileSync(shim, `#!/bin/sh\nexec node "${FAKE_COPILOT}" "$@"\n`, { mode: 0o755 }); + fs.chmodSync(shim, 0o755); +} + +function gitCommitFile(repoDir, relPath, content, message) { + const absPath = path.join(repoDir, relPath); + fs.mkdirSync(path.dirname(absPath), { recursive: true }); + fs.writeFileSync(absPath, content); + for (const args of [ + ['add', relPath], + ['commit', '-m', message], + ]) { + const result = spawnSync('git', args, { cwd: repoDir, encoding: 'utf8' }); + if (result.status !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${result.stderr || result.stdout}`); + } + } +} + +describe('e2e: copilot provider', function () { + this.timeout(60000); + + let env; + + beforeEach(() => { + env = setupE2ERepo(); + installFakeCopilot(env.binDir); + }); + + afterEach(() => { + cleanupE2ERepo(env); + }); + + it('runs a cluster through the copilot provider and writes into the worktree', async function () { + const issueDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-issue-')); + const issuePath = path.join(issueDir, 'feature.md'); + fs.writeFileSync(issuePath, '# Add feature\n\nDo X.\n'); + + const clusterId = 'e2e-copilot-worker'; + const result = runZeroshot( + env, + ['run', issuePath, '--worktree', '--provider', 'copilot', '--config', CONFIG_PATH], + { ZEROSHOT_CLUSTER_ID: clusterId } + ); + + assert.strictEqual( + result.status, + 0, + `zeroshot run exited ${result.status}\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}` + ); + + const cluster = await waitForClusterState(env, clusterId, ['stopped', 'killed']); + assert.strictEqual( + cluster.state, + 'stopped', + 'cluster should stop cleanly via the completion hook' + ); + + // The fake copilot's file write lands in the worktree (proves cwd injection + real spawn). + const worktreeDir = worktreePath(env, clusterId); + const writtenFile = path.join(worktreeDir, 'output.txt'); + assert.ok(fs.existsSync(writtenFile), `expected ${writtenFile} to exist`); + assert.strictEqual(fs.readFileSync(writtenFile, 'utf8'), 'copilot implemented\n'); + + // Nothing leaks into the main checkout. + assert.ok( + !fs.existsSync(path.join(env.repoDir, 'output.txt')), + 'output.txt must not leak into the main checkout' + ); + assert.strictEqual(gitStatusPorcelain(env.repoDir), '', 'main checkout should remain clean'); + + // The completion hook only fires if the copilot JSONL `result` event was parsed as success, + // proving the copilot adapter/parser ran end to end (not just that a subprocess executed). + const completions = readLedgerMessages(env, clusterId, 'TASK_COMPLETE'); + assert.ok( + completions.length >= 1, + 'expected a TASK_COMPLETE message from the worker onComplete hook' + ); + + fs.rmSync(issueDir, { recursive: true, force: true }); + }); + + it('forwards the repo .mcp.json to copilot as --additional-mcp-config', async function () { + // The repo's `.claude/.mcp.json` (the same MCP source Claude consumes) must be committed so it + // lands in the HEAD-based worktree the worker runs in. + const mcpJson = JSON.stringify({ + mcpServers: { demo: { command: 'demo-mcp-bin', args: ['--stdio'] } }, + }); + gitCommitFile(env.repoDir, path.join('.claude', '.mcp.json'), mcpJson, 'Add MCP config'); + + const issueDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-issue-')); + const issuePath = path.join(issueDir, 'feature.md'); + fs.writeFileSync(issuePath, '# Add feature\n\nDo X.\n'); + + const clusterId = 'e2e-copilot-mcp'; + const result = runZeroshot( + env, + ['run', issuePath, '--worktree', '--provider', 'copilot', '--config', CONFIG_PATH], + { ZEROSHOT_CLUSTER_ID: clusterId } + ); + + assert.strictEqual( + result.status, + 0, + `zeroshot run exited ${result.status}\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}` + ); + + const cluster = await waitForClusterState(env, clusterId, ['stopped', 'killed']); + assert.strictEqual(cluster.state, 'stopped', 'cluster should stop cleanly'); + + // The fake copilot records the exact argv it was spawned with, into the worktree cwd. + const worktreeDir = worktreePath(env, clusterId); + const argvLog = path.join(worktreeDir, 'copilot-received-argv.json'); + assert.ok(fs.existsSync(argvLog), `expected ${argvLog} to exist`); + + const argv = JSON.parse(fs.readFileSync(argvLog, 'utf8')); + const flagIndex = argv.indexOf('--additional-mcp-config'); + assert.ok( + flagIndex >= 0, + `expected copilot to receive --additional-mcp-config; got argv: ${JSON.stringify(argv)}` + ); + // The repo .mcp.json content is inlined verbatim as the flag value (no @path, no translation). + assert.strictEqual( + argv[flagIndex + 1], + mcpJson, + 'the inlined MCP config value must equal the repo .mcp.json content' + ); + + fs.rmSync(issueDir, { recursive: true, force: true }); + }); +}); diff --git a/tests/e2e/failing-agent-resume.test.js b/tests/e2e/failing-agent-resume.test.js new file mode 100644 index 00000000..5c4cb509 --- /dev/null +++ b/tests/e2e/failing-agent-resume.test.js @@ -0,0 +1,101 @@ +/** + * Tier 1 e2e: resume path. A scenario that always fails exhausts the + * orchestrator's built-in retry budget (src/agent/agent-lifecycle.js), which + * publishes AGENT_ERROR and persists cluster.failureInfo + * (src/orchestrator.js:_registerAgentErrorHandler). For an 'implementation' + * role agent with 3+ attempts, the orchestrator auto-stops the cluster - no + * custom onError/topic wiring needed (config-validator's reachability check + * only tracks hooks.onComplete as a topic producer, so a hand-rolled + * onError-publishes-a-topic path would be flagged as an unproduced topic). + * `zeroshot resume` then re-runs the failed agent; pointing it at a + * succeeding scenario should drive the cluster to completion. + */ + +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { + setupE2ERepo, + cleanupE2ERepo, + runZeroshot, + readCluster, + worktreePath, + scenarioPath, +} = require('./helpers/e2e-harness'); + +const CONFIG_PATH = path.join(__dirname, 'fixtures', 'single-worker-config.json'); + +function pollUntil(predicate, timeoutMs, intervalMs = 200) { + const start = Date.now(); + return new Promise((resolve, reject) => { + const tick = () => { + const value = predicate(); + if (value) return resolve(value); + if (Date.now() - start > timeoutMs) { + return reject(new Error(`Condition not met within ${timeoutMs}ms`)); + } + setTimeout(tick, intervalMs); + }; + tick(); + }); +} + +describe('e2e: failing agent + resume', function () { + this.timeout(90000); + + let env; + + beforeEach(() => { + env = setupE2ERepo(); + }); + + afterEach(() => { + cleanupE2ERepo(env); + }); + + it('records failureInfo on exhaustion and completes after resume with a working scenario', async function () { + const issueDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-issue-')); + const issuePath = path.join(issueDir, 'feature.md'); + fs.writeFileSync(issuePath, '# Add feature\n\nDo X.\n'); + + const clusterId = 'e2e-failing-agent'; + const firstRun = runZeroshot(env, ['run', issuePath, '--worktree', '--config', CONFIG_PATH], { + ZEROSHOT_CLUSTER_ID: clusterId, + FAKE_AGENT_SCENARIO: scenarioPath('failing-agent'), + }); + + assert.strictEqual( + firstRun.status, + 0, + `zeroshot run exited ${firstRun.status}\nSTDOUT:\n${firstRun.stdout}\nSTDERR:\n${firstRun.stderr}` + ); + + const failedCluster = await pollUntil(() => { + const cluster = readCluster(env, clusterId); + return cluster?.failureInfo ? cluster : null; + }, 30000); + assert.strictEqual(failedCluster.failureInfo.agentId, 'worker'); + assert.ok(!fs.existsSync(path.join(worktreePath(env, clusterId), 'output.txt'))); + + // -d returns once the resume has been scheduled; the process still stays + // alive until the reconstructed cluster's terminal subscriptions observe + // completion and stop the resumed agents. + const resumeResult = runZeroshot(env, ['resume', clusterId, '-d'], { + FAKE_AGENT_SCENARIO: scenarioPath('single-worker-success'), + timeout: 15000, + }); + assert.strictEqual( + resumeResult.status, + 0, + `zeroshot resume exited ${resumeResult.status}\nSTDOUT:\n${resumeResult.stdout}\nSTDERR:\n${resumeResult.stderr}` + ); + + const writtenFile = path.join(worktreePath(env, clusterId), 'output.txt'); + await pollUntil(() => fs.existsSync(writtenFile), 30000); + assert.strictEqual(fs.readFileSync(writtenFile, 'utf8'), 'feature implemented\n'); + + fs.rmSync(issueDir, { recursive: true, force: true }); + }); +}); diff --git a/tests/e2e/fixtures/copilot-worker-config.json b/tests/e2e/fixtures/copilot-worker-config.json new file mode 100644 index 00000000..da0f69dc --- /dev/null +++ b/tests/e2e/fixtures/copilot-worker-config.json @@ -0,0 +1,24 @@ +{ + "defaultProvider": "copilot", + "agents": [ + { + "id": "worker", + "role": "implementation", + "timeout": 0, + "triggers": [{ "topic": "ISSUE_OPENED", "action": "execute_task" }], + "prompt": "Implement the requested feature.", + "hooks": { + "onComplete": { + "action": "publish_message", + "config": { "topic": "TASK_COMPLETE", "content": { "text": "Done" } } + } + } + }, + { + "id": "completion-detector", + "role": "orchestrator", + "timeout": 0, + "triggers": [{ "topic": "TASK_COMPLETE", "action": "stop_cluster" }] + } + ] +} diff --git a/tests/e2e/fixtures/fake-copilot.js b/tests/e2e/fixtures/fake-copilot.js new file mode 100644 index 00000000..b23855b1 --- /dev/null +++ b/tests/e2e/fixtures/fake-copilot.js @@ -0,0 +1,120 @@ +#!/usr/bin/env node +/** + * Fake GitHub Copilot CLI for the deterministic Copilot-provider e2e test. + * + * Unlike the fake-agent (which shims `claude` via ZEROSHOT_CLAUDE_COMMAND), a generic + * registry provider is resolved from PATH by its binary name. The e2e harness prepends an + * isolated bin dir to PATH, so dropping an executable literally named `copilot` there makes + * the whole stack (CLI parsing -> registry resolution -> preflight availability probe -> + * spawn -> Copilot JSONL parsing -> ledger -> hooks) run for real, offline, with no API calls. + * + * Behaviour: + * copilot --version -> print a version line, exit 0 (availability probe). + * copilot --help / -h -> print usage listing every flag the adapter emits, exit 0 + * (drives detectCliFeatures + the help-or-version probe). + * copilot ... -p ... -> a non-interactive run: write COPILOT_FAKE_FILE (default + * output.txt) into process.cwd() and emit the assumed Copilot + * `--output-format json` JSONL (assistant text + success result). + * + * The prompt is the LAST argv element (adapter emits `-p ` last). The runtime sets the + * child cwd to the worktree, so files are written relative to process.cwd() (proves cwd injection). + */ + +const fs = require('fs'); +const path = require('path'); + +const USAGE = [ + 'GitHub Copilot CLI (fake)', + '', + 'Usage: copilot [options]', + ' -p, --prompt Execute a prompt in non-interactive mode', + " --output-format Output format: 'text' (default) or 'json' (JSONL)", + ' --model Set the AI model to use', + ' --allow-all Enable all permissions', + ' --no-ask-user Disable the ask_user tool', + ' --add-dir Add a directory to the allowed list', + ' --additional-mcp-config JSON string or @file MCP config (repeatable)', +].join('\n'); + +function emit(event) { + process.stdout.write(`${JSON.stringify(event)}\n`); +} + +function promptValue(argv) { + const flag = argv.lastIndexOf('-p') >= 0 ? argv.lastIndexOf('-p') : argv.lastIndexOf('--prompt'); + if (flag >= 0 && flag + 1 < argv.length) return argv[flag + 1]; + return argv[argv.length - 1] || ''; +} + +function main() { + const argv = process.argv.slice(2); + + if (argv.includes('--version')) { + process.stdout.write('GitHub Copilot CLI 1.0.69 (fake)\n'); + process.exit(0); + } + + const isRun = argv.includes('-p') || argv.includes('--prompt'); + if (!isRun || argv.includes('--help') || argv.includes('-h')) { + process.stdout.write(`${USAGE}\n`); + process.exit(0); + } + + // Non-interactive run: apply the (fake) model's file change in the worktree cwd, then stream + // Copilot-shaped JSONL that the copilot adapter parser normalizes into engine OutputEvents. + const outFile = process.env.COPILOT_FAKE_FILE || 'output.txt'; + const content = process.env.COPILOT_FAKE_CONTENT || 'copilot implemented\n'; + const target = path.resolve(process.cwd(), outFile); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, content); + process.stderr.write(`fake-copilot: wrote ${target} (prompt="${promptValue(argv)}")\n`); + + // Record the exact argv this run received so the MCP e2e can assert that + // `--additional-mcp-config ` was forwarded through the whole stack. + const argvLog = path.resolve(process.cwd(), 'copilot-received-argv.json'); + fs.writeFileSync(argvLog, JSON.stringify(argv)); + + // Emit the real Copilot `--output-format json` JSONL shape (verified against copilot v1.0.69): + // dot-namespaced types, payload under `data`, success derived from the terminal `exitCode`. + emit({ type: 'session.tools_updated', data: { model: 'fake' }, ephemeral: true }); + // A `commentary` message (parsed as thinking) followed by the `final_answer` (parsed as text). + emit({ + type: 'assistant.message_start', + data: { messageId: 'm0', phase: 'commentary' }, + ephemeral: true, + }); + emit({ + type: 'assistant.message', + data: { + messageId: 'm0', + phase: 'commentary', + content: 'Writing the requested file.', + outputTokens: 2, + }, + }); + emit({ + type: 'assistant.message_start', + data: { messageId: 'm1', phase: 'final_answer' }, + ephemeral: true, + }); + emit({ + type: 'assistant.message_delta', + data: { messageId: 'm1', deltaContent: 'Implemented the requested feature.' }, + ephemeral: true, + }); + emit({ + type: 'assistant.message', + data: { + messageId: 'm1', + phase: 'final_answer', + content: 'Implemented the requested feature.', + toolRequests: [], + outputTokens: 3, + }, + }); + emit({ type: 'assistant.idle', data: {}, ephemeral: true }); + emit({ type: 'result', sessionId: 'fake', exitCode: 0, usage: { premiumRequests: 0 } }); + process.exit(0); +} + +main(); diff --git a/tests/e2e/fixtures/single-worker-config.json b/tests/e2e/fixtures/single-worker-config.json new file mode 100644 index 00000000..aea12d1c --- /dev/null +++ b/tests/e2e/fixtures/single-worker-config.json @@ -0,0 +1,23 @@ +{ + "agents": [ + { + "id": "worker", + "role": "implementation", + "timeout": 0, + "triggers": [{ "topic": "ISSUE_OPENED", "action": "execute_task" }], + "prompt": "FAKE_AGENT_ID=worker Implement the requested feature.", + "hooks": { + "onComplete": { + "action": "publish_message", + "config": { "topic": "TASK_COMPLETE", "content": { "text": "Done" } } + } + } + }, + { + "id": "completion-detector", + "role": "orchestrator", + "timeout": 0, + "triggers": [{ "topic": "TASK_COMPLETE", "action": "stop_cluster" }] + } + ] +} diff --git a/tests/e2e/fixtures/worker-validator-config.json b/tests/e2e/fixtures/worker-validator-config.json new file mode 100644 index 00000000..fa748ff2 --- /dev/null +++ b/tests/e2e/fixtures/worker-validator-config.json @@ -0,0 +1,52 @@ +{ + "agents": [ + { + "id": "worker", + "role": "implementation", + "timeout": 0, + "triggers": [ + { "topic": "ISSUE_OPENED", "action": "execute_task" }, + { + "topic": "VALIDATION_RESULT", + "logic": { + "engine": "javascript", + "script": "const lastPush = ledger.findLast({ topic: 'IMPLEMENTATION_READY' });\nif (!lastPush) return false;\nconst response = ledger.findLast({ topic: 'VALIDATION_RESULT', since: lastPush.timestamp });\nreturn response?.content?.data?.approved === false || response?.content?.data?.approved === 'false';" + }, + "action": "execute_task" + } + ], + "prompt": "FAKE_AGENT_ID=worker Implement the requested feature.", + "hooks": { + "onComplete": { + "action": "publish_message", + "config": { + "topic": "IMPLEMENTATION_READY", + "content": { "text": "Ready for validation" } + } + } + } + }, + { + "id": "validator", + "role": "validator", + "timeout": 0, + "triggers": [{ "topic": "IMPLEMENTATION_READY", "action": "execute_task" }], + "prompt": "FAKE_AGENT_ID=validator Validate the implementation.", + "hooks": { + "onComplete": { + "action": "publish_message", + "config": { + "topic": "VALIDATION_RESULT", + "content": { "text": "Approved", "data": { "approved": true } } + } + } + } + }, + { + "id": "completion-detector", + "role": "orchestrator", + "timeout": 0, + "triggers": [{ "topic": "VALIDATION_RESULT", "action": "stop_cluster" }] + } + ] +} diff --git a/tests/e2e/helpers/e2e-harness.js b/tests/e2e/helpers/e2e-harness.js new file mode 100644 index 00000000..8c4b0399 --- /dev/null +++ b/tests/e2e/helpers/e2e-harness.js @@ -0,0 +1,245 @@ +/** + * Harness for deterministic end-to-end tests. + * + * Drives the real `zeroshot` binary (cli/index.js) as a subprocess, with + * ZEROSHOT_CLAUDE_COMMAND pointed at tests/fixtures/fake-agent so every layer + * above the model (CLI parsing, orchestrator, message bus, ledger, trigger + * engine, agent spawning, stream-json parsing, hooks, worktree isolation) + * runs for real, offline, with no API calls. + * + * Isolation note: most storage paths in this codebase are derived from + * `os.homedir()` directly (orchestrator storageDir, worktree paths, clusters.json), + * NOT from a ZEROSHOT_HOME env var. Node's os.homedir() reads the HOME env var on + * POSIX, so isolation here is achieved by overriding HOME for the child process, + * not by inventing a ZEROSHOT_HOME-only scheme. + * + * PATH shim note: agent execution self-spawns via `which zeroshot` + * (src/agent/agent-task-executor.js:_resolveZeroshotPath), i.e. it does NOT + * reuse the exact script path used to launch the top-level process. A temp + * directory containing a `zeroshot` shim script is prepended to PATH so the + * self-spawned subprocess resolves back to this checkout's cli/index.js, + * without requiring `npm link` as a precondition. + */ + +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const { execFileSync, spawnSync } = require('child_process'); + +const REPO_ROOT = path.resolve(__dirname, '..', '..', '..'); +const CLI_ENTRY = path.join(REPO_ROOT, 'cli', 'index.js'); +const FAKE_AGENT_PATH = path.join(REPO_ROOT, 'tests', 'fixtures', 'fake-agent', 'index.js'); +const FIXTURES_DIR = path.join(REPO_ROOT, 'tests', 'fixtures', 'fake-agent-scenarios'); + +function scenarioPath(name) { + return path.join(FIXTURES_DIR, name.endsWith('.json') ? name : `${name}.json`); +} + +function runGit(args, cwd) { + const result = spawnSync('git', args, { cwd, encoding: 'utf8', stdio: 'pipe' }); + if (result.status !== 0) { + throw new Error( + `git ${args.join(' ')} failed in ${cwd}: ${result.stderr || result.error?.message}` + ); + } + return result.stdout; +} + +function writeZeroshotShim(binDir) { + const shimPath = path.join(binDir, 'zeroshot'); + fs.writeFileSync(shimPath, `#!/bin/sh\nexec node "${CLI_ENTRY}" "$@"\n`, { mode: 0o755 }); + fs.chmodSync(shimPath, 0o755); + return shimPath; +} + +/** + * Sets up an isolated temp git repo (with a local bare "origin") plus an + * isolated fake HOME so `zeroshot` state (clusters.json, worktrees, settings) + * never touches the real user environment. + */ +function setupE2ERepo() { + const repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-repo-')); + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-home-')); + const binDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-bin-')); + const originDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-origin-')); + + runGit(['init', '--bare'], originDir); + + runGit(['init'], repoDir); + runGit(['config', 'user.email', 'e2e@example.com'], repoDir); + runGit(['config', 'user.name', 'E2E Test'], repoDir); + runGit(['remote', 'add', 'origin', originDir], repoDir); + fs.writeFileSync(path.join(repoDir, 'README.md'), '# e2e test repo\n'); + runGit(['add', 'README.md'], repoDir); + runGit(['commit', '-m', 'Initial commit'], repoDir); + runGit(['push', 'origin', 'HEAD'], repoDir); + + writeZeroshotShim(binDir); + fs.mkdirSync(path.join(homeDir, '.zeroshot'), { recursive: true }); + // Prevent the CLI's npm-registry update check (network call + interactive + // prompt) from firing during offline, deterministic test runs. + fs.writeFileSync( + path.join(homeDir, '.zeroshot', 'settings.json'), + JSON.stringify({ autoCheckUpdates: false }, null, 2) + ); + + return { repoDir, homeDir, binDir, originDir }; +} + +function cleanupE2ERepo(env) { + if (!env) return; + for (const dir of [env.repoDir, env.homeDir, env.binDir, env.originDir]) { + if (dir && fs.existsSync(dir)) { + fs.rmSync(dir, { recursive: true, force: true }); + } + } +} + +/** + * Strips ambient ZEROSHOT_ and CMDPROOF_ prefixed env vars from the base env + * before building a child env. Without this, running the e2e suite from inside a + * zeroshot-managed session (e.g. a zeroshot worker agent) leaks that + * session's own ZEROSHOT_RUN_OPTIONS/ZEROSHOT_PR/ZEROSHOT_CLUSTER_ID etc. + * into the spawned test subprocess, silently changing its behavior + * (observed: an inherited ZEROSHOT_RUN_OPTIONS with pr:true injected a + * git-pusher agent and strict schema into an unrelated custom --config). + */ +function baseEnv() { + const result = {}; + for (const [key, value] of Object.entries(process.env)) { + if (key.startsWith('ZEROSHOT_') || key.startsWith('CMDPROOF_')) continue; + result[key] = value; + } + return result; +} + +function buildEnv(env, envOverrides = {}) { + return { + ...baseEnv(), + HOME: env.homeDir, + ZEROSHOT_HOME: env.homeDir, + ZEROSHOT_SETTINGS_FILE: path.join(env.homeDir, '.zeroshot', 'settings.json'), + ZEROSHOT_CLAUDE_COMMAND: `node ${FAKE_AGENT_PATH}`, + PATH: `${env.binDir}${path.delimiter}${process.env.PATH}`, + ...envOverrides, + }; +} + +/** + * Runs `zeroshot ` as a real subprocess against the isolated repo/home. + * Returns { status, stdout, stderr } instead of throwing, so tests can assert + * on non-zero exit codes (e.g. the failing-agent scenario). + */ +function runZeroshot(env, args, envOverrides = {}) { + const result = spawnSync('node', [CLI_ENTRY, ...args], { + cwd: env.repoDir, + env: buildEnv(env, envOverrides), + encoding: 'utf8', + timeout: envOverrides.timeout || 90000, + }); + return { + status: result.status, + stdout: result.stdout || '', + stderr: result.stderr || '', + error: result.error || null, + }; +} + +function clustersFilePath(env) { + return path.join(env.homeDir, '.zeroshot', 'clusters.json'); +} + +function readClusters(env) { + const file = clustersFilePath(env); + if (!fs.existsSync(file)) return {}; + return JSON.parse(fs.readFileSync(file, 'utf8')); +} + +function readCluster(env, clusterId) { + return readClusters(env)[clusterId] || null; +} + +/** + * clusters.json does not persist the worktree path (src/orchestrator.js + * _saveClusters only persists `isolation`, the Docker-mode counterpart, not + * `worktree`). The path is deterministic (src/isolation-manager.js + * createWorktreeIsolation): /.zeroshot/worktrees/. + */ +function worktreePath(env, clusterId) { + return path.join(env.homeDir, '.zeroshot', 'worktrees', clusterId); +} + +async function waitForClusterState(env, clusterId, targetStates, timeoutMs = 30000) { + const states = Array.isArray(targetStates) ? targetStates : [targetStates]; + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const cluster = readCluster(env, clusterId); + if (cluster && states.includes(cluster.state)) { + return cluster; + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error( + `Cluster ${clusterId} did not reach state [${states.join(', ')}] within ${timeoutMs}ms. ` + + `Last known: ${JSON.stringify(readCluster(env, clusterId))}` + ); +} + +function clusterDbPath(env, clusterId) { + return path.join(env.homeDir, '.zeroshot', `${clusterId}.db`); +} + +/** + * Mirrors src/ledger.js's row deserialization: content_text/content_data/metadata + * are stored as raw TEXT/JSON columns, not reconstructed by a bare SELECT *. + */ +function rowToMessage(row) { + const message = { ...row, content: {} }; + if (row.content_text) message.content.text = row.content_text; + if (row.content_data) message.content.data = JSON.parse(row.content_data); + if (row.metadata) message.metadata = JSON.parse(row.metadata); + return message; +} + +function readLedgerMessages(env, clusterId, topic) { + const Database = require('better-sqlite3'); + const dbPath = clusterDbPath(env, clusterId); + if (!fs.existsSync(dbPath)) return []; + const db = new Database(dbPath, { readonly: true, timeout: 5000 }); + try { + const rows = db + .prepare('SELECT * FROM messages WHERE cluster_id = ? AND topic = ? ORDER BY id') + .all(clusterId, topic); + return rows.map(rowToMessage); + } finally { + db.close(); + } +} + +function gitStatusPorcelain(repoDir) { + return runGit(['status', '--porcelain'], repoDir).trim(); +} + +function gitWorktreeList(repoDir) { + return runGit(['worktree', 'list'], repoDir); +} + +module.exports = { + REPO_ROOT, + CLI_ENTRY, + FAKE_AGENT_PATH, + scenarioPath, + setupE2ERepo, + cleanupE2ERepo, + runZeroshot, + clustersFilePath, + readClusters, + readCluster, + worktreePath, + waitForClusterState, + clusterDbPath, + readLedgerMessages, + gitStatusPorcelain, + gitWorktreeList, + execFileSync, +}; diff --git a/tests/e2e/single-worker.test.js b/tests/e2e/single-worker.test.js new file mode 100644 index 00000000..2aacaa28 --- /dev/null +++ b/tests/e2e/single-worker.test.js @@ -0,0 +1,78 @@ +/** + * Tier 1 e2e: full pipeline through the real `zeroshot` binary with the + * fake provider CLI standing in for the model. Proves: CLI parsing -> config + * load -> orchestrator start -> real subprocess spawn -> stream-json parsing + * -> hook-driven completion -> file write happens in the worktree, not the + * main checkout. + */ + +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { + setupE2ERepo, + cleanupE2ERepo, + runZeroshot, + readCluster, + worktreePath, + waitForClusterState, + scenarioPath, + gitStatusPorcelain, +} = require('./helpers/e2e-harness'); + +const CONFIG_PATH = path.join(__dirname, 'fixtures', 'single-worker-config.json'); + +describe('e2e: single worker', function () { + this.timeout(60000); + + let env; + + beforeEach(() => { + env = setupE2ERepo(); + }); + + afterEach(() => { + cleanupE2ERepo(env); + }); + + it('runs the full pipeline and writes the file into the worktree, not the main checkout', async function () { + // The issue file lives outside the repo so the main checkout's git status + // stays meaningful as a "did anything leak in" signal. + const issueDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-issue-')); + const issuePath = path.join(issueDir, 'feature.md'); + fs.writeFileSync(issuePath, '# Add feature\n\nDo X.\n'); + + const clusterId = 'e2e-single-worker'; + const result = runZeroshot(env, ['run', issuePath, '--worktree', '--config', CONFIG_PATH], { + ZEROSHOT_CLUSTER_ID: clusterId, + FAKE_AGENT_SCENARIO: scenarioPath('single-worker-success'), + }); + + assert.strictEqual( + result.status, + 0, + `zeroshot run exited ${result.status}\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}` + ); + + const cluster = await waitForClusterState(env, clusterId, ['stopped', 'killed']); + const worktreeDir = worktreePath(env, clusterId); + assert.ok(fs.existsSync(worktreeDir), 'worktree directory should exist'); + + const writtenFile = path.join(worktreeDir, 'output.txt'); + assert.ok(fs.existsSync(writtenFile), `expected ${writtenFile} to exist`); + assert.strictEqual(fs.readFileSync(writtenFile, 'utf8'), 'feature implemented\n'); + + assert.ok( + !fs.existsSync(path.join(env.repoDir, 'output.txt')), + 'output.txt must not leak into the main checkout' + ); + assert.strictEqual(gitStatusPorcelain(env.repoDir), '', 'main checkout should remain clean'); + + const clusterAfter = readCluster(env, clusterId); + assert.strictEqual(clusterAfter.state, cluster.state); + + fs.rmSync(issueDir, { recursive: true, force: true }); + }); +}); diff --git a/tests/e2e/worker-validator.test.js b/tests/e2e/worker-validator.test.js new file mode 100644 index 00000000..acf520ba --- /dev/null +++ b/tests/e2e/worker-validator.test.js @@ -0,0 +1,76 @@ +/** + * Tier 1 e2e: worker -> validator flow with distinct fake-agent scenarios per + * agent. Both agents share one process-wide ZEROSHOT_CLAUDE_COMMAND, so + * per-agent scenario dispatch relies on the FAKE_AGENT_ID= marker + * embedded in each agent's `prompt` field (see tests/fixtures/fake-agent). + */ + +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { + setupE2ERepo, + cleanupE2ERepo, + runZeroshot, + worktreePath, + readLedgerMessages, + scenarioPath, +} = require('./helpers/e2e-harness'); + +const CONFIG_PATH = path.join(__dirname, 'fixtures', 'worker-validator-config.json'); + +describe('e2e: worker -> validator', function () { + this.timeout(60000); + + let env; + + beforeEach(() => { + env = setupE2ERepo(); + }); + + afterEach(() => { + cleanupE2ERepo(env); + }); + + it('runs both agents with distinct scenarios and reaches validation approval', function () { + const issueDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-issue-')); + const issuePath = path.join(issueDir, 'feature.md'); + fs.writeFileSync(issuePath, '# Add feature\n\nDo X.\n'); + + const clusterId = 'e2e-worker-validator'; + const result = runZeroshot(env, ['run', issuePath, '--worktree', '--config', CONFIG_PATH], { + ZEROSHOT_CLUSTER_ID: clusterId, + FAKE_AGENT_SCENARIO_WORKER: scenarioPath('worker-success'), + FAKE_AGENT_SCENARIO_VALIDATOR: scenarioPath('validator-approve'), + }); + + assert.strictEqual( + result.status, + 0, + `zeroshot run exited ${result.status}\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}` + ); + + const worktreeDir = worktreePath(env, clusterId); + assert.ok( + fs.existsSync(path.join(worktreeDir, 'implementation.txt')), + 'worker should have written implementation.txt' + ); + assert.ok( + fs.existsSync(path.join(worktreeDir, 'validation-report.txt')), + 'validator should have written validation-report.txt' + ); + + const implementationReady = readLedgerMessages(env, clusterId, 'IMPLEMENTATION_READY'); + assert.strictEqual(implementationReady.length, 1); + assert.strictEqual(implementationReady[0].sender, 'worker'); + + const validationResults = readLedgerMessages(env, clusterId, 'VALIDATION_RESULT'); + assert.strictEqual(validationResults.length, 1); + assert.strictEqual(validationResults[0].sender, 'validator'); + assert.strictEqual(validationResults[0].content.data.approved, true); + + fs.rmSync(issueDir, { recursive: true, force: true }); + }); +}); diff --git a/tests/e2e/worktree-isolation-no-leak.test.js b/tests/e2e/worktree-isolation-no-leak.test.js new file mode 100644 index 00000000..ffca7120 --- /dev/null +++ b/tests/e2e/worktree-isolation-no-leak.test.js @@ -0,0 +1,79 @@ +/** + * Tier 1 e2e: --worktree mode leaves the main checkout untouched and + * registers a real git worktree/branch for the cluster, with no leaked + * process for the completed cluster. + */ + +const assert = require('node:assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { + setupE2ERepo, + cleanupE2ERepo, + runZeroshot, + worktreePath, + gitStatusPorcelain, + gitWorktreeList, + scenarioPath, +} = require('./helpers/e2e-harness'); + +const CONFIG_PATH = path.join(__dirname, 'fixtures', 'single-worker-config.json'); + +describe('e2e: worktree isolation, no leak', function () { + this.timeout(60000); + + let env; + + beforeEach(() => { + env = setupE2ERepo(); + }); + + afterEach(() => { + cleanupE2ERepo(env); + }); + + it('leaves the main checkout clean and registers a real worktree/branch', function () { + const issueDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-e2e-issue-')); + const issuePath = path.join(issueDir, 'feature.md'); + fs.writeFileSync(issuePath, '# Add feature\n\nDo X.\n'); + + const clusterId = 'e2e-worktree-isolation'; + const result = runZeroshot(env, ['run', issuePath, '--worktree', '--config', CONFIG_PATH], { + ZEROSHOT_CLUSTER_ID: clusterId, + FAKE_AGENT_SCENARIO: scenarioPath('single-worker-success'), + }); + + assert.strictEqual( + result.status, + 0, + `zeroshot run exited ${result.status}\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}` + ); + + assert.strictEqual( + gitStatusPorcelain(env.repoDir), + '', + 'main checkout should have no uncommitted residue' + ); + + const worktreeList = gitWorktreeList(env.repoDir); + assert.ok( + worktreeList.includes(`zeroshot/${clusterId}`), + `expected git worktree list to include zeroshot/${clusterId}, got:\n${worktreeList}` + ); + assert.ok(fs.existsSync(worktreePath(env, clusterId)), 'worktree directory should exist'); + + const psOutput = require('child_process').execSync('ps -A -o command=').toString(); + const leakedLines = psOutput + .split('\n') + .filter((line) => line.includes(clusterId) && !line.includes('grep')); + assert.strictEqual( + leakedLines.length, + 0, + `expected no leaked processes referencing ${clusterId}, found:\n${leakedLines.join('\n')}` + ); + + fs.rmSync(issueDir, { recursive: true, force: true }); + }); +}); diff --git a/tests/first-run.test.js b/tests/first-run.test.js index f378426c..a17627f0 100644 --- a/tests/first-run.test.js +++ b/tests/first-run.test.js @@ -28,6 +28,7 @@ const firstRunPath = require.resolve('../cli/lib/first-run'); // Variables to hold fresh module references (set in before() hook) let settingsModule; let firstRunModule; +let providerNamesModule; describe('First-Run Setup', function () { this.timeout(10000); @@ -50,6 +51,7 @@ describe('First-Run Setup', function () { // Now require fresh modules that will read our env var settingsModule = require('../lib/settings'); firstRunModule = require('../cli/lib/first-run'); + providerNamesModule = require('../lib/provider-names'); // Verify env var is set correctly (sanity check) assert.strictEqual( @@ -132,6 +134,39 @@ describe('First-Run Setup', function () { }); }); + describe('promptProvider()', function () { + it('should print registry-backed install instructions when no providers are detected', function () { + const logged = []; + const originalLog = console.log; + const originalExit = process.exit; + + console.log = (...args) => logged.push(args.join(' ')); + process.exit = (code) => { + const error = new Error(`exit ${code}`); + error.code = code; + throw error; + }; + + try { + assert.throws( + () => firstRunModule.promptProvider({}, {}), + (error) => error && error.code === 1 + ); + } finally { + console.log = originalLog; + process.exit = originalExit; + } + + const output = logged.join('\n'); + for (const provider of providerNamesModule.listProviderMetadata()) { + assert.ok(output.includes(`- ${provider.displayName}:`)); + for (const line of provider.installInstructions.split('\n')) { + assert.ok(output.includes(line)); + } + } + }); + }); + describe('printComplete()', function () { it('should not throw when called with settings object', function () { const settings = { diff --git a/tests/fixtures/copilot/empty.jsonl b/tests/fixtures/copilot/empty.jsonl new file mode 100644 index 00000000..4f9866f9 --- /dev/null +++ b/tests/fixtures/copilot/empty.jsonl @@ -0,0 +1 @@ +{"type":"session.mcp_servers_loaded","data":{"servers":[]},"ephemeral":true} diff --git a/tests/fixtures/copilot/failure.jsonl b/tests/fixtures/copilot/failure.jsonl new file mode 100644 index 00000000..fc5ae8d5 --- /dev/null +++ b/tests/fixtures/copilot/failure.jsonl @@ -0,0 +1,4 @@ +{"type":"assistant.message_start","data":{"messageId":"m1","phase":"final_answer"},"ephemeral":true} +{"type":"assistant.message_delta","data":{"messageId":"m1","deltaContent":"I could not complete the request."},"ephemeral":true} +{"type":"assistant.message","data":{"messageId":"m1","content":"I could not complete the request.","outputTokens":9}} +{"type":"result","sessionId":"s1","exitCode":1,"usage":{}} diff --git a/tests/fixtures/copilot/reasoning.jsonl b/tests/fixtures/copilot/reasoning.jsonl new file mode 100644 index 00000000..ad9c012f --- /dev/null +++ b/tests/fixtures/copilot/reasoning.jsonl @@ -0,0 +1,5 @@ +{"type":"assistant.turn_start","data":{"turnId":"0","model":"gpt-5.3-codex"}} +{"type":"assistant.reasoning","data":{"reasoningId":"r1","content":"Let me think about this."}} +{"type":"assistant.message_start","data":{"messageId":"m1","phase":"final_answer"},"ephemeral":true} +{"type":"assistant.message","data":{"messageId":"m1","content":"answer","outputTokens":5}} +{"type":"result","sessionId":"s1","exitCode":0,"usage":{}} diff --git a/tests/fixtures/copilot/text.jsonl b/tests/fixtures/copilot/text.jsonl new file mode 100644 index 00000000..d8a629e7 --- /dev/null +++ b/tests/fixtures/copilot/text.jsonl @@ -0,0 +1,9 @@ +{"type":"session.tools_updated","data":{"model":"gpt-5.3-codex"},"ephemeral":true} +{"type":"user.message","data":{"content":"ping"}} +{"type":"assistant.turn_start","data":{"turnId":"0","model":"gpt-5.3-codex"}} +{"type":"assistant.message_start","data":{"messageId":"m1","phase":"final_answer"},"ephemeral":true} +{"type":"assistant.message_delta","data":{"messageId":"m1","deltaContent":"pong"},"ephemeral":true} +{"type":"assistant.message","data":{"messageId":"m1","model":"gpt-5.3-codex","content":"pong","toolRequests":[],"outputTokens":21}} +{"type":"assistant.turn_end","data":{"turnId":"0"}} +{"type":"assistant.idle","data":{},"ephemeral":true} +{"type":"result","sessionId":"s1","exitCode":0,"usage":{"premiumRequests":0}} diff --git a/tests/fixtures/copilot/tool-object-args.jsonl b/tests/fixtures/copilot/tool-object-args.jsonl new file mode 100644 index 00000000..38d31e7f --- /dev/null +++ b/tests/fixtures/copilot/tool-object-args.jsonl @@ -0,0 +1,5 @@ +{"type":"assistant.turn_start","data":{"turnId":"0","model":"gpt-5.3-codex"}} +{"type":"tool.execution_start","data":{"toolCallId":"call_9","toolName":"bash","arguments":{"command":"cat hello.txt"}}} +{"type":"tool.execution_complete","data":{"toolCallId":"call_9","success":true,"result":{"content":"hi there\n"}}} +{"type":"assistant.message","data":{"messageId":"m1","content":"It printed hi there.","outputTokens":6}} +{"type":"result","sessionId":"s1","exitCode":0,"usage":{}} diff --git a/tests/fixtures/copilot/tool.jsonl b/tests/fixtures/copilot/tool.jsonl new file mode 100644 index 00000000..370b5a0c --- /dev/null +++ b/tests/fixtures/copilot/tool.jsonl @@ -0,0 +1,12 @@ +{"type":"session.tools_updated","data":{"model":"gpt-5.3-codex"},"ephemeral":true} +{"type":"assistant.turn_start","data":{"turnId":"0","model":"gpt-5.3-codex"}} +{"type":"assistant.message_start","data":{"messageId":"m1","phase":"commentary"},"ephemeral":true} +{"type":"assistant.message","data":{"messageId":"m1","phase":"commentary","content":"Creating hello.txt now.","toolRequests":[{"toolCallId":"call_1","name":"apply_patch","arguments":"*** Begin Patch\n*** Add File: hello.txt\n+hi there\n*** End Patch\n","type":"custom"}],"outputTokens":181}} +{"type":"assistant.tool_call_delta","data":{"toolCallId":"call_1","toolName":"apply_patch","inputDelta":"***"},"ephemeral":true} +{"type":"tool.execution_start","data":{"toolCallId":"call_1","toolName":"apply_patch","arguments":"*** Begin Patch\n*** Add File: hello.txt\n+hi there\n*** End Patch\n","turnId":"0"}} +{"type":"tool.execution_partial_result","data":{"toolCallId":"call_1","partialOutput":"hi there\n"}} +{"type":"tool.execution_complete","data":{"toolCallId":"call_1","success":true,"result":{"content":"Added 1 file(s): hello.txt","detailedContent":"diff --git a/hello.txt b/hello.txt\n"}}} +{"type":"assistant.message_start","data":{"messageId":"m2","phase":"final_answer"},"ephemeral":true} +{"type":"assistant.message_delta","data":{"messageId":"m2","deltaContent":"Done. It printed: hi there"},"ephemeral":true} +{"type":"assistant.message","data":{"messageId":"m2","phase":"final_answer","content":"Done. It printed: hi there","outputTokens":42}} +{"type":"result","sessionId":"s1","exitCode":0,"usage":{"premiumRequests":0}} diff --git a/tests/fixtures/copilot/unknown-event.jsonl b/tests/fixtures/copilot/unknown-event.jsonl new file mode 100644 index 00000000..54aa9741 --- /dev/null +++ b/tests/fixtures/copilot/unknown-event.jsonl @@ -0,0 +1,7 @@ +{"type":"session.background_tasks_changed","data":{"tasks":[]},"ephemeral":true} +{"type":"assistant.future_event","data":{"whatever":true}} +{"type":"assistant.message_start","data":{"messageId":"m1"},"ephemeral":true} +{"type":"assistant.message_delta","data":{"messageId":"m1","deltaContent":"partial"},"ephemeral":true} +{"type":"assistant.message","data":{"messageId":"m1","content":"partial","outputTokens":3}} +{"type":"some.brand_new_namespace","data":{"x":1}} +{"type":"result","sessionId":"s1","exitCode":0,"usage":{}} diff --git a/tests/fixtures/fake-agent-scenarios/failing-agent.json b/tests/fixtures/fake-agent-scenarios/failing-agent.json new file mode 100644 index 00000000..52890990 --- /dev/null +++ b/tests/fixtures/fake-agent-scenarios/failing-agent.json @@ -0,0 +1,5 @@ +{ + "files": [], + "messages": ["Task failed: simulated fatal error."], + "exitCode": 1 +} diff --git a/tests/fixtures/fake-agent-scenarios/single-worker-success.json b/tests/fixtures/fake-agent-scenarios/single-worker-success.json new file mode 100644 index 00000000..15072d41 --- /dev/null +++ b/tests/fixtures/fake-agent-scenarios/single-worker-success.json @@ -0,0 +1,8 @@ +{ + "files": [{ "path": "output.txt", "content": "feature implemented\n" }], + "messages": [ + "Implementing the requested feature.", + "{\"summary\": \"Implemented the requested feature\", \"result\": \"Wrote output.txt\"}" + ], + "exitCode": 0 +} diff --git a/tests/fixtures/fake-agent-scenarios/validator-approve.json b/tests/fixtures/fake-agent-scenarios/validator-approve.json new file mode 100644 index 00000000..4131fc03 --- /dev/null +++ b/tests/fixtures/fake-agent-scenarios/validator-approve.json @@ -0,0 +1,8 @@ +{ + "files": [{ "path": "validation-report.txt", "content": "APPROVED\n" }], + "messages": [ + "Validating the implementation.", + "{\"summary\": \"Validation passed\", \"result\": \"All checks green\"}" + ], + "exitCode": 0 +} diff --git a/tests/fixtures/fake-agent-scenarios/worker-success.json b/tests/fixtures/fake-agent-scenarios/worker-success.json new file mode 100644 index 00000000..3b0d0839 --- /dev/null +++ b/tests/fixtures/fake-agent-scenarios/worker-success.json @@ -0,0 +1,8 @@ +{ + "files": [{ "path": "implementation.txt", "content": "implementation output\n" }], + "messages": [ + "Implementing the requested feature.", + "{\"summary\": \"Implementation complete\", \"result\": \"Wrote implementation.txt\"}" + ], + "exitCode": 0 +} diff --git a/tests/fixtures/fake-agent/index.js b/tests/fixtures/fake-agent/index.js new file mode 100755 index 00000000..1a40a04d --- /dev/null +++ b/tests/fixtures/fake-agent/index.js @@ -0,0 +1,163 @@ +#!/usr/bin/env node +/** + * Fake provider CLI for deterministic end-to-end tests. + * + * Stands in for the real `claude` binary via ZEROSHOT_CLAUDE_COMMAND (see + * lib/settings.js:getClaudeCommand and src/agent-cli-provider/adapters/claude.ts:buildCommand). + * Everything above this process (CLI parsing, orchestrator, message bus, ledger, + * trigger/logic engine, agent spawning, stream-json parsing, hooks, worktree + * isolation) runs for real; only the model's cognition is faked. + * + * Argv contract (buildCommand): [...prefixArgs, '--print', '--input-format', 'text', + * ...outputArgs, ...schemaArgs, ...modelArgs, ...autoApproveArgs, ...sessionArgs, context] + * The prompt/context is always the LAST argv element. + * + * Scenario selection: + * FAKE_AGENT_SCENARIO= Used for every agent by default. + * FAKE_AGENT_SCENARIO_ Overrides the default for one agent id. + * + * Per-agent override convention (not in the original issue spec, needed for multi-agent + * scenarios): ZEROSHOT_CLAUDE_COMMAND is a single process-wide env var, so every agent in + * a cluster invokes the exact same fake-agent command. The only per-invocation signal + * available is the prompt text itself (the last argv element), which embeds the agent's + * configured `prompt` string verbatim (see src/agent/agent-context-sections.js + * buildHeaderContext). Fixture configs that need distinct per-agent scenarios must include + * the literal marker `FAKE_AGENT_ID=` somewhere in that agent's `prompt` field; this + * script extracts it from the context and uses it to look up + * FAKE_AGENT_SCENARIO_, falling back to FAKE_AGENT_SCENARIO. + * + * Scenario JSON shape: + * { + * "files": [{ "path": "relative/path.txt", "content": "..." }], + * "edits": [{ "path": "relative/path.txt", "find": "...", "replace": "..." }], + * "messages": ["assistant message 1", "assistant message 2"], + * "exitCode": 0, + * "delayMs": 0 + * } + * files/edits are applied relative to process.cwd() (proves worktree cwd injection). + */ + +const fs = require('fs'); +const path = require('path'); + +function extractAgentId(context) { + const match = /FAKE_AGENT_ID=([A-Za-z0-9_-]+)/.exec(context); + return match ? match[1] : null; +} + +function resolveScenarioPath(context) { + const agentId = extractAgentId(context); + if (agentId) { + const perAgentKey = `FAKE_AGENT_SCENARIO_${agentId.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`; + if (process.env[perAgentKey]) { + return process.env[perAgentKey]; + } + } + return process.env.FAKE_AGENT_SCENARIO; +} + +function loadScenario(context) { + const scenarioPath = resolveScenarioPath(context); + if (!scenarioPath) { + throw new Error('fake-agent: FAKE_AGENT_SCENARIO (or a per-agent override) must be set'); + } + const raw = fs.readFileSync(scenarioPath, 'utf8'); + return JSON.parse(raw); +} + +function applyFiles(scenario) { + for (const file of scenario.files || []) { + const target = path.resolve(process.cwd(), file.path); + process.stderr.write(`fake-agent: writing ${target}\n`); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, file.content ?? ''); + } +} + +function applyEdits(scenario) { + for (const edit of scenario.edits || []) { + const target = path.resolve(process.cwd(), edit.path); + const current = fs.readFileSync(target, 'utf8'); + fs.writeFileSync(target, current.split(edit.find).join(edit.replace)); + } +} + +function parseArgv(argv) { + const outputFormatIndex = argv.indexOf('--output-format'); + const modelIndex = argv.indexOf('--model'); + return { + outputFormat: outputFormatIndex >= 0 ? argv[outputFormatIndex + 1] : null, + model: modelIndex >= 0 ? argv[modelIndex + 1] : null, + context: argv[argv.length - 1] || '', + }; +} + +function emitEvent(event) { + process.stdout.write(`${JSON.stringify(event)}\n`); +} + +function emitMessages(messages) { + for (const text of messages) { + emitEvent({ + type: 'assistant', + message: { content: [{ type: 'text', text }] }, + }); + } +} + +function emitResult(scenario, messages) { + const exitCode = scenario.exitCode ?? 0; + const lastMessage = messages[messages.length - 1] ?? ''; + emitEvent({ + type: 'result', + subtype: exitCode === 0 ? 'success' : 'error', + is_error: exitCode !== 0, + result: lastMessage, + total_cost_usd: 0, + duration_ms: 1, + usage: { input_tokens: 1, output_tokens: 1 }, + }); +} + +function isVersionProbe(argv) { + // src/preflight.js:getClaudeVersion runs ` --version` (and only that) + // to detect CLI presence, with no scenario/task context. Must not touch + // FAKE_AGENT_SCENARIO or write files for this bare capability probe. + return argv.includes('--version') && !argv.includes('--print'); +} + +function main() { + const argv = process.argv.slice(2); + if (isVersionProbe(argv)) { + process.stdout.write('1.0.0 (fake-agent)\n'); + process.exit(0); + } + + const { outputFormat, model, context } = parseArgv(argv); + const scenario = loadScenario(context); + process.stderr.write( + `fake-agent: output-format=${outputFormat} model=${model} cwd=${process.cwd()}\n` + ); + + const exitCode = scenario.exitCode ?? 0; + const messages = scenario.messages || []; + + if (exitCode === 0) { + applyFiles(scenario); + applyEdits(scenario); + } + + const run = () => { + emitMessages(messages); + emitResult(scenario, messages); + process.exit(exitCode); + }; + + if (scenario.delayMs) { + setTimeout(run, scenario.delayMs); + } else { + run(); + } +} + +main(); diff --git a/tests/fixtures/kiro/auth-failure.jsonl b/tests/fixtures/kiro/auth-failure.jsonl new file mode 100644 index 00000000..dacd5172 --- /dev/null +++ b/tests/fixtures/kiro/auth-failure.jsonl @@ -0,0 +1 @@ +{"jsonrpc":"2.0","id":2,"error":{"code":401,"message":"authentication required: run kiro auth login"}} diff --git a/tests/fixtures/kiro/cancelled.jsonl b/tests/fixtures/kiro/cancelled.jsonl new file mode 100644 index 00000000..5c194f50 --- /dev/null +++ b/tests/fixtures/kiro/cancelled.jsonl @@ -0,0 +1,2 @@ +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"agent_message_chunk","messageId":"msg-3","content":{"type":"text","text":"stopped"}}}} +{"jsonrpc":"2.0","id":5,"result":{"stopReason":"cancelled","error":"cancelled by user","usage":{"inputTokens":1,"outputTokens":1,"cacheReadInputTokens":0,"cacheCreationInputTokens":0}}} diff --git a/tests/fixtures/kiro/empty.jsonl b/tests/fixtures/kiro/empty.jsonl new file mode 100644 index 00000000..35f6854d --- /dev/null +++ b/tests/fixtures/kiro/empty.jsonl @@ -0,0 +1,2 @@ +{"jsonrpc":"2.0","method":"_kiro.dev/progress","params":{"message":"ignored extension"}} +{"jsonrpc":"2.0","id":2,"result":{"sessionId":"kiro-session-1"}} diff --git a/tests/fixtures/kiro/malformed.jsonl b/tests/fixtures/kiro/malformed.jsonl new file mode 100644 index 00000000..a5a5387a --- /dev/null +++ b/tests/fixtures/kiro/malformed.jsonl @@ -0,0 +1,2 @@ +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"agent_message_chunk","messageId":"msg-4","content":{"type":"text","text":"ok"}}}} +{bad json diff --git a/tests/fixtures/kiro/text.jsonl b/tests/fixtures/kiro/text.jsonl new file mode 100644 index 00000000..662b539f --- /dev/null +++ b/tests/fixtures/kiro/text.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"agent_message_chunk","messageId":"msg-1","content":{"type":"text","text":"Hello"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"agent_message_chunk","messageId":"msg-1","content":{"type":"text","text":"Hello from Kiro"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"usage_update","usage":{"inputTokens":7,"outputTokens":3,"cacheReadInputTokens":1,"cacheCreationInputTokens":0}}}} +{"jsonrpc":"2.0","id":3,"result":{"stopReason":"end_turn"}} diff --git a/tests/fixtures/kiro/thought.jsonl b/tests/fixtures/kiro/thought.jsonl new file mode 100644 index 00000000..7a841f10 --- /dev/null +++ b/tests/fixtures/kiro/thought.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"agent_thought_chunk","messageId":"msg-thought-1","content":{"type":"text","text":"Need a plan"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"agent_thought_chunk","messageId":"msg-thought-1","content":{"type":"text","text":"Need a plan first"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"agent_message_chunk","messageId":"msg-thought-1","content":{"type":"text","text":"Done"}}}} +{"jsonrpc":"2.0","id":6,"result":{"stopReason":"end_turn"}} diff --git a/tests/fixtures/kiro/tool.jsonl b/tests/fixtures/kiro/tool.jsonl new file mode 100644 index 00000000..d59c5bfa --- /dev/null +++ b/tests/fixtures/kiro/tool.jsonl @@ -0,0 +1,6 @@ +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"tool_call","toolCallId":"tool-1","title":"bash","rawInput":{"command":"pwd"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"tool_call_update","toolCallId":"tool-1","status":"in_progress","rawOutput":"line1"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"tool_call_update","toolCallId":"tool-1","status":"completed","rawOutput":"line1\n/tmp/kiro"}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"agent_message_chunk","messageId":"msg-2","content":{"type":"text","text":"done"}}}} +{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"kiro-session-1","update":{"sessionUpdate":"usage_update","usage":{"inputTokens":9,"outputTokens":4,"cacheReadInputTokens":0,"cacheCreationInputTokens":0}}}} +{"jsonrpc":"2.0","id":4,"result":{"stopReason":"end_turn"}} diff --git a/tests/fixtures/pi/auth-failure.jsonl b/tests/fixtures/pi/auth-failure.jsonl new file mode 100644 index 00000000..affa5ba9 --- /dev/null +++ b/tests/fixtures/pi/auth-failure.jsonl @@ -0,0 +1,2 @@ +{"type":"session","version":3,"id":"pi-auth-failure","timestamp":"2026-07-09T00:00:00.000Z","cwd":"/tmp/pi"} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"auth failed"}],"usage":{"input":1,"output":1,"cacheRead":0,"cacheWrite":0},"stopReason":"error","errorMessage":"authentication required: run /login"},"toolResults":[]} diff --git a/tests/fixtures/pi/cancelled.jsonl b/tests/fixtures/pi/cancelled.jsonl new file mode 100644 index 00000000..505180df --- /dev/null +++ b/tests/fixtures/pi/cancelled.jsonl @@ -0,0 +1,2 @@ +{"type":"session","version":3,"id":"pi-cancelled","timestamp":"2026-07-09T00:00:00.000Z","cwd":"/tmp/pi"} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"stopped"}],"usage":{"input":1,"output":1,"cacheRead":0,"cacheWrite":0},"stopReason":"aborted","errorMessage":"cancelled by user"},"toolResults":[]} diff --git a/tests/fixtures/pi/command-failure.jsonl b/tests/fixtures/pi/command-failure.jsonl new file mode 100644 index 00000000..6eed6fee --- /dev/null +++ b/tests/fixtures/pi/command-failure.jsonl @@ -0,0 +1,4 @@ +{"type":"session","version":3,"id":"pi-command-failure","timestamp":"2026-07-09T00:00:00.000Z","cwd":"/tmp/pi"} +{"type":"tool_execution_start","toolCallId":"tool-err","toolName":"bash","args":{"command":"pi --bogus"}} +{"type":"tool_execution_end","toolCallId":"tool-err","toolName":"bash","result":"Unknown option --bogus","isError":true} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"command failed"}],"usage":{"input":3,"output":2,"cacheRead":0,"cacheWrite":0},"stopReason":"error","errorMessage":"Unknown option --bogus"},"toolResults":[]} diff --git a/tests/fixtures/pi/empty.jsonl b/tests/fixtures/pi/empty.jsonl new file mode 100644 index 00000000..440799af --- /dev/null +++ b/tests/fixtures/pi/empty.jsonl @@ -0,0 +1 @@ +{"type":"session","version":3,"id":"pi-empty","timestamp":"2026-07-09T00:00:00.000Z","cwd":"/tmp/pi"} diff --git a/tests/fixtures/pi/rate-limit.jsonl b/tests/fixtures/pi/rate-limit.jsonl new file mode 100644 index 00000000..a6d3cfe4 --- /dev/null +++ b/tests/fixtures/pi/rate-limit.jsonl @@ -0,0 +1,2 @@ +{"type":"session","version":3,"id":"pi-rate-limit","timestamp":"2026-07-09T00:00:00.000Z","cwd":"/tmp/pi"} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"slow down"}],"usage":{"input":1,"output":1,"cacheRead":0,"cacheWrite":0},"stopReason":"error","errorMessage":"rate limit exceeded; retry later"},"toolResults":[]} diff --git a/tests/fixtures/pi/text.jsonl b/tests/fixtures/pi/text.jsonl new file mode 100644 index 00000000..2a89af8d --- /dev/null +++ b/tests/fixtures/pi/text.jsonl @@ -0,0 +1,8 @@ +{"type":"session","version":3,"id":"pi-text","timestamp":"2026-07-09T00:00:00.000Z","cwd":"/tmp/pi"} +{"type":"agent_start"} +{"type":"turn_start"} +{"type":"message_start","message":{"role":"assistant","content":[]}} +{"type":"message_update","message":{"role":"assistant","content":[{"type":"text","text":"Hello from Pi"}]},"assistantMessageEvent":{"type":"text_delta","delta":"Hello from Pi"}} +{"type":"message_end","message":{"role":"assistant","content":[{"type":"text","text":"Hello from Pi"}],"usage":{"input":7,"output":3,"cacheRead":1,"cacheWrite":0},"stopReason":"stop"}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"Hello from Pi"}],"usage":{"input":7,"output":3,"cacheRead":1,"cacheWrite":0},"stopReason":"stop"},"toolResults":[]} +{"type":"agent_end","messages":[]} diff --git a/tests/fixtures/pi/tool.jsonl b/tests/fixtures/pi/tool.jsonl new file mode 100644 index 00000000..a81508c0 --- /dev/null +++ b/tests/fixtures/pi/tool.jsonl @@ -0,0 +1,10 @@ +{"type":"session","version":3,"id":"pi-tool","timestamp":"2026-07-09T00:00:00.000Z","cwd":"/tmp/pi"} +{"type":"agent_start"} +{"type":"turn_start"} +{"type":"tool_execution_start","toolCallId":"tool-1","toolName":"bash","args":{"command":"pwd"}} +{"type":"tool_execution_update","toolCallId":"tool-1","toolName":"bash","args":{"command":"pwd"},"partialResult":"line1"} +{"type":"tool_execution_end","toolCallId":"tool-1","toolName":"bash","result":"line1\n/tmp/pi","isError":false} +{"type":"message_start","message":{"role":"assistant","content":[]}} +{"type":"message_update","message":{"role":"assistant","content":[{"type":"text","text":"done"}]},"assistantMessageEvent":{"type":"text_delta","delta":"done"}} +{"type":"turn_end","message":{"role":"assistant","content":[{"type":"text","text":"done"}],"usage":{"input":9,"output":4,"cacheRead":0,"cacheWrite":0},"stopReason":"stop"},"toolResults":[]} +{"type":"agent_end","messages":[]} diff --git a/tests/git-pusher-pr-mode.test.js b/tests/git-pusher-pr-mode.test.js index d4a4fcbf..9783f2e5 100644 --- a/tests/git-pusher-pr-mode.test.js +++ b/tests/git-pusher-pr-mode.test.js @@ -122,23 +122,25 @@ describe('git-pusher --pr vs --ship (autoMerge)', function () { assert.notStrictEqual(prOptionsForPr, null); }); - it('resolveAutoMerge is the single source for the autoMerge decision', function () { - assert.strictEqual(Orchestrator.resolveAutoMerge({ ship: true }), true); - assert.strictEqual(Orchestrator.resolveAutoMerge({ pr: true }), false); - assert.strictEqual(Orchestrator.resolveAutoMerge({ pr: true, autoMerge: true }), true); + it('resolveRunPlan is the single source for the autoMerge decision', function () { + assert.strictEqual(Orchestrator.resolveRunPlan({ ship: true }).autoMerge, true); + assert.strictEqual(Orchestrator.resolveRunPlan({ pr: true }).autoMerge, false); + // Explicit autoMerge is ship-equivalent and is NOT clobbered (the old + // normalizeRunOptions overwrite bug reset this to false). + assert.strictEqual(Orchestrator.resolveRunPlan({ pr: true, autoMerge: true }).autoMerge, true); }); - it('buildPrOptions and git-pusher config cannot diverge (both derive from resolveAutoMerge)', function () { + it('buildPrOptions and git-pusher config cannot diverge (both derive from resolveRunPlan)', function () { const shipOptions = { ship: true }; const prOptions = { pr: true }; assert.strictEqual( Orchestrator.buildPrOptions(shipOptions, []).autoMerge, - Orchestrator.resolveAutoMerge(shipOptions) + Orchestrator.resolveRunPlan(shipOptions).autoMerge ); assert.strictEqual( Orchestrator.buildPrOptions(prOptions, []).autoMerge, - Orchestrator.resolveAutoMerge(prOptions) + Orchestrator.resolveRunPlan(prOptions).autoMerge ); }); }); diff --git a/tests/integration/copilot-mcp-docker.test.js b/tests/integration/copilot-mcp-docker.test.js new file mode 100644 index 00000000..7fd95d67 --- /dev/null +++ b/tests/integration/copilot-mcp-docker.test.js @@ -0,0 +1,153 @@ +/** + * Focused Docker-isolation proof for Copilot MCP config. + * + * The heavyweight `zeroshot run --docker` path (tests/integration/e2e-isolation-and-auto.test.sh) + * needs the real `zeroshot-cluster-base` image plus live Claude/Copilot credentials — none are + * available offline, and COPILOT_GITHUB_TOKEN is unset, so a live Copilot API call is impossible. + * + * This proves the Docker-specific piece for the MCP feature: the copilot adapter's inlined + * `--additional-mcp-config ` argument survives the container boundary intact. It builds the + * REAL copilot argv on the host with the compiled adapter (reading the repo `.mcp.json`, exactly as + * the agent spawn path does), then delivers it into a container via `docker exec -i + * ` — the exact mechanism IsolationManager.spawnInContainer uses. A fake `copilot` inside + * the container records the argv it received; the host asserts the inlined config arrived unchanged. + * + * Fully offline: no zeroshot-cluster-base image, no credentials, no Copilot API call. + * + * REQUIRES: Docker installed and running, and the node:20-slim image already pulled. SKIPS + * otherwise (or under CI) — the base image pull is left to the environment to avoid slow network + * pulls inside a test hook. + */ + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); +const IsolationManager = require('../../src/isolation-manager'); +const { prepareSingleAgentProviderCommand } = require('../../task-lib/provider-helper-runtime.js'); + +const IMAGE = 'node:20-slim'; +const REPO_ROOT = path.resolve(__dirname, '..', '..'); +const FAKE_COPILOT = path.join(REPO_ROOT, 'tests', 'e2e', 'fixtures', 'fake-copilot.js'); +const MCP_JSON = JSON.stringify({ + mcpServers: { demo: { command: 'demo-mcp-bin', args: ['--stdio'] } }, +}); + +function dockerCli(args, opts = {}) { + return spawnSync('docker', args, { encoding: 'utf8', ...opts }); +} + +function imagePresent(image) { + return dockerCli(['image', 'inspect', image]).status === 0; +} + +describe('copilot MCP config under Docker isolation', function () { + this.timeout(120000); + + let ctxDir; + let container; + let hostArgs; + + before(function () { + if (process.env.CI || !IsolationManager.isDockerAvailable() || !imagePresent(IMAGE)) { + this.skip(); + } + }); + + beforeEach(function () { + ctxDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-copilot-mcp-docker-')); + fs.writeFileSync(path.join(ctxDir, '.mcp.json'), MCP_JSON); + fs.copyFileSync(FAKE_COPILOT, path.join(ctxDir, 'fake-copilot.js')); + const shim = path.join(ctxDir, 'copilot'); + fs.writeFileSync(shim, '#!/bin/sh\nexec node "$(dirname "$0")/fake-copilot.js" "$@"\n', { + mode: 0o755, + }); + fs.chmodSync(shim, 0o755); + + // Build the copilot argv on the host exactly as the agent spawn path does: the repo .mcp.json + // content is inlined into options.mcpConfig, and the copilot adapter emits it verbatim. + const content = fs.readFileSync(path.join(ctxDir, '.mcp.json'), 'utf8').trim(); + const prepared = prepareSingleAgentProviderCommand({ + provider: 'copilot', + context: 'do the work', + options: { + autoApprove: true, + outputFormat: 'json', + cwd: '/app', + mcpConfig: [content], + cliFeatures: { + supportsJsonOutput: true, + supportsAllowAll: true, + supportsNoAskUser: true, + supportsAddDir: true, + supportsMcpConfig: true, + }, + }, + }); + hostArgs = prepared.commandSpec.args; + container = null; + }); + + afterEach(function () { + if (container) { + dockerCli(['rm', '-f', container], { stdio: 'pipe' }); + container = null; + } + if (ctxDir) { + fs.rmSync(ctxDir, { recursive: true, force: true }); + } + }); + + it('delivers --additional-mcp-config to copilot intact through docker exec', function () { + // Sanity: the host-built argv must carry the inlined MCP config (else the container proof is + // vacuous). + const hostFlag = hostArgs.indexOf('--additional-mcp-config'); + assert.ok( + hostFlag >= 0 && hostArgs[hostFlag + 1] === MCP_JSON, + 'adapter did not emit MCP flag' + ); + + const run = dockerCli(['run', '-d', IMAGE, 'tail', '-f', '/dev/null']); + assert.strictEqual(run.status, 0, `docker run failed: ${run.stderr}`); + container = run.stdout.trim(); + + assert.strictEqual(dockerCli(['exec', container, 'mkdir', '-p', '/app']).status, 0); + const cp = dockerCli(['cp', `${ctxDir}/.`, `${container}:/app`]); + assert.strictEqual(cp.status, 0, `docker cp failed: ${cp.stderr}`); + dockerCli(['exec', container, 'chmod', '+x', '/app/copilot']); + + // Mirror IsolationManager.spawnInContainer: `docker exec -i ` with the + // provider argv the host built. -w /app makes the fake copilot record argv under /app. + const exec = dockerCli([ + 'exec', + '-i', + '-w', + '/app', + '-e', + 'PATH=/app:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin', + container, + 'copilot', + ...hostArgs, + ]); + assert.strictEqual( + exec.status, + 0, + `copilot failed inside container:\nSTDOUT:\n${exec.stdout}\nSTDERR:\n${exec.stderr}` + ); + + const out = dockerCli(['exec', container, 'cat', '/app/copilot-received-argv.json']); + assert.strictEqual(out.status, 0, `could not read recorded argv: ${out.stderr}`); + const argv = JSON.parse(out.stdout); + + // The argv the fake copilot received inside the container must match what the host sent, with + // the inlined MCP config unchanged. + assert.deepStrictEqual(argv, hostArgs, 'argv mutated crossing the container boundary'); + const flagIndex = argv.indexOf('--additional-mcp-config'); + assert.ok( + flagIndex >= 0, + `no --additional-mcp-config in container argv: ${JSON.stringify(argv)}` + ); + assert.strictEqual(argv[flagIndex + 1], MCP_JSON); + }); +}); diff --git a/tests/integration/e2e-framework.test.ts b/tests/integration/e2e-framework.test.ts deleted file mode 100644 index 885d3d00..00000000 --- a/tests/integration/e2e-framework.test.ts +++ /dev/null @@ -1,165 +0,0 @@ -/** - * E2E Test for Vibe Framework - * - * Tests that the zeroshot framework can successfully: - * 1. Run a simple single-agent task - * 2. Track the task lifecycle - * 3. Retrieve task logs - * 4. Clean up properly - */ - -import { test, expect } from '@playwright/test'; -import { exec } from 'child_process'; -import { promisify } from 'util'; -import * as fs from 'fs/promises'; -import * as path from 'path'; - -const execAsync = promisify(exec); - -// Test configuration -const VIBE_CLI = 'zeroshot'; -const TEST_TIMEOUT = 120000; // 2 minutes for task execution - -test.describe('Vibe Framework E2E', () => { - let taskId: string | null = null; - - test.afterEach(async () => { - // Clean up: kill task if still running - if (taskId) { - try { - await execAsync(`${VIBE_CLI} kill ${taskId}`); - } catch (error) { - // Task might already be completed, ignore errors - } - taskId = null; - } - }); - - test('should run a simple single-agent task successfully', async () => { - test.setTimeout(TEST_TIMEOUT); - // Step 1: Verify zeroshot CLI is installed and available - const { stdout: versionOutput } = await execAsync(`${VIBE_CLI} --version`); - expect(versionOutput).toContain('1.0.0'); - console.log('[OK] Vibe CLI is available:', versionOutput.trim()); - - // Step 2: Run a simple task that should complete quickly - const taskPrompt = 'Echo hello from zeroshot and exit successfully'; - let runOutput = ''; - try { - const result = await execAsync( - `${VIBE_CLI} task run '${taskPrompt}' 2>&1`, - { timeout: 10000 } // 10 seconds to spawn - ); - runOutput = result.stdout; - } catch (error: any) { - // Command might exit with non-zero but still spawn task successfully - runOutput = error.stdout || error.stderr || ''; - if (!runOutput.includes('spawned:')) { - throw error; - } - } - - // Extract task ID from output (format: "Task spawned: --") - // Note: Output contains ANSI color codes, so we need to account for those - const taskIdMatch = runOutput.match(/spawned:\s+(?:\x1b\[\d+m)*([a-z]+-[a-z]+-\d+)/); - if (!taskIdMatch) { - console.error('Failed to extract task ID from output'); - console.error('Output:', runOutput); - throw new Error(`Could not extract task ID from output`); - } - taskId = taskIdMatch[1]; - console.log('[OK] Task spawned with ID:', taskId); - - // Step 3: Verify task appears in list - const { stdout: listOutput } = await execAsync(`${VIBE_CLI} list`); - expect(listOutput).toContain(taskId); - console.log('[OK] Task appears in list'); - - // Step 4: Wait for task to complete (poll status) - let attempts = 0; - const maxAttempts = 24; // 2 minutes with 5-second intervals - let taskCompleted = false; - - while (attempts < maxAttempts && !taskCompleted) { - await new Promise((resolve) => setTimeout(resolve, 5000)); // Wait 5 seconds - - try { - const { stdout: statusOutput } = await execAsync(`${VIBE_CLI} status ${taskId}`); - - if ( - statusOutput.toLowerCase().includes('completed') || - statusOutput.toLowerCase().includes('stopped') - ) { - taskCompleted = true; - console.log('[OK] Task completed'); - } else if ( - statusOutput.toLowerCase().includes('failed') || - statusOutput.toLowerCase().includes('error') - ) { - throw new Error(`Task failed: ${statusOutput}`); - } else { - console.log(`[INFO] Task still running (attempt ${attempts + 1}/${maxAttempts})`); - } - } catch (error) { - // Status command might fail if task just completed, try logs instead - try { - const { stdout: logsOutput } = await execAsync(`${VIBE_CLI} logs ${taskId}`); - if (logsOutput.includes('Task completed') || logsOutput.includes('exited')) { - taskCompleted = true; - console.log('[OK] Task completed (detected via logs)'); - } - } catch (logError) { - console.log(`[INFO] Could not get logs (attempt ${attempts + 1}/${maxAttempts})`); - } - } - - attempts++; - } - - expect(taskCompleted).toBe(true); - - // Step 5: Retrieve and verify logs - const { stdout: logsOutput } = await execAsync(`${VIBE_CLI} logs ${taskId}`); - expect(logsOutput.length).toBeGreaterThan(0); - console.log('[OK] Task logs retrieved'); - console.log('Task logs snippet:', logsOutput.substring(0, 500)); - }); - - test('should list tasks correctly', async () => { - // Verify list command works without errors - const { stdout: listOutput } = await execAsync(`${VIBE_CLI} list`); - expect(listOutput).toBeTruthy(); - console.log('[OK] List command works'); - }); - - test('should show settings', async () => { - // Verify settings command works - const { stdout: settingsOutput } = await execAsync(`${VIBE_CLI} settings`); - expect(settingsOutput).toBeTruthy(); - console.log('[OK] Settings command works'); - }); - - test('should verify zeroshot database is accessible', async () => { - // Check that zeroshot directory exists - const homeDir = process.env.HOME || process.env.USERPROFILE || '/home/eivind'; - const zeroshotDir = path.join(homeDir, '.zeroshot'); - - try { - await fs.access(zeroshotDir); - console.log('[OK] Vibe directory exists:', zeroshotDir); - - // Check for clusters.json - const clustersFile = path.join(zeroshotDir, 'clusters.json'); - try { - const clustersData = await fs.readFile(clustersFile, 'utf-8'); - const clusters = JSON.parse(clustersData); - console.log('[OK] Clusters metadata accessible, count:', Object.keys(clusters).length); - } catch { - console.log('[INFO] No clusters.json found (normal for fresh install)'); - } - } catch (error) { - // Vibe directory might not exist if no tasks have been run yet - console.log('[INFO] Vibe directory not found or empty (normal for fresh install)'); - } - }); -}); diff --git a/tests/integration/hello-e2e.test.ts b/tests/integration/hello-e2e.test.ts deleted file mode 100644 index 3d17c315..00000000 --- a/tests/integration/hello-e2e.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Hello Vibe Test - * - * Simple test that echoes "hello from zeroshot" and exits successfully. - */ - -import { test, expect } from '@playwright/test'; - -test('should echo hello from zeroshot', async () => { - // Echo hello from zeroshot - console.log('hello from zeroshot'); - - // Exit successfully - expect(true).toBe(true); -}); diff --git a/tests/integration/message-bus.test.js b/tests/integration/message-bus.test.js index 13007802..94d93ae4 100644 --- a/tests/integration/message-bus.test.js +++ b/tests/integration/message-bus.test.js @@ -647,6 +647,25 @@ function registerTaskIdTokenTotalsTest() { }); } +function registerCloseTests() { + describe('Close lifecycle', () => { + it('should be idempotent and ignore publishes after close', () => { + messageBus.close(); + messageBus.close(); + ledger.close(); + + const published = messageBus.publish({ + cluster_id: 'closed-cluster', + topic: 'AFTER_CLOSE', + sender: 'test', + content: { text: 'ignored' }, + }); + + assert.strictEqual(published, null); + }); + }); +} + describe('MessageBus Integration', function () { this.timeout(10000); @@ -672,4 +691,5 @@ describe('MessageBus Integration', function () { registerEventEmissionTests(); registerBatchPublishingTests(); registerTaskIdCausalLinkingTests(); + registerCloseTests(); }); diff --git a/tests/integration/orchestrator-worktree.test.js b/tests/integration/orchestrator-worktree.test.js index 99a17db8..c537a9ce 100644 --- a/tests/integration/orchestrator-worktree.test.js +++ b/tests/integration/orchestrator-worktree.test.js @@ -81,7 +81,7 @@ function registerWorktreeHooks() { execSync('git config user.email "test@test.com"', { cwd: testRepoDir, stdio: 'pipe' }); execSync('git config user.name "Test User"', { cwd: testRepoDir, stdio: 'pipe' }); fs.writeFileSync(path.join(testRepoDir, 'README.md'), '# Test Repo'); - execSync('git add -A', { cwd: testRepoDir, stdio: 'pipe' }); + execSync('git add README.md', { cwd: testRepoDir, stdio: 'pipe' }); execSync('git commit -m "Initial commit"', { cwd: testRepoDir, stdio: 'pipe' }); mockRunner = new MockTaskRunner(); diff --git a/tests/preflight.test.js b/tests/preflight.test.js index 83bcddcb..e2d45839 100644 --- a/tests/preflight.test.js +++ b/tests/preflight.test.js @@ -15,6 +15,7 @@ const { execSync, spawnSync } = require('child_process'); const path = require('path'); const fs = require('fs'); const os = require('os'); +const { VALID_PROVIDERS, getProviderMetadata } = require('../lib/provider-names'); const { runPreflight, @@ -318,51 +319,197 @@ function defineRunPreflightTests() { const originalPath = process.env.PATH; process.env.PATH = '/nonexistent'; - const result = await runPreflight({ - requireGh: false, - requireDocker: false, - quiet: true, - provider: 'claude', - }); - - process.env.PATH = originalPath; + try { + const result = await runPreflight({ + requireGh: false, + requireDocker: false, + quiet: true, + provider: 'claude', + }); - expect(result.valid).to.be.false; - expect(result.errors.join('')).to.include('Claude command not available'); + const errorText = result.errors.join(''); + const metadata = getProviderMetadata('claude'); + + expect(result.valid).to.be.false; + expect(errorText).to.include('Claude command not available'); + for (const line of metadata.installInstructions.split('\n')) { + expect(errorText).to.include(line); + } + expect(errorText).to.include(`Then run: ${metadata.binary} --version`); + } finally { + process.env.PATH = originalPath; + } }); - it('should fail when Codex CLI is missing', async () => { + it('should keep custom Claude command recovery when override is missing', async () => { const originalPath = process.env.PATH; process.env.PATH = '/nonexistent'; - const result = await runPreflight({ - requireGh: false, - requireDocker: false, - quiet: true, - provider: 'codex', - }); + try { + const result = await runPreflight({ + requireGh: false, + requireDocker: false, + quiet: true, + provider: 'claude', + claudeCommand: 'ccr code', + }); - process.env.PATH = originalPath; + const errorText = result.errors.join(''); + expect(result.valid).to.be.false; + expect(errorText).to.include("Command 'ccr code' not found"); + expect(errorText).to.include( + 'Update claudeCommand: zeroshot settings set claudeCommand "your-command"' + ); + } finally { + process.env.PATH = originalPath; + } + }); + + it('should fail when any registry-backed provider CLI is missing', async () => { + const originalPath = process.env.PATH; + process.env.PATH = '/nonexistent'; - expect(result.valid).to.be.false; - expect(result.errors.join('')).to.include('Codex CLI not available'); + try { + const registryBackedProviders = VALID_PROVIDERS.filter( + (provider) => getProviderMetadata(provider).command.kind !== 'configured-claude' + ); + + for (const provider of registryBackedProviders) { + const result = await runPreflight({ + requireGh: false, + requireDocker: false, + quiet: true, + provider, + }); + + expect(result.valid).to.be.false; + if (provider === 'gateway') { + expect(result.errors.join('')).to.include('Gateway provider not configured'); + continue; + } + expect(result.errors.join('')).to.include( + `${getProviderMetadata(provider).displayName} CLI not available` + ); + } + } finally { + process.env.PATH = originalPath; + } }); - it('should fail when Gemini CLI is missing', async () => { + it('should allow a configured gateway provider when PATH has no node shim', async () => { + const settingsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'preflight-gateway-')); + const settingsFile = path.join(settingsDir, 'settings.json'); const originalPath = process.env.PATH; + const originalSettingsFile = process.env.ZEROSHOT_SETTINGS_FILE; + + fs.writeFileSync( + settingsFile, + JSON.stringify( + { + defaultProvider: 'gateway', + providerSettings: { + gateway: { + baseUrl: 'http://127.0.0.1:11434/v1', + apiKey: 'gateway-key', + model: 'openrouter/test-model', + toolPolicy: { + roots: ['.'], + commands: ['node'], + }, + }, + }, + }, + null, + 2 + ) + ); + + process.env.ZEROSHOT_SETTINGS_FILE = settingsFile; process.env.PATH = '/nonexistent'; - const result = await runPreflight({ - requireGh: false, - requireDocker: false, - quiet: true, - provider: 'gemini', - }); + try { + const result = await runPreflight({ + requireGh: false, + requireDocker: false, + quiet: true, + provider: 'gateway', + }); - process.env.PATH = originalPath; + expect(result.valid).to.be.true; + expect(result.errors).to.deep.equal([]); + } finally { + process.env.PATH = originalPath; + if (originalSettingsFile === undefined) { + delete process.env.ZEROSHOT_SETTINGS_FILE; + } else { + process.env.ZEROSHOT_SETTINGS_FILE = originalSettingsFile; + } + fs.rmSync(settingsDir, { recursive: true, force: true }); + } + }); + + it('should fail Pi preflight when the command exists but help/version probing fails', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'preflight-pi-')); + const originalPath = process.env.PATH; + const piPath = path.join(tempDir, 'pi'); + + fs.writeFileSync( + piPath, + '#!/usr/bin/env node\nif (process.argv.includes("--help") || process.argv.includes("--version")) process.exit(9);\nprocess.exit(0);\n', + { mode: 0o755 } + ); + process.env.PATH = `${tempDir}${path.delimiter}${originalPath || ''}`; - expect(result.valid).to.be.false; - expect(result.errors.join('')).to.include('Gemini CLI not available'); + try { + const result = await runPreflight({ + requireGh: false, + requireDocker: false, + quiet: true, + provider: 'pi', + }); + + expect(result.valid).to.be.false; + expect(result.errors.join('')).to.include( + 'Command "pi" is installed but did not produce usable --help/--version output' + ); + expect(result.errors.join('')).to.include( + 'npm install -g --ignore-scripts @earendil-works/pi-coding-agent@0.80.3' + ); + } finally { + process.env.PATH = originalPath; + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it('should fail Copilot preflight when the command exists but help/version probing fails', async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'preflight-copilot-')); + const originalPath = process.env.PATH; + const copilotPath = path.join(tempDir, 'copilot'); + + fs.writeFileSync( + copilotPath, + '#!/usr/bin/env node\nif (process.argv.includes("--help") || process.argv.includes("--version")) process.exit(9);\nprocess.exit(0);\n', + { mode: 0o755 } + ); + process.env.PATH = `${tempDir}${path.delimiter}${originalPath || ''}`; + + try { + const result = await runPreflight({ + requireGh: false, + requireDocker: false, + quiet: true, + provider: 'copilot', + }); + + expect(result.valid).to.be.false; + expect(result.errors.join('')).to.include( + 'Command "copilot" is installed but did not produce usable --help/--version output' + ); + expect(result.errors.join('')).to.include('npm install -g @github/copilot'); + } finally { + process.env.PATH = originalPath; + fs.rmSync(tempDir, { recursive: true, force: true }); + } }); it('should not require Claude auth when CLI is installed', async function () { diff --git a/tests/provider-cli-builder.test.js b/tests/provider-cli-builder.test.js index b01ed216..def1fa13 100644 --- a/tests/provider-cli-builder.test.js +++ b/tests/provider-cli-builder.test.js @@ -169,6 +169,14 @@ describe('Gemini provider helper builder', function () { assert.ok(result.args.includes('--output-format')); assert.ok(result.args.includes('stream-json')); }); + + it('marks headless workspaces trusted for noninteractive Gemini runs', function () { + const result = buildCommand('gemini', 'test', { + cwd: '/tmp/project', + }); + + assert.strictEqual(result.env.GEMINI_CLI_TRUST_WORKSPACE, 'true'); + }); }); describe('Opencode provider helper builder', function () { @@ -226,6 +234,66 @@ describe('Opencode provider helper builder', function () { assert.ok(result.args.includes('--variant')); assert.ok(result.args.includes('high')); }); + + it('prefers --dir over --cwd when opencode supports both', function () { + const result = buildCommand('opencode', 'test', { + cwd: '/tmp/worktree', + cliFeatures: { supportsDir: true, supportsCwd: true }, + }); + + assert.ok(result.args.includes('--dir')); + assert.ok(!result.args.includes('--cwd')); + assert.deepStrictEqual(result.args.slice(1, 3), ['--dir', '/tmp/worktree']); + }); + + it('falls back to --cwd when opencode does not support --dir', function () { + const result = buildCommand('opencode', 'test', { + cwd: '/tmp/worktree', + cliFeatures: { supportsDir: false, supportsCwd: true }, + }); + + assert.ok(!result.args.includes('--dir')); + assert.ok(result.args.includes('--cwd')); + assert.deepStrictEqual(result.args.slice(1, 3), ['--cwd', '/tmp/worktree']); + }); +}); + +describe('Pi provider helper builder', function () { + it('fails closed when resume or continue session control is requested', function () { + assert.throws( + () => + buildCommand('pi', 'test context', { + resumeSessionId: 'session-123', + }), + /does not support resume\/continue session control/ + ); + + assert.throws( + () => + buildCommand('pi', 'test context', { + continueSession: true, + }), + /does not support resume\/continue session control/ + ); + }); +}); + +describe('Copilot provider helper builder', function () { + it('emits auto-approve flags when autoApprove is requested', function () { + const spec = buildCommand('copilot', 'test context', { + autoApprove: true, + }); + assert.ok(spec.args.includes('--allow-all')); + assert.ok(spec.args.includes('--no-ask-user')); + }); + + it('warns and ignores unsupported session control instead of failing closed', function () { + const resumed = buildCommand('copilot', 'test context', { + resumeSessionId: 'session-123', + }); + assert.ok(!resumed.args.includes('--resume')); + assert.ok(resumed.warnings.some((warning) => warning.code === 'unsupported-session-control')); + }); }); describe('Claude provider helper builder', function () { diff --git a/tests/providers/detection.test.js b/tests/providers/detection.test.js index ef02018d..0020b0e7 100644 --- a/tests/providers/detection.test.js +++ b/tests/providers/detection.test.js @@ -1,4 +1,7 @@ const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); const { commandExists, getCommandPath, @@ -27,4 +30,22 @@ describe('Provider CLI detection', () => { assert.ok(typeof version === 'string'); assert.ok(version.length > 0); }); + + it('ignores failing fallback help/version probes', () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'provider-detect-')); + const cliPath = path.join(tempDir, 'pi'); + + fs.writeFileSync( + cliPath, + '#!/usr/bin/env node\nif (process.argv.includes("--help")) process.exit(0);\nif (process.argv.includes("--version")) { process.stdout.write("0.80.3\\n"); process.exit(0); }\nprocess.stderr.write("unknown option -h\\n"); process.exit(1);\n', + { mode: 0o755 } + ); + + try { + assert.strictEqual(getHelpOutput(cliPath), ''); + assert.strictEqual(getVersionOutput(cliPath), '0.80.3'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); }); diff --git a/tests/quality-gate-schema.test.js b/tests/quality-gate-schema.test.js new file mode 100644 index 00000000..abc5e605 --- /dev/null +++ b/tests/quality-gate-schema.test.js @@ -0,0 +1,72 @@ +const assert = require('assert'); +const Ajv = require('ajv'); + +const { buildQualityGateSchema } = require('../src/agent/agent-quality-gate-schema'); + +// Regression for #614: the quality-gate schema is injected into validator agent +// output schemas and compiled by strict-mode AJV consumers. Union +// `type: ['string','number']` arrays throw under strict mode +// ("strict mode: use allowUnionTypes ... (strictTypes)"), which crashed +// validation before the validator's output could be checked and burned +// validator retries until runs failed. +// +// The prior tests only asserted whether the schema was *injected*, never that it +// *compiles* under a strict AJV. AJV's default `new Ajv()` only LOGS strictTypes +// (it does not throw), so a naive test would pass on the buggy schema too — that +// is exactly the gap that let this ship. These tests compile with the throwing +// `{ strict: true }` config that reproduces the runtime failure, and the guard +// below proves that config actually discriminates the bug. +const STRICT = { strict: true }; + +describe('quality gate schema (strict AJV compatibility)', function () { + it('guard: the strict config throws on a union `type` array (so this test can catch the regression)', function () { + const unionSchema = { type: 'object', properties: { t: { type: ['string', 'number'] } } }; + assert.throws(() => new Ajv(STRICT).compile(unionSchema), /allowUnionTypes|strictTypes/); + }); + + it('compiles under strict AJV without strictTypes errors', function () { + const schema = buildQualityGateSchema(); + assert.doesNotThrow(() => new Ajv(STRICT).compile(schema)); + }); + + it('accepts a gate with a string completedAt/timestamp', function () { + const validate = new Ajv(STRICT).compile(buildQualityGateSchema()); + const ok = validate([ + { + id: 'ci', + status: 'PASS', + evidence: { command: 'npm test', exitCode: 0, output: 'ok' }, + completedAt: '2026-07-09T17:00:00Z', + timestamp: '2026-07-09T17:00:00Z', + }, + ]); + assert.strictEqual(ok, true, JSON.stringify(validate.errors)); + }); + + it('accepts a gate with a numeric completedAt/timestamp', function () { + const validate = new Ajv(STRICT).compile(buildQualityGateSchema()); + const ok = validate([ + { + id: 'ci', + status: 'PASS', + evidence: { command: 'npm test', exitCode: 0, output: 'ok' }, + completedAt: 1752080400000, + timestamp: 1752080400000, + }, + ]); + assert.strictEqual(ok, true, JSON.stringify(validate.errors)); + }); + + it('still rejects a wrong-typed completedAt (anyOf keeps the type constraint)', function () { + const validate = new Ajv(STRICT).compile(buildQualityGateSchema()); + const ok = validate([ + { + id: 'ci', + status: 'PASS', + evidence: { command: 'npm test', exitCode: 0, output: 'ok' }, + completedAt: true, + }, + ]); + assert.strictEqual(ok, false); + }); +}); diff --git a/tests/settings-providers.test.js b/tests/settings-providers.test.js index 1e01d5e7..f132a050 100644 --- a/tests/settings-providers.test.js +++ b/tests/settings-providers.test.js @@ -76,6 +76,38 @@ describe('Provider settings', function () { }, /reasoningEffort overrides are only supported/); }); + it('validates gateway settings and accepts arbitrary model ids', function () { + assert.doesNotThrow(() => { + validateProviderSettings('gateway', { + minLevel: 'level1', + maxLevel: 'level3', + defaultLevel: 'level2', + baseUrl: 'http://127.0.0.1:11434', + apiKey: 'gateway-key', + model: 'openrouter/meta-llama/test', + toolPolicy: { + roots: ['.'], + commands: ['node'], + }, + levelOverrides: { + level2: { model: 'openrouter/meta-llama/test' }, + }, + }); + }); + + assert.throws(() => { + validateProviderSettings('gateway', { + baseUrl: 'http://127.0.0.1:11434', + apiKey: 'gateway-key', + model: 'test-model', + toolPolicy: { + roots: '.', + commands: ['node'], + }, + }); + }, /toolPolicy\.roots must be an array of strings/); + }); + it('applies legacy maxModel to claude levels', function () { process.env.ZEROSHOT_SETTINGS_FILE = settingsFile; fs.writeFileSync(settingsFile, JSON.stringify({ maxModel: 'haiku' }, null, 2), 'utf8'); diff --git a/tests/unit/agent-lifecycle-stop.test.js b/tests/unit/agent-lifecycle-stop.test.js new file mode 100644 index 00000000..f55a5fe2 --- /dev/null +++ b/tests/unit/agent-lifecycle-stop.test.js @@ -0,0 +1,17 @@ +const assert = require('node:assert'); + +const { stop } = require('../../src/agent/agent-lifecycle'); + +describe('Agent lifecycle stop', function () { + it('clears liveness monitoring even when the agent is already not running', async function () { + const interval = setInterval(() => {}, 60_000); + const agent = { + running: false, + livenessCheckInterval: interval, + }; + + await stop(agent); + + assert.strictEqual(agent.livenessCheckInterval, null); + }); +}); diff --git a/tests/unit/cli-invalid-command.test.js b/tests/unit/cli-invalid-command.test.js index 7c9e4908..4f785214 100644 --- a/tests/unit/cli-invalid-command.test.js +++ b/tests/unit/cli-invalid-command.test.js @@ -42,6 +42,8 @@ describe('CLI Invalid Command Handling', function () { 'claude', 'gemini', 'opencode', + 'pi', + 'copilot', 'attach', 'agents', 'config', @@ -127,6 +129,14 @@ describe('CLI Invalid Command Handling', function () { it('should not prepend run for "opencode" command', function () { assert.strictEqual(shouldPrependRun(['opencode']), false); }); + + it('should not prepend run for "pi" command', function () { + assert.strictEqual(shouldPrependRun(['pi']), false); + }); + + it('should not prepend run for "copilot" command', function () { + assert.strictEqual(shouldPrependRun(['copilot']), false); + }); }); describe('Flags should not trigger run prepending', function () { diff --git a/tests/unit/cli-provider-override.test.js b/tests/unit/cli-provider-override.test.js index 59bc0752..55ac0201 100644 --- a/tests/unit/cli-provider-override.test.js +++ b/tests/unit/cli-provider-override.test.js @@ -6,15 +6,7 @@ */ const assert = require('assert'); - -function normalizeProviderName(name) { - if (!name || typeof name !== 'string') return name; - const normalized = name.toLowerCase(); - if (normalized === 'anthropic') return 'claude'; - if (normalized === 'openai') return 'codex'; - if (normalized === 'google') return 'gemini'; - return normalized; -} +const { normalizeProviderName } = require('../../lib/provider-names'); // Mirrors resolveProviderOverride in cli/index.js function resolveProviderOverride(options) { diff --git a/tests/unit/docker-config.test.js b/tests/unit/docker-config.test.js index 22646a3e..6a39227a 100644 --- a/tests/unit/docker-config.test.js +++ b/tests/unit/docker-config.test.js @@ -9,6 +9,7 @@ */ const assert = require('assert'); +const { listProviderMetadata } = require('../../lib/provider-names'); const { MOUNT_PRESETS, ENV_PRESETS, @@ -34,12 +35,28 @@ describe('Docker Configuration', function () { function registerMountPresetTests() { describe('MOUNT_PRESETS', function () { it('should have all expected presets', function () { - const expected = ['gh', 'git', 'ssh', 'aws', 'azure', 'kube', 'terraform', 'gcloud']; + const expected = [ + 'gh', + 'git', + 'ssh', + 'aws', + 'azure', + 'kube', + 'terraform', + 'gcloud', + ...listProviderMetadata().map((metadata) => metadata.id), + ]; for (const preset of expected) { assert.ok(MOUNT_PRESETS[preset], `Missing preset: ${preset}`); } }); + it('should build provider mount presets from provider registry metadata', function () { + for (const metadata of listProviderMetadata()) { + assert.deepStrictEqual(MOUNT_PRESETS[metadata.id], metadata.docker.mount); + } + }); + it('should use $HOME placeholder in container paths', function () { for (const [name, preset] of Object.entries(MOUNT_PRESETS)) { assert.ok( @@ -77,6 +94,9 @@ function registerEnvPresetTests() { assert.ok(ENV_PRESETS.gcloud); assert.ok(ENV_PRESETS.kube); assert.ok(ENV_PRESETS.terraform); + for (const metadata of listProviderMetadata()) { + assert.ok(ENV_PRESETS[metadata.id]); + } }); it('should include AWS_PAGER= forced value for aws preset', function () { @@ -92,6 +112,12 @@ function registerEnvPresetTests() { 'Terraform preset should include TF_VAR_* pattern' ); }); + + it('should build provider env presets from provider registry metadata', function () { + for (const metadata of listProviderMetadata()) { + assert.deepStrictEqual(ENV_PRESETS[metadata.id], metadata.docker.envPassthrough); + } + }); }); } diff --git a/tests/unit/docker-image-cacerts.test.js b/tests/unit/docker-image-cacerts.test.js new file mode 100644 index 00000000..b4900fc2 --- /dev/null +++ b/tests/unit/docker-image-cacerts.test.js @@ -0,0 +1,66 @@ +/** + * Test: Docker base image keeps CA certificates + * + * Regression guard for a hard-to-diagnose failure mode. + * + * The GitHub Copilot CLI (and other Rust-based provider CLIs) use the *system* CA certificate + * store for HTTPS, not a bundled one. If `ca-certificates` is missing from the cluster image, + * copilot authentication fails with a cryptic error that looks like a network problem: + * + * Failed to fetch PAT user login: network fetch failed: request failed: builder error + * (~/.copilot/logs) No CA certificates were loaded from the system + * + * ...even when COPILOT_GITHUB_TOKEN is valid and general egress works (Node's own fetch keeps + * working because Node bundles its own CAs, which makes this doubly confusing to debug). + * + * The base image is derived from `node:20-slim`, which does NOT ship system CA certificates, + * so the Dockerfile must install them explicitly. This test fails loudly if that line is ever + * removed, so the breakage is caught at review time instead of inside a container. + */ + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +const DOCKERFILE_PATH = path.join( + __dirname, + '..', + '..', + 'docker', + 'zeroshot-cluster', + 'Dockerfile' +); + +describe('Docker cluster image: CA certificates', function () { + let dockerfile; + + before(function () { + dockerfile = fs.readFileSync(DOCKERFILE_PATH, 'utf8'); + }); + + it('installs ca-certificates (required for copilot/Rust CLI HTTPS auth)', function () { + // Strip comment lines so a mention in a comment can never satisfy the guard. + const instructions = dockerfile + .split('\n') + .filter((line) => !line.trim().startsWith('#')) + .join('\n'); + + assert.match( + instructions, + /\bca-certificates\b/, + 'docker/zeroshot-cluster/Dockerfile must install the `ca-certificates` package. ' + + 'Without it, the Copilot CLI (Rust) cannot load system CA certs and authentication ' + + 'fails with a misleading "network fetch failed: builder error". Do not remove it.' + ); + }); + + it('derives from node:20-slim, which is why ca-certificates must be explicit', function () { + // If the base image ever changes to one that bundles system CA certs, this test documents + // the assumption behind the guard above (slim images ship no system CA store). + assert.match( + dockerfile, + /^FROM\s+node:\d+-slim/m, + 'Base image is expected to be a slim node image (no system CA certs by default).' + ); + }); +}); diff --git a/tests/unit/docker-provider-preset.test.js b/tests/unit/docker-provider-preset.test.js new file mode 100644 index 00000000..22839943 --- /dev/null +++ b/tests/unit/docker-provider-preset.test.js @@ -0,0 +1,129 @@ +/** + * The provider actually running under `--docker` must have its credential preset (mount + env + * passthrough) auto-activated, so `--docker --provider

` authenticates without the user manually + * listing

in dockerMounts. Regression guard for the Copilot case: its OAuth token is not in a + * mountable dir (it lives in the OS keychain), so forwarding COPILOT_GITHUB_TOKEN is the only path. + */ +const assert = require('assert'); +const IsolationManager = require('../../src/isolation-manager'); + +function envSpecs(args) { + // args are a flat docker argv; `-e` is followed by `NAME=value`. + const out = []; + for (let i = 0; i < args.length - 1; i++) { + if (args[i] === '-e') out.push(args[i + 1]); + } + return out; +} + +describe('docker active-provider credential preset', function () { + const settings = { dockerMounts: ['gh', 'git', 'ssh'], dockerEnvPassthrough: [] }; + let manager; + + beforeEach(function () { + manager = new IsolationManager(); + }); + + describe('_withActiveProviderPreset', function () { + it('appends the running provider so its preset activates', function () { + assert.deepEqual(manager._withActiveProviderPreset(['gh', 'git'], 'copilot'), [ + 'gh', + 'git', + 'copilot', + ]); + }); + + it('does not duplicate a provider already listed', function () { + assert.deepEqual(manager._withActiveProviderPreset(['copilot'], 'copilot'), ['copilot']); + }); + + it('skips claude (mounted separately) and unknown providers', function () { + assert.deepEqual(manager._withActiveProviderPreset(['gh'], 'claude'), ['gh']); + assert.deepEqual(manager._withActiveProviderPreset(['gh'], 'not-a-provider'), ['gh']); + }); + }); + + describe('_applyCredentialMounts forwards the provider token', function () { + const KEY = 'COPILOT_GITHUB_TOKEN'; + let saved; + + beforeEach(function () { + saved = process.env[KEY]; + process.env[KEY] = 'tok-sentinel'; + }); + + afterEach(function () { + if (saved === undefined) delete process.env[KEY]; + else process.env[KEY] = saved; + }); + + it('forwards COPILOT_GITHUB_TOKEN when copilot is the active provider', function () { + const args = []; + manager._applyCredentialMounts(args, {}, settings, '/root', 'copilot'); + assert.ok( + envSpecs(args).includes('COPILOT_GITHUB_TOKEN=tok-sentinel'), + `expected COPILOT_GITHUB_TOKEN to be forwarded, got: ${JSON.stringify(envSpecs(args))}` + ); + }); + + it('does not forward it for an unrelated active provider', function () { + const args = []; + manager._applyCredentialMounts(args, {}, settings, '/root', 'codex'); + assert.ok(!envSpecs(args).some((spec) => spec.startsWith('COPILOT_GITHUB_TOKEN='))); + }); + + it('is disabled by noMounts', function () { + const args = []; + manager._applyCredentialMounts(args, { noMounts: true }, settings, '/root', 'copilot'); + assert.equal(args.length, 0); + }); + }); + + describe('_warnMissingProviderCredentials for a keychain-token provider (copilot)', function () { + const KEYS = ['COPILOT_GITHUB_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN']; + let savedEnv; + let savedWarn; + let warnings; + + beforeEach(function () { + savedEnv = {}; + for (const k of KEYS) { + savedEnv[k] = process.env[k]; + delete process.env[k]; + } + warnings = []; + savedWarn = console.warn; + console.warn = (msg) => warnings.push(msg); + }); + + afterEach(function () { + console.warn = savedWarn; + for (const k of KEYS) { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + } + }); + + it('warns to export the token even though ~/.copilot is mounted (mount holds no secret)', function () { + manager._warnMissingProviderCredentials( + 'copilot', + [require('os').homedir() + '/.copilot'], + {}, + '/root' + ); + assert.equal(warnings.length, 1); + assert.match(warnings[0], /COPILOT_GITHUB_TOKEN/); + }); + + it('stays silent once the token is exported', function () { + process.env.COPILOT_GITHUB_TOKEN = 'tok-sentinel'; + manager._warnMissingProviderCredentials( + 'copilot', + [require('os').homedir() + '/.copilot'], + {}, + '/root' + ); + assert.equal(warnings.length, 0); + }); + }); +}); diff --git a/tests/unit/provider-docker-image.test.js b/tests/unit/provider-docker-image.test.js new file mode 100644 index 00000000..09a1fe7d --- /dev/null +++ b/tests/unit/provider-docker-image.test.js @@ -0,0 +1,99 @@ +/** + * Test: per-provider Docker image selection + * + * Providers whose CLI is not baked into the base image (copilot, codex, gemini) run on a + * per-provider image variant `-` whose CLI install is a Docker-cached build + * layer. Providers baked into the base image (claude) — or with no single-command installer + * (opencode) — run on the base image directly. + * + * The install command is sourced from the provider registry (docker.install), never hardcoded + * here, so this stays general-purpose across current and future providers. + */ + +const assert = require('assert'); +const IsolationManager = require('../../src/isolation-manager'); +const { getProviderMetadata } = require('../../lib/provider-names'); + +describe('IsolationManager: per-provider image selection', function () { + describe('imageForProvider', function () { + it('returns the base image for a provider baked into it (claude)', function () { + assert.strictEqual(IsolationManager.imageForProvider('claude'), 'zeroshot-cluster-base'); + }); + + it('returns a per-provider variant for a provider with docker.install (copilot)', function () { + assert.strictEqual( + IsolationManager.imageForProvider('copilot'), + 'zeroshot-cluster-base-copilot' + ); + }); + + it('returns a per-provider variant for codex', function () { + assert.strictEqual(IsolationManager.imageForProvider('codex'), 'zeroshot-cluster-base-codex'); + }); + + it('honors a custom base image when building the variant name', function () { + assert.strictEqual( + IsolationManager.imageForProvider('copilot', 'my-base'), + 'my-base-copilot' + ); + }); + + it('normalizes provider aliases to the canonical image (no duplicate per alias)', function () { + // `openai` is an alias of `codex`; both must resolve to the same variant image so we don't + // build a redundant `-openai` image alongside `-codex`. + assert.strictEqual( + IsolationManager.imageForProvider('openai'), + IsolationManager.imageForProvider('codex') + ); + assert.strictEqual( + IsolationManager.imageForProvider('openai'), + 'zeroshot-cluster-base-codex' + ); + }); + + it('falls back to the base image for a provider with no docker.install (opencode)', function () { + assert.strictEqual(IsolationManager.imageForProvider('opencode'), 'zeroshot-cluster-base'); + }); + }); + + describe('providerBuildArgs', function () { + it('returns no build args for a baked-in provider (claude)', function () { + assert.deepStrictEqual(IsolationManager.providerBuildArgs('claude'), []); + }); + + it('emits PROVIDER_INSTALL for copilot from the registry command', function () { + assert.deepStrictEqual(IsolationManager.providerBuildArgs('copilot'), [ + 'PROVIDER_INSTALL=npm install -g @github/copilot', + ]); + }); + + it('emits PROVIDER_INSTALL for codex from the registry command', function () { + assert.deepStrictEqual(IsolationManager.providerBuildArgs('codex'), [ + 'PROVIDER_INSTALL=npm install -g @openai/codex', + ]); + }); + + it('matches the value the registry advertises (no hardcoded drift)', function () { + const registryInstall = getProviderMetadata('copilot').docker.install; + assert.deepStrictEqual(IsolationManager.providerBuildArgs('copilot'), [ + `PROVIDER_INSTALL=${registryInstall}`, + ]); + }); + }); + + describe('registry docker.install', function () { + it('is set for npm-installable providers and absent for baked-in claude', function () { + assert.ok( + getProviderMetadata('copilot').docker.install, + 'copilot should have docker.install' + ); + assert.ok(getProviderMetadata('codex').docker.install, 'codex should have docker.install'); + assert.ok(getProviderMetadata('gemini').docker.install, 'gemini should have docker.install'); + assert.strictEqual( + getProviderMetadata('claude').docker.install, + undefined, + 'claude is baked into the base image and must not declare docker.install' + ); + }); + }); +}); diff --git a/tests/unit/repo-settings.test.js b/tests/unit/repo-settings.test.js new file mode 100644 index 00000000..713ffc99 --- /dev/null +++ b/tests/unit/repo-settings.test.js @@ -0,0 +1,67 @@ +/** + * Test: repo-local settings read/write + * + * writeRepoSettings() + readRepoSettings() round-trip against a real + * `.zeroshot/settings.json` in a temp git repo. + */ + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const crypto = require('crypto'); +const { execSync } = require('child_process'); + +const { readRepoSettings, writeRepoSettings } = require('../../lib/repo-settings'); + +describe('repo-settings', function () { + this.timeout(10000); + + let repoRoot; + + beforeEach(function () { + repoRoot = path.join( + os.tmpdir(), + 'zeroshot-repo-settings-test-' + crypto.randomBytes(8).toString('hex') + ); + fs.mkdirSync(repoRoot, { recursive: true }); + execSync('git init', { cwd: repoRoot, stdio: 'ignore' }); + }); + + afterEach(function () { + fs.rmSync(repoRoot, { recursive: true, force: true }); + }); + + it('writeRepoSettings creates .zeroshot/settings.json', function () { + const settingsPath = writeRepoSettings(repoRoot, { prBase: 'main' }); + assert.strictEqual(settingsPath, path.join(repoRoot, '.zeroshot', 'settings.json')); + assert.ok(fs.existsSync(settingsPath)); + const onDisk = JSON.parse(fs.readFileSync(settingsPath, 'utf8')); + assert.deepStrictEqual(onDisk, { prBase: 'main' }); + }); + + it('readRepoSettings round-trips what writeRepoSettings wrote', function () { + writeRepoSettings(repoRoot, { prBase: 'dev', dockerMounts: ['gh', 'git'] }); + const { repoRoot: detectedRoot, settings, settingsPath } = readRepoSettings(repoRoot); + // git resolves symlinks (e.g. macOS /tmp -> /private/tmp), so compare via realpath. + assert.strictEqual( + settingsPath, + path.join(fs.realpathSync(repoRoot), '.zeroshot', 'settings.json') + ); + assert.ok(detectedRoot); + assert.deepStrictEqual(settings, { prBase: 'dev', dockerMounts: ['gh', 'git'] }); + }); + + it('writeRepoSettings overwrites a previous file', function () { + writeRepoSettings(repoRoot, { prBase: 'main' }); + writeRepoSettings(repoRoot, { prBase: 'dev' }); + const { settings } = readRepoSettings(repoRoot); + assert.deepStrictEqual(settings, { prBase: 'dev' }); + }); + + it('readRepoSettings returns null settings when no file exists yet', function () { + const { settings, repoRoot: detectedRoot } = readRepoSettings(repoRoot); + assert.strictEqual(settings, null); + assert.ok(detectedRoot); + }); +}); diff --git a/tests/unit/run-plan.test.js b/tests/unit/run-plan.test.js new file mode 100644 index 00000000..d77bcb69 --- /dev/null +++ b/tests/unit/run-plan.test.js @@ -0,0 +1,120 @@ +/** + * Test: resolveRunPlan (the single canonical run-mode resolver) + * + * Verifies isolation/delivery/autoMerge for every flag combination, and — the + * point of #582 — that the run-mode LABEL (resolveRunMode) is a pure view of the + * same plan, so the label and the behavior cannot drift. + */ + +const assert = require('assert'); +const { resolveRunPlan } = require('../../lib/run-plan'); +const { resolveRunMode } = require('../../lib/run-mode'); + +describe('resolveRunPlan', function () { + it('resolves {} to no isolation, no delivery', function () { + assert.deepStrictEqual(resolveRunPlan({}), { + isolation: 'none', + delivery: 'none', + autoMerge: false, + }); + }); + + it('resolves {worktree: true}', function () { + assert.deepStrictEqual(resolveRunPlan({ worktree: true }), { + isolation: 'worktree', + delivery: 'none', + autoMerge: false, + }); + }); + + it('resolves {docker: true}', function () { + assert.deepStrictEqual(resolveRunPlan({ docker: true }), { + isolation: 'docker', + delivery: 'none', + autoMerge: false, + }); + }); + + it('resolves {pr: true} to pr delivery, no auto-merge', function () { + assert.deepStrictEqual(resolveRunPlan({ pr: true }), { + isolation: 'worktree', + delivery: 'pr', + autoMerge: false, + }); + }); + + it('resolves {ship: true} to ship delivery with auto-merge', function () { + assert.deepStrictEqual(resolveRunPlan({ ship: true }), { + isolation: 'worktree', + delivery: 'ship', + autoMerge: true, + }); + }); + + it('resolves {pr: true, docker: true}', function () { + assert.deepStrictEqual(resolveRunPlan({ pr: true, docker: true }), { + isolation: 'docker', + delivery: 'pr', + autoMerge: false, + }); + }); + + it('resolves {ship: true, docker: true}', function () { + assert.deepStrictEqual(resolveRunPlan({ ship: true, docker: true }), { + isolation: 'docker', + delivery: 'ship', + autoMerge: true, + }); + }); + + it('treats an explicit autoMerge as ship-equivalent (future --auto-merge flag)', function () { + // Regression for the normalizeRunOptions overwrite bug: an explicit + // autoMerge intent must survive, not be reset to false. + assert.deepStrictEqual(resolveRunPlan({ pr: true, autoMerge: true }), { + isolation: 'worktree', + delivery: 'ship', + autoMerge: true, + }); + }); + + it('returns a frozen object', function () { + assert.strictEqual(Object.isFrozen(resolveRunPlan({})), true); + }); + + it('defaults options to {} when called with no arguments', function () { + assert.deepStrictEqual(resolveRunPlan(), { + isolation: 'none', + delivery: 'none', + autoMerge: false, + }); + }); +}); + +describe('resolveRunMode is a view of resolveRunPlan (cannot drift)', function () { + // Expected label derived ONLY from the plan. If resolveRunMode ever grows an + // independent cascade again, this binding breaks. + function labelFromPlan(options) { + const { isolation, delivery } = resolveRunPlan(options); + const suffix = isolation === 'docker' ? '+docker' : ''; + if (delivery === 'ship') return `ship${suffix}`; + if (delivery === 'pr') return `pr${suffix}`; + if (isolation === 'docker') return 'docker'; + if (isolation === 'worktree') return 'worktree'; + return null; + } + + it('agrees with the plan across all 16 flag combinations', function () { + const flags = ['worktree', 'docker', 'pr', 'ship']; + for (let mask = 0; mask < 1 << flags.length; mask++) { + const options = {}; + flags.forEach((f, i) => { + if (mask & (1 << i)) options[f] = true; + }); + assert.strictEqual( + resolveRunMode(options), + labelFromPlan(options), + `run-mode label drifted from plan for ${JSON.stringify(options)}` + ); + } + }); +}); diff --git a/tests/unit/setup-apply.test.js b/tests/unit/setup-apply.test.js new file mode 100644 index 00000000..2b3e3045 --- /dev/null +++ b/tests/unit/setup-apply.test.js @@ -0,0 +1,302 @@ +/** + * Test: `zeroshot setup apply` (lib/setup-apply.js) + * + * Verifies the issue #606 contract: + * - Fail-closed validation (unknown decisionId / out-of-domain value -> zero writes) + * - Idempotent re-apply (second run is a no-op) + * - Writes confined to global settings, .zeroshot/settings.json, and the undo journal + * - Dead settings keys (no consumer) are refused, not written + * - No secret-shaped path is ever written + * - defaultDelivery=ship requires explicit opt-in and is then live in startClusterFromText + * - github issue-source hint prints a login command instead of storing anything + */ + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const crypto = require('crypto'); +const { execSync } = require('child_process'); + +const settingsPath = require.resolve('../../lib/settings'); +const setupApplyPath = require.resolve('../../lib/setup-apply'); +const setupUndoPath = require.resolve('../../lib/setup-undo'); +const setupJournalPath = require.resolve('../../lib/setup-journal'); +const repoSettingsPath = require.resolve('../../lib/repo-settings'); + +describe('setup-apply', function () { + this.timeout(15000); + + let TEST_DIR; + let TEST_SETTINGS_FILE; + let TEST_JOURNAL_FILE; + let repoRoot; + let settingsModule; + let applyModule; + let startClusterModule; + + function decisionsFile(obj) { + const p = path.join(TEST_DIR, 'decisions.json'); + fs.writeFileSync(p, JSON.stringify(obj), 'utf8'); + return p; + } + + function readSettings() { + return JSON.parse(fs.readFileSync(TEST_SETTINGS_FILE, 'utf8')); + } + + function readJournal() { + return JSON.parse(fs.readFileSync(TEST_JOURNAL_FILE, 'utf8')); + } + + beforeEach(function () { + TEST_DIR = path.join( + os.tmpdir(), + 'zeroshot-setup-apply-test-' + crypto.randomBytes(8).toString('hex') + ); + fs.mkdirSync(TEST_DIR, { recursive: true }); + TEST_SETTINGS_FILE = path.join(TEST_DIR, 'settings.json'); + TEST_JOURNAL_FILE = path.join(TEST_DIR, 'setup-undo-journal.json'); + + repoRoot = path.join(TEST_DIR, 'repo'); + fs.mkdirSync(repoRoot, { recursive: true }); + execSync('git init', { cwd: repoRoot, stdio: 'ignore' }); + + process.env.ZEROSHOT_SETTINGS_FILE = TEST_SETTINGS_FILE; + delete require.cache[settingsPath]; + delete require.cache[setupApplyPath]; + delete require.cache[setupUndoPath]; + delete require.cache[setupJournalPath]; + delete require.cache[repoSettingsPath]; + + settingsModule = require('../../lib/settings'); + applyModule = require('../../lib/setup-apply'); + startClusterModule = require('../../lib/start-cluster'); + }); + + afterEach(function () { + delete process.env.ZEROSHOT_SETTINGS_FILE; + fs.rmSync(TEST_DIR, { recursive: true, force: true }); + }); + + it('applies a valid decisions file and writes settings + journal', function () { + const results = applyModule.applyDecisions({ + decisionsPath: decisionsFile({ defaultProvider: 'codex' }), + cwd: repoRoot, + }); + + assert.deepStrictEqual(results, [ + { decisionId: 'defaultProvider', applied: true, from: 'claude', to: 'codex' }, + ]); + assert.strictEqual(readSettings().defaultProvider, 'codex'); + const journal = readJournal(); + assert.strictEqual(journal.entries.length, 1); + assert.strictEqual(journal.entries[0].priorValue, 'claude'); + assert.strictEqual(journal.entries[0].appliedValue, 'codex'); + }); + + it('is idempotent: applying identical decisions twice writes only on the first run', function () { + const decisions = decisionsFile({ defaultProvider: 'codex', defaultDelivery: 'pr' }); + + applyModule.applyDecisions({ decisionsPath: decisions, cwd: repoRoot }); + const settingsAfterFirst = fs.readFileSync(TEST_SETTINGS_FILE, 'utf8'); + const journalAfterFirst = fs.readFileSync(TEST_JOURNAL_FILE, 'utf8'); + + const secondResults = applyModule.applyDecisions({ decisionsPath: decisions, cwd: repoRoot }); + + assert.ok(secondResults.every((r) => r.applied === false && r.skippedReason === 'unchanged')); + assert.strictEqual(fs.readFileSync(TEST_SETTINGS_FILE, 'utf8'), settingsAfterFirst); + assert.strictEqual(fs.readFileSync(TEST_JOURNAL_FILE, 'utf8'), journalAfterFirst); + assert.strictEqual(JSON.parse(journalAfterFirst).entries.length, 2); + }); + + it('rejects an unknown decision ID without writing anything', function () { + const decisions = decisionsFile({ bogusDecisionId: 'value' }); + assert.throws( + () => applyModule.applyDecisions({ decisionsPath: decisions, cwd: repoRoot }), + /Unknown decision ID: bogusDecisionId/ + ); + assert.ok(!fs.existsSync(TEST_SETTINGS_FILE)); + assert.ok(!fs.existsSync(TEST_JOURNAL_FILE)); + }); + + it('rejects an out-of-domain value without writing anything', function () { + const decisions = decisionsFile({ defaultProvider: 'not-a-real-provider' }); + assert.throws( + () => applyModule.applyDecisions({ decisionsPath: decisions, cwd: repoRoot }), + /Invalid value for decision "defaultProvider"/ + ); + assert.ok(!fs.existsSync(TEST_SETTINGS_FILE)); + }); + + it('rejects a mixed request (one valid, one invalid decision) atomically', function () { + const decisions = decisionsFile({ + defaultProvider: 'codex', + defaultDelivery: 'not-a-real-delivery', + }); + assert.throws(() => applyModule.applyDecisions({ decisionsPath: decisions, cwd: repoRoot })); + assert.ok(!fs.existsSync(TEST_SETTINGS_FILE)); + }); + + it('refuses to write dead settings keys (no consumer), reporting skippedReason "no-consumer"', function () { + const decisions = decisionsFile({ allowLocalNoIsolation: true, prBase: 'main' }); + const results = applyModule.applyDecisions({ decisionsPath: decisions, cwd: repoRoot }); + + assert.deepStrictEqual( + results.map((r) => ({ + decisionId: r.decisionId, + applied: r.applied, + skippedReason: r.skippedReason, + })), + [ + { decisionId: 'allowLocalNoIsolation', applied: false, skippedReason: 'no-consumer' }, + { decisionId: 'prBase', applied: false, skippedReason: 'no-consumer' }, + ] + ); + assert.ok(!fs.existsSync(TEST_SETTINGS_FILE)); + assert.ok(!fs.existsSync(path.join(repoRoot, '.zeroshot', 'settings.json'))); + }); + + it('the secret-shaped path guard throws for a fabricated secret path', function () { + assert.throws( + () => applyModule.assertSecretSafePath('providerSettings.claude.apiKey'), + /Refusing to write secret-shaped settings path/ + ); + assert.doesNotThrow(() => applyModule.assertSecretSafePath('defaultProvider')); + }); + + it('skips storing defaultDelivery=ship without --allow-risky-defaults', function () { + const decisions = decisionsFile({ defaultDelivery: 'ship' }); + const results = applyModule.applyDecisions({ + decisionsPath: decisions, + cwd: repoRoot, + allowRiskyDefaults: false, + }); + + assert.deepStrictEqual(results, [ + { + decisionId: 'defaultDelivery', + applied: false, + from: 'none', + to: 'ship', + skippedReason: 'requires-explicit-opt-in', + }, + ]); + assert.ok(!fs.existsSync(TEST_SETTINGS_FILE)); + }); + + it('stores defaultDelivery=ship with --allow-risky-defaults and it is live in startClusterFromText', function () { + const decisions = decisionsFile({ defaultDelivery: 'ship' }); + const results = applyModule.applyDecisions({ + decisionsPath: decisions, + cwd: repoRoot, + allowRiskyDefaults: true, + }); + + assert.strictEqual(results[0].applied, true); + const settings = settingsModule.loadSettings(); + assert.strictEqual(settings.defaultDelivery, 'ship'); + + const startOptions = startClusterModule.startClusterFromText({ + orchestrator: { + start(_config, _input, options) { + return options; + }, + }, + config: { agents: [] }, + clusterId: 'c1', + text: 'hello', + options: {}, + settings, + }); + assert.strictEqual(startOptions.autoPr, true); + assert.strictEqual(startOptions.autoMerge, true); + assert.strictEqual(startOptions.worktree, true); + }); + + it('confines writes to global settings, repo .zeroshot/settings.json, and the undo journal', function () { + const decisions = decisionsFile({ defaultProvider: 'codex', dockerMounts: ['gh'] }); + const before = new Set(fs.readdirSync(TEST_DIR)); + applyModule.applyDecisions({ decisionsPath: decisions, cwd: repoRoot }); + + const after = fs.readdirSync(TEST_DIR); + const added = after.filter((f) => !before.has(f)); + assert.deepStrictEqual(new Set(added), new Set(['settings.json', 'setup-undo-journal.json'])); + + // No decision in this request is repo-scoped and consumed today, so the repo dir + // must be untouched (still just the bare git repo, no .zeroshot directory). + assert.ok(!fs.existsSync(path.join(repoRoot, '.zeroshot'))); + }); + + it('prints a login command and stores nothing when applying defaultIssueSource=github unauthenticated', function () { + // Default settings already default to defaultIssueSource='github', so start + // from a different value to force an actual write (not a no-op 'unchanged'). + settingsModule.saveSettings({ ...settingsModule.loadSettings(), defaultIssueSource: 'gitlab' }); + + const logs = []; + const originalLog = console.log; + console.log = (...args) => logs.push(args.join(' ')); + + try { + const decisions = decisionsFile({ defaultIssueSource: 'github' }); + applyModule.applyDecisions({ + decisionsPath: decisions, + cwd: repoRoot, + deps: { checkGhAuth: () => ({ authenticated: false }) }, + }); + } finally { + console.log = originalLog; + } + + assert.ok(logs.some((line) => line.includes('gh auth login'))); + const settings = readSettings(); + assert.strictEqual(settings.defaultIssueSource, 'github'); + for (const key of Object.keys(settings)) { + assert.ok( + !/token|secret|password|api[_-]?key|credential/i.test(key) || settings[key] === null + ); + } + }); + + it('does not print a login hint when already gh-authenticated', function () { + const logs = []; + const originalLog = console.log; + console.log = (...args) => logs.push(args.join(' ')); + + try { + const decisions = decisionsFile({ defaultIssueSource: 'github' }); + applyModule.applyDecisions({ + decisionsPath: decisions, + cwd: repoRoot, + deps: { checkGhAuth: () => ({ authenticated: true }) }, + }); + } finally { + console.log = originalLog; + } + + assert.ok(!logs.some((line) => line.includes('gh auth login'))); + }); + + it('converts providerLevel. min/default/max into providerSettings levels', function () { + // codex's stock defaults are already haiku/sonnet/opus (level1/level2/level3), + // so submit a value that actually differs to exercise a real write. + const decisions = decisionsFile({ + 'providerLevel.codex': { min: 'sonnet', default: 'sonnet', max: 'opus' }, + }); + const results = applyModule.applyDecisions({ decisionsPath: decisions, cwd: repoRoot }); + + assert.strictEqual(results[0].applied, true); + const settings = readSettings(); + assert.strictEqual(settings.providerSettings.codex.minLevel, 'level2'); + assert.strictEqual(settings.providerSettings.codex.defaultLevel, 'level2'); + assert.strictEqual(settings.providerSettings.codex.maxLevel, 'level3'); + }); + + it('rejects an out-of-domain providerLevel value without writing anything', function () { + const decisions = decisionsFile({ + 'providerLevel.codex': { min: 'not-a-model', default: 'sonnet', max: 'opus' }, + }); + assert.throws(() => applyModule.applyDecisions({ decisionsPath: decisions, cwd: repoRoot })); + assert.ok(!fs.existsSync(TEST_SETTINGS_FILE)); + }); +}); diff --git a/tests/unit/setup-plan.test.js b/tests/unit/setup-plan.test.js new file mode 100644 index 00000000..5810f80a --- /dev/null +++ b/tests/unit/setup-plan.test.js @@ -0,0 +1,309 @@ +/** + * Test: Setup plan contract (buildSetupPlan) + * + * Verifies the pinned, versioned setup contract from issue #605: + * - Pure over injected inputs, no writes, no prompts + * - Stable decisionId registry + * - defaultIsolation/defaultDelivery map to canonical settings keys only + * - No secrets ever surface in the plan + */ + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { buildSetupPlan, isConsumedPath } = require('../../lib/setup-plan'); + +const PROVIDER_NAMES = ['claude', 'codex', 'gemini', 'opencode']; + +const EXPECTED_DECISION_IDS = new Set([ + 'defaultProvider', + ...PROVIDER_NAMES.map((name) => `providerLevel.${name}`), + 'defaultIsolation', + 'allowLocalNoIsolation', + 'defaultDelivery', + 'defaultIssueSource', + 'prBase', + 'dockerMounts', + 'dockerEnvPassthrough', + 'updatePolicy', +]); + +function makeProviderDefaults() { + const defaults = {}; + for (const name of PROVIDER_NAMES) { + defaults[name] = { + minLevel: 'level1', + maxLevel: 'level3', + defaultLevel: 'level2', + levelOverrides: {}, + }; + } + return defaults; +} + +function makeDeps(overrides = {}) { + return { + commandExists: () => false, + getCommandPath: () => null, + checkDocker: () => ({ available: false }), + checkGhAuth: () => ({ authenticated: false }), + execSync: () => { + throw new Error('not a repo'); + }, + listProviders: () => PROVIDER_NAMES, + getProvider: (name) => ({ + cliCommand: name, + resolveModelSpec: (level) => ({ model: `${name}-${level}-model` }), + }), + getProviderDefaults: () => makeProviderDefaults(), + getNodeVersion: () => 'v99.0.0', + getPackageVersion: () => '9.9.9', + ...overrides, + }; +} + +function freshMachinePlan() { + return buildSetupPlan({ + cwd: '/fresh/machine/cwd', + settings: { __meta: { fileExists: false } }, + repoSettings: null, + env: {}, + deps: makeDeps(), + }); +} + +function fullyConfiguredExecSync(command) { + if (command.includes('is-inside-work-tree')) return 'true\n'; + if (command.includes('abbrev-ref origin/HEAD')) return 'origin/main\n'; + if (command.includes('abbrev-ref HEAD')) return 'main\n'; + if (command.includes('remote get-url origin')) return 'https://github.com/acme/widgets.git\n'; + throw new Error(`unexpected command: ${command}`); +} + +function fullyConfiguredPlan() { + const deps = makeDeps({ + commandExists: () => true, + getCommandPath: (cmd) => `/usr/local/bin/${cmd}`, + checkDocker: () => ({ available: true }), + checkGhAuth: () => ({ authenticated: true }), + execSync: (command) => fullyConfiguredExecSync(command), + }); + + return buildSetupPlan({ + cwd: '/configured/repo', + settings: { + __meta: { fileExists: true }, + defaultProvider: 'claude', + defaultDocker: false, + defaultDelivery: 'none', + allowLocalNoIsolation: false, + defaultIssueSource: 'github', + dockerMounts: ['gh', 'git', 'ssh'], + dockerEnvPassthrough: [], + updatePolicy: 'notify', + }, + repoSettings: { prBase: 'main' }, + env: {}, + deps, + }); +} + +describe('buildSetupPlan', function () { + describe('shape', function () { + it('always returns a typed schemaVersion/facts/decisions/recommended/risk/proposedWrites', function () { + const plan = freshMachinePlan(); + assert.strictEqual(plan.schemaVersion, 1); + assert.strictEqual(typeof plan.facts, 'object'); + assert.ok(Array.isArray(plan.decisions)); + assert.strictEqual(typeof plan.recommended, 'object'); + assert.strictEqual(typeof plan.risk, 'object'); + assert.ok(Array.isArray(plan.proposedWrites)); + }); + }); + + describe('fresh machine', function () { + it('includes every registry decisionId when no global settings exist', function () { + const plan = freshMachinePlan(); + const ids = plan.decisions.map((d) => d.decisionId).sort(); + assert.deepStrictEqual(ids, [...EXPECTED_DECISION_IDS].sort()); + }); + + it('never surfaces a secret anywhere in the plan', function () { + const plan = freshMachinePlan(); + const serialized = JSON.stringify(plan); + assert.ok(!/apikey|token|secret/i.test(serialized), serialized); + }); + + it('proposedWrites reference only the canonical settings keys', function () { + const plan = freshMachinePlan(); + const allowedPaths = new Set([ + 'defaultProvider', + 'defaultDocker', + 'defaultDelivery', + 'defaultIssueSource', + 'dockerMounts', + 'dockerEnvPassthrough', + 'updatePolicy', + ...PROVIDER_NAMES.map((name) => `providerSettings.${name}`), + ]); + assert.ok(plan.proposedWrites.length > 0); + for (const write of plan.proposedWrites) { + assert.ok(allowedPaths.has(write.path), `unexpected path: ${write.path}`); + assert.ok(['global', 'repo'].includes(write.scope)); + assert.ok('from' in write); + assert.ok('to' in write); + assert.ok(typeof write.decisionId === 'string'); + } + }); + + it('never proposes a write for a settings key no resolver consumes (no dead writes)', function () { + const plan = freshMachinePlan(); + // allowLocalNoIsolation/prBase are still valid *decisions* (a future wizard + // could surface them), but confirmed dead by grep as of #606 — no resolver + // reads them, so proposing a write for them would advertise a write apply + // will always skip. Assert every proposedWrites path is one apply will + // actually perform. + assert.ok(plan.decisions.some((d) => d.decisionId === 'allowLocalNoIsolation')); + assert.ok(plan.decisions.some((d) => d.decisionId === 'prBase')); + assert.ok(!plan.proposedWrites.some((w) => w.decisionId === 'allowLocalNoIsolation')); + assert.ok(!plan.proposedWrites.some((w) => w.decisionId === 'prBase')); + for (const write of plan.proposedWrites) { + assert.ok( + isConsumedPath(write.scope, write.path), + `unconsumed path: ${write.scope}:${write.path}` + ); + } + }); + + it('proposes a providerSettings. write for every providerLevel decision', function () { + const plan = freshMachinePlan(); + const providerLevelWrites = plan.proposedWrites.filter((w) => + w.decisionId.startsWith('providerLevel.') + ); + assert.deepStrictEqual( + providerLevelWrites.map((w) => w.decisionId).sort(), + PROVIDER_NAMES.map((name) => `providerLevel.${name}`).sort() + ); + for (const write of providerLevelWrites) { + const providerName = write.decisionId.slice('providerLevel.'.length); + assert.strictEqual(write.path, `providerSettings.${providerName}`); + assert.strictEqual(write.scope, 'global'); + assert.strictEqual(write.from, null); + assert.deepStrictEqual(write.to, { + min: `${providerName}-level1-model`, + default: `${providerName}-level2-model`, + max: `${providerName}-level3-model`, + }); + } + }); + }); + + describe('fully configured', function () { + it('excludes inferable decisionIds and proposes fewer/no writes', function () { + const plan = fullyConfiguredPlan(); + const ids = plan.decisions.map((d) => d.decisionId); + assert.ok(!ids.includes('defaultIssueSource')); + assert.ok(!ids.includes('prBase')); + assert.strictEqual(plan.decisions.length, 0); + assert.strictEqual(plan.proposedWrites.length, 0); + }); + }); + + describe('CI / non-TTY', function () { + it('recommends updatePolicy=off', function () { + const plan = buildSetupPlan({ + cwd: '/ci/cwd', + settings: { __meta: { fileExists: true } }, + repoSettings: null, + env: { CI: 'true', __isTTY: false }, + deps: makeDeps(), + }); + assert.strictEqual(plan.recommended.updatePolicy, 'off'); + }); + + it('recommends updatePolicy=notify for interactive TTY sessions', function () { + const plan = buildSetupPlan({ + cwd: '/tty/cwd', + settings: { __meta: { fileExists: true } }, + repoSettings: null, + env: { __isTTY: true }, + deps: makeDeps(), + }); + assert.strictEqual(plan.recommended.updatePolicy, 'notify'); + }); + }); + + describe('repo vs non-repo cwd', function () { + it('nulls git branch/remote/ghAuthed and never recommends worktree when not a repo', function () { + const plan = buildSetupPlan({ + cwd: '/not/a/repo', + settings: { __meta: { fileExists: true } }, + repoSettings: null, + env: {}, + deps: makeDeps({ + commandExists: (cmd) => cmd === 'gh', + checkDocker: () => ({ available: false }), + execSync: () => { + throw new Error('not a repo'); + }, + }), + }); + assert.strictEqual(plan.facts.git.isRepo, false); + assert.strictEqual(plan.facts.git.branch, null); + assert.strictEqual(plan.facts.git.remote, null); + assert.strictEqual(plan.facts.git.ghAuthed, null); + assert.notStrictEqual(plan.recommended.defaultIsolation, 'worktree'); + }); + + it('recommends worktree isolation when cwd is a git repo', function () { + const plan = fullyConfiguredPlan(); + assert.strictEqual(plan.facts.git.isRepo, true); + assert.strictEqual(plan.recommended.defaultIsolation, 'worktree'); + }); + }); + + describe('decisionId registry snapshot', function () { + it('matches the exact registry ID set (catches accidental renames)', function () { + const plan = freshMachinePlan(); + const recommendedIds = new Set(Object.keys(plan.recommended)); + assert.deepStrictEqual(recommendedIds, EXPECTED_DECISION_IDS); + }); + }); + + describe('canonical write paths', function () { + it('defaultIsolation only ever writes defaultDocker, defaultDelivery only ever writes defaultDelivery', function () { + const plan = freshMachinePlan(); + const isolationWrites = plan.proposedWrites.filter( + (w) => w.decisionId === 'defaultIsolation' + ); + const deliveryWrites = plan.proposedWrites.filter((w) => w.decisionId === 'defaultDelivery'); + assert.ok(isolationWrites.length > 0); + assert.ok(deliveryWrites.length > 0); + assert.ok(isolationWrites.every((w) => w.path === 'defaultDocker')); + assert.ok(deliveryWrites.every((w) => w.path === 'defaultDelivery')); + assert.ok(isolationWrites.every((w) => typeof w.to === 'boolean')); + }); + }); + + describe('no writes', function () { + it('never touches the filesystem', function () { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'zeroshot-setup-plan-')); + try { + const before = fs.readdirSync(tempDir).sort(); + buildSetupPlan({ + cwd: tempDir, + settings: { __meta: { fileExists: false } }, + repoSettings: null, + env: {}, + deps: makeDeps(), + }); + const after = fs.readdirSync(tempDir).sort(); + assert.deepStrictEqual(before, after); + assert.deepStrictEqual(before, []); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + }); +}); diff --git a/tests/unit/setup-undo.test.js b/tests/unit/setup-undo.test.js new file mode 100644 index 00000000..8d57dbea --- /dev/null +++ b/tests/unit/setup-undo.test.js @@ -0,0 +1,193 @@ +/** + * Test: `zeroshot setup undo` (lib/setup-undo.js) + * + * Verifies the three-way conflict rule from issue #606: + * - current === appliedValue -> restore priorValue (delete if null) + * - current === priorValue -> already-restored (no-op) + * - otherwise (changed since apply) -> skipped-modified, never clobbered + * - undo is idempotent + * - full round trip: plan -> apply -> undo returns settings to exact pre-apply bytes + */ + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const crypto = require('crypto'); +const { execSync } = require('child_process'); + +const settingsPath = require.resolve('../../lib/settings'); +const setupApplyPath = require.resolve('../../lib/setup-apply'); +const setupUndoPath = require.resolve('../../lib/setup-undo'); +const setupJournalPath = require.resolve('../../lib/setup-journal'); +const repoSettingsPath = require.resolve('../../lib/repo-settings'); + +describe('setup-undo', function () { + this.timeout(15000); + + let TEST_DIR; + let TEST_SETTINGS_FILE; + let repoRoot; + let applyModule; + let undoModule; + + function decisionsFile(obj) { + const p = path.join(TEST_DIR, 'decisions.json'); + fs.writeFileSync(p, JSON.stringify(obj), 'utf8'); + return p; + } + + function readSettings() { + return JSON.parse(fs.readFileSync(TEST_SETTINGS_FILE, 'utf8')); + } + + beforeEach(function () { + TEST_DIR = path.join( + os.tmpdir(), + 'zeroshot-setup-undo-test-' + crypto.randomBytes(8).toString('hex') + ); + fs.mkdirSync(TEST_DIR, { recursive: true }); + TEST_SETTINGS_FILE = path.join(TEST_DIR, 'settings.json'); + + repoRoot = path.join(TEST_DIR, 'repo'); + fs.mkdirSync(repoRoot, { recursive: true }); + execSync('git init', { cwd: repoRoot, stdio: 'ignore' }); + + process.env.ZEROSHOT_SETTINGS_FILE = TEST_SETTINGS_FILE; + delete require.cache[settingsPath]; + delete require.cache[setupApplyPath]; + delete require.cache[setupUndoPath]; + delete require.cache[setupJournalPath]; + delete require.cache[repoSettingsPath]; + + applyModule = require('../../lib/setup-apply'); + undoModule = require('../../lib/setup-undo'); + }); + + afterEach(function () { + delete process.env.ZEROSHOT_SETTINGS_FILE; + fs.rmSync(TEST_DIR, { recursive: true, force: true }); + }); + + it('restores priorValue for a key unmodified since apply', function () { + applyModule.applyDecisions({ + decisionsPath: decisionsFile({ defaultProvider: 'codex' }), + cwd: repoRoot, + }); + assert.strictEqual(readSettings().defaultProvider, 'codex'); + + const results = undoModule.undo({}); + assert.strictEqual(results.length, 1); + assert.strictEqual(results[0].status, 'restored'); + assert.strictEqual(readSettings().defaultProvider, 'claude'); + }); + + it('deletes the key when priorValue is null (key did not exist before apply)', function () { + // A journaled write whose key did not exist pre-apply (priorValue: null) — + // built directly against the journal file, since a real apply of a key that + // did not previously exist is not reachable via any currently-consumed + // decisionId (every consumed settings key already has a default value). + const settings = { defaultProvider: 'claude', syntheticNewKey: 'created-by-apply' }; + fs.mkdirSync(TEST_DIR, { recursive: true }); + fs.writeFileSync(TEST_SETTINGS_FILE, JSON.stringify(settings, null, 2)); + + const journalPath = path.join(TEST_DIR, 'setup-undo-journal.json'); + fs.writeFileSync( + journalPath, + JSON.stringify( + { + version: 1, + entries: [ + { + scope: 'global', + path: 'syntheticNewKey', + repoRoot: null, + priorValue: null, + appliedValue: 'created-by-apply', + appliedAt: new Date(0).toISOString(), + }, + ], + }, + null, + 2 + ) + ); + + const results = undoModule.undo({}); + const syntheticResult = results.find((r) => r.path === 'syntheticNewKey'); + assert.strictEqual(syntheticResult.status, 'deleted'); + assert.strictEqual('syntheticNewKey' in readSettings(), false); + }); + + it('skips (never clobbers) a key changed externally since apply, reporting skipped-modified', function () { + applyModule.applyDecisions({ + decisionsPath: decisionsFile({ defaultProvider: 'codex' }), + cwd: repoRoot, + }); + + // External tooling changes the same key after apply, before undo runs. + const settings = readSettings(); + settings.defaultProvider = 'gemini'; + fs.writeFileSync(TEST_SETTINGS_FILE, JSON.stringify(settings, null, 2)); + + const results = undoModule.undo({}); + assert.strictEqual(results[0].status, 'skipped-modified'); + assert.strictEqual(results[0].current, 'gemini'); + assert.strictEqual(results[0].wouldRestore, 'claude'); + // Must NOT clobber the externally-set value. + assert.strictEqual(readSettings().defaultProvider, 'gemini'); + }); + + it('is idempotent: running undo twice reports already-restored on the second run with no writes', function () { + applyModule.applyDecisions({ + decisionsPath: decisionsFile({ defaultProvider: 'codex', defaultDelivery: 'pr' }), + cwd: repoRoot, + }); + + const first = undoModule.undo({}); + assert.ok(first.every((r) => r.status === 'restored')); + const settingsAfterFirstUndo = fs.readFileSync(TEST_SETTINGS_FILE, 'utf8'); + + const second = undoModule.undo({}); + assert.ok(second.every((r) => r.status === 'already-restored')); + assert.strictEqual(fs.readFileSync(TEST_SETTINGS_FILE, 'utf8'), settingsAfterFirstUndo); + }); + + it('full round trip: plan -> apply -> undo returns global settings to exact pre-apply bytes', function () { + const { buildSetupPlan } = require('../../lib/setup-plan'); + const { loadSettings, saveSettings, settingsFileExists } = require('../../lib/settings'); + const { readRepoSettings } = require('../../lib/repo-settings'); + + // Establish a concrete pre-apply settings file (simulates a user who has + // already customized some settings before ever running `setup apply`). + // logLevel has no cascading normalization on load/save, unlike maxModel + // (which recomputes providerSettings.claude.maxLevel on every loadSettings() + // call) - picking it keeps this test isolated to the apply/undo round trip. + const preExisting = loadSettings(); + preExisting.logLevel = 'verbose'; + saveSettings(preExisting); + const preApplyBytes = fs.readFileSync(TEST_SETTINGS_FILE, 'utf8'); + + const cwd = repoRoot; + const settings = loadSettings(); + settings.__meta = { fileExists: settingsFileExists() }; + const { settings: repoSettings } = readRepoSettings(cwd); + const plan = buildSetupPlan({ + cwd, + settings, + repoSettings, + env: { ...process.env, __isTTY: false }, + }); + assert.ok(plan.schemaVersion); + + applyModule.applyDecisions({ + decisionsPath: decisionsFile({ defaultProvider: 'codex', defaultDelivery: 'pr' }), + cwd: repoRoot, + }); + assert.notStrictEqual(fs.readFileSync(TEST_SETTINGS_FILE, 'utf8'), preApplyBytes); + + undoModule.undo({}); + + assert.strictEqual(fs.readFileSync(TEST_SETTINGS_FILE, 'utf8'), preApplyBytes); + }); +}); diff --git a/tests/unit/start-cluster-config.test.js b/tests/unit/start-cluster-config.test.js index c9074a21..0ac201e3 100644 --- a/tests/unit/start-cluster-config.test.js +++ b/tests/unit/start-cluster-config.test.js @@ -67,6 +67,47 @@ describe('buildStartOptions() runMode', function () { }); }); +describe('buildStartOptions() isolation (single producer via run plan)', function () { + it('derives isolation=true / worktree=false for --docker', function () { + const result = buildStartOptions({ clusterId: 'c1', options: { docker: true } }); + assert.strictEqual(result.isolation, true); + assert.strictEqual(result.worktree, false); + }); + + it('derives worktree=true / isolation=false for --pr (worktree delivery)', function () { + const result = buildStartOptions({ clusterId: 'c1', options: { pr: true } }); + assert.strictEqual(result.isolation, false); + assert.strictEqual(result.worktree, true); + }); + + it('folds settings.defaultDocker into the plan (isolation without a CLI flag)', function () { + const result = buildStartOptions({ + clusterId: 'c1', + options: {}, + settings: { defaultDocker: true }, + }); + assert.strictEqual(result.isolation, true); + assert.strictEqual(result.worktree, false); + // Label reflects the SAME effective plan, not just the raw flags. + assert.strictEqual(result.runMode, 'docker'); + }); + + it('isolation and worktree are mutually exclusive (docker wins over worktree)', function () { + const result = buildStartOptions({ + clusterId: 'c1', + options: { worktree: true, docker: true }, + }); + assert.strictEqual(result.isolation, true); + assert.strictEqual(result.worktree, false); + }); + + it('no isolation when no flags and no settings', function () { + const result = buildStartOptions({ clusterId: 'c1', options: {}, settings: {} }); + assert.strictEqual(result.isolation, false); + assert.strictEqual(result.worktree, false); + }); +}); + describe('buildStartOptions() autoMerge', function () { it('resolves autoMerge=true for --ship', function () { const result = buildStartOptions({ clusterId: 'c1', options: { ship: true } }); diff --git a/tests/unit/start-cluster-default-delivery.test.js b/tests/unit/start-cluster-default-delivery.test.js new file mode 100644 index 00000000..ac102af5 --- /dev/null +++ b/tests/unit/start-cluster-default-delivery.test.js @@ -0,0 +1,54 @@ +const assert = require('assert'); + +const { resolveEffectiveRunPlan, startClusterFromText } = require('../../lib/start-cluster'); + +function captureStartOptions(settings) { + const config = { agents: [] }; + const orchestrator = { + start(_config, _input, startOptions) { + return startOptions; + }, + }; + return startClusterFromText({ + orchestrator, + config, + clusterId: 'c1', + text: 'hello', + options: {}, + settings, + }); +} + +describe('resolveEffectiveRunPlan() settings.defaultDelivery (issue #606)', function () { + it('folds merged defaultDelivery=ship into delivery + autoMerge', function () { + const plan = resolveEffectiveRunPlan({ ship: true }, {}); + assert.strictEqual(plan.delivery, 'ship'); + assert.strictEqual(plan.autoMerge, true); + assert.strictEqual(plan.isolation, 'worktree'); + }); + + it('folds merged defaultDelivery=pr into delivery without autoMerge', function () { + const plan = resolveEffectiveRunPlan({ pr: true }, {}); + assert.strictEqual(plan.delivery, 'pr'); + assert.strictEqual(plan.autoMerge, false); + assert.strictEqual(plan.isolation, 'worktree'); + }); + + it('defaults to delivery=none when settings.defaultDelivery is unset', function () { + const plan = resolveEffectiveRunPlan({}, {}); + assert.strictEqual(plan.delivery, 'none'); + assert.strictEqual(plan.autoMerge, false); + }); + + it('a CLI --pr flag still wins when settings.defaultDelivery=none', function () { + const plan = resolveEffectiveRunPlan({ pr: true }, { defaultDelivery: 'none' }); + assert.strictEqual(plan.delivery, 'pr'); + }); + + it('startClusterFromText folds settings.defaultDelivery into autoPr/autoMerge', function () { + const result = captureStartOptions({ defaultDelivery: 'ship' }); + assert.strictEqual(result.autoPr, true); + assert.strictEqual(result.autoMerge, true); + assert.strictEqual(result.worktree, true); + }); +}); diff --git a/tests/unit/stream-json-parser-codex.test.js b/tests/unit/stream-json-parser-codex.test.js index 3bcea16d..64ee62a6 100644 --- a/tests/unit/stream-json-parser-codex.test.js +++ b/tests/unit/stream-json-parser-codex.test.js @@ -48,4 +48,31 @@ describe('stream-json parser (Codex)', () => { const events = parseChunk(chunk); assert.deepStrictEqual(events, [{ type: 'result', success: false, error: 'boom' }]); }); + + it('does not leak Gemini tool ids across parseChunk calls', () => { + const toolUse = JSON.stringify({ + type: 'tool_use', + tool_call_id: 'tool-1', + tool_name: 'bash', + input: { cmd: 'ls' }, + }); + const toolResult = JSON.stringify({ type: 'tool_result', output: 'ok', success: true }); + + assert.deepStrictEqual(parseChunk(toolUse), [ + { + type: 'tool_call', + toolName: 'bash', + toolId: 'tool-1', + input: { cmd: 'ls' }, + }, + ]); + assert.deepStrictEqual(parseChunk(toolResult), [ + { + type: 'tool_result', + toolId: undefined, + content: 'ok', + isError: false, + }, + ]); + }); });