From 33ff97b493268ca1f7e3e702f3edf891af66752e Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Tue, 20 Oct 2026 22:25:38 -0500 Subject: [PATCH 01/18] git: one runner for every engine git call, with replace refs and grafts off refs/replace/* and $GIT_DIR/info/grafts are repo state every worktree shares, so an agent can write either, and every git the engine ran there read the agent's stand-in: a `git replace` of the pushed commit hid a refused file and a big one from pushProblems (2 problems became 0), and a graft that made the base reach the head emptied the range (2 became 0). `--no-replace-objects` alone still honours grafts (measured on git 2.43). src/os/git.ts is now the one way src/ runs git: every call gets `--no-replace-objects` and GIT_NO_REPLACE_OBJECTS=1, and GIT_GRAFT_FILE names a path that can't exist, set last over the caller's environment. A runner that takes any command (the setup probes' and init's run) gives git the settings by its name (commandSpawnSync). scripts/lib/git.cjs holds the same settings for scripts; the files that run on their own (the push gate and its hook, new-deps, the test-run VM script) write them out, and the hook hands them to the gate's environment. gitleaks, which runs git itself, gets them in its environment, in the engine's scan and in the push gate. Display and prompt readers follow the issue: the owner's decision diff, the fix-forward culprit diff and the patch hash read every file's lines whatever the change's attributes say (--text, no textconv, no external diff). The evaluator's sources line (src/roles.ts) needs the owner's role eval and the no-newline size count needs the owner's formula: both stay for the owner. test/git-runner.test.ts fails on a git call in src/ or scripts/ that doesn't come through a runner (a scan, a backstop next to the runner, with its exceptions registered as a never-grow table), and is locked with the runner itself. Test (fails without): test/git-runner.test.ts: the replace route and the graft route on pushProblems (each 2 problems became 0 on main), the runner scan (81 files bypassed it on main), the gitleaks and push-gate replace routes. Signed-off-by: Jayden Vasquez --- .worklane/review.yaml | 1 + scripts/bench/grade.ts | 6 +- scripts/check-dco.mjs | 5 +- scripts/check-denylist.mjs | 4 +- scripts/ci/affected.mjs | 4 +- scripts/ci/installed-engine-config.mjs | 7 +- scripts/ci/new-deps.mjs | 12 +- scripts/ci/testrun-real-systemd.mjs | 8 +- scripts/ci/updater-setup-root.mjs | 10 +- scripts/ci/wizard-e2e.mjs | 5 +- scripts/dev/compare-screenshots.mjs | 7 +- scripts/eval-roles.mjs | 4 +- scripts/lib/git.cjs | 44 ++++ scripts/lib/release-tree.mjs | 4 +- scripts/pin-engine.mjs | 6 +- scripts/push-gate-hook.mjs | 19 +- scripts/push-gate.mjs | 16 +- scripts/release.mjs | 3 +- scripts/rename.mjs | 4 +- scripts/setup-hooks.mjs | 4 +- scripts/setup/engine-release.cjs | 7 +- scripts/verify-release-tag.mjs | 5 +- scripts/verify-release-tree.mjs | 3 +- src/auto-merge-resume.ts | 7 +- src/check-pin-block.ts | 3 +- src/checkout-sync.ts | 4 +- src/checkout.ts | 4 +- src/claims.ts | 6 +- src/cli.ts | 7 +- src/comment-policy.ts | 4 +- src/config-change.ts | 3 +- src/config/live.ts | 8 +- src/config/load.ts | 4 +- src/coordinator.ts | 22 +- src/coordinator/base.ts | 4 +- src/coordinator/change-files.ts | 19 +- src/coordinator/checks-from-feedback.ts | 8 +- src/coordinator/conversation.ts | 4 +- src/coordinator/decisions.ts | 10 +- src/coordinator/fix-runs.ts | 10 +- src/coordinator/intake.ts | 6 +- src/coordinator/landing.ts | 6 +- src/coordinator/learnings.ts | 8 +- src/coordinator/merge-flow.ts | 43 ++-- src/coordinator/pr-batch.ts | 4 +- src/coordinator/pr-hygiene.ts | 6 +- src/coordinator/test-runs.ts | 4 +- src/dashboard.ts | 5 +- src/decision-detail.ts | 11 +- src/demo.ts | 5 +- src/deploy-gate/wire.ts | 4 +- src/direct-land.ts | 4 +- src/doctor.ts | 3 +- src/experiments/view.ts | 6 +- src/harness-evals/replay.ts | 4 +- src/harness-evals/schedule.ts | 6 +- src/init.ts | 6 +- src/install.ts | 8 +- src/main-watch.ts | 6 +- src/merge-base.ts | 4 +- src/merge-policy-weakening.ts | 4 + src/nightly.ts | 6 +- src/os/git.ts | 112 +++++++++ src/plugins/fetch.ts | 3 +- src/push-check.ts | 8 +- src/review.ts | 9 +- src/scan/secrets.ts | 7 +- src/server-auto-config.ts | 4 +- src/setup-actions.ts | 8 +- src/setup-steps.ts | 5 +- src/skill-lint.ts | 4 +- src/starter-ci.ts | 5 +- src/testrun/cache.ts | 4 +- src/testrun/host.ts | 6 +- src/testrun/measure-cli.ts | 4 +- src/testrun/predates.ts | 6 +- src/testrun/select.ts | 4 +- src/testrun/workspace.ts | 8 +- src/tune.ts | 5 +- src/worktree-pool.ts | 4 +- src/worktrees.ts | 9 +- test/git-runner.test.ts | 287 ++++++++++++++++++++++++ test/review.test.ts | 2 +- 83 files changed, 758 insertions(+), 220 deletions(-) create mode 100644 scripts/lib/git.cjs create mode 100644 src/os/git.ts create mode 100644 test/git-runner.test.ts diff --git a/.worklane/review.yaml b/.worklane/review.yaml index bf46c2235..6892eb0dc 100644 --- a/.worklane/review.yaml +++ b/.worklane/review.yaml @@ -28,6 +28,7 @@ categories: - test/package-files.test.ts - test/managed-apps.test.ts - test/merge-idempotent.test.ts + - test/git-runner.test.ts # Listed here only, not locked in code: the tests' own machine (every test runs inside it), the check that engine # code never writes a slot-directory file in place (and that a slot path reaches no other writer, #772), and the # tests that assert the agents' tool guard denies a call. diff --git a/scripts/bench/grade.ts b/scripts/bench/grade.ts index 1d17b8383..742f6f5b0 100644 --- a/scripts/bench/grade.ts +++ b/scripts/bench/grade.ts @@ -4,10 +4,10 @@ // graded is exactly what was exported. It then writes SWE-bench predictions (one set per label: each needs its own // harness run id) into the held-out store, the only place a task key meets its instance id. The official harness // grades them in fresh containers from the pristine images, after the whole run. -import { execFileSync } from 'node:child_process'; import { lstatSync, mkdtempSync, readdirSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { gitExecFileSync } from '../../src/os/git.js'; import { ownerOnlyDir, writeOwnerOnly } from '../../src/os/index.js'; import type { BenchLayout } from './paths.js'; import { readManifest, TASK_KEY, writeStore } from './store.js'; @@ -45,8 +45,8 @@ export function exportPatch(o: { layout: BenchLayout; runId: string; key: string } const bare = mkdtempSync(join(tmpdir(), 'bench-export-')); try { - execFileSync('git', ['init', '--bare', '-q', '--template=', bare], { env: GIT_ENV(), timeout: 60_000 }); - const diff = execFileSync('git', [...GIT_HARDENED, '--git-dir', bare, 'diff', '--binary', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', o.base, o.head, '--'], { + gitExecFileSync(['init', '--bare', '-q', '--template=', bare], { env: GIT_ENV(), timeout: 60_000 }); + const diff = gitExecFileSync([...GIT_HARDENED, '--git-dir', bare, 'diff', '--binary', '--no-color', '--no-ext-diff', '--no-textconv', '--src-prefix=a/', '--dst-prefix=b/', o.base, o.head, '--'], { env: GIT_ENV({ GIT_ALTERNATE_OBJECT_DIRECTORIES: objects }), encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, diff --git a/scripts/check-dco.mjs b/scripts/check-dco.mjs index d9a092842..d58e661e6 100644 --- a/scripts/check-dco.mjs +++ b/scripts/check-dco.mjs @@ -2,7 +2,7 @@ // Fail if any commit in .. lacks a Signed-off-by trailer that // matches its author (Developer Certificate of Origin, see CONTRIBUTING.md). // node scripts/check-dco.mjs [head] -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './lib/git.cjs'; const [base, head = 'HEAD'] = process.argv.slice(2); if (!base) { @@ -11,8 +11,7 @@ if (!base) { } const SEP = '\x1e'; -const log = execFileSync( - 'git', +const log = gitExecFileSync( ['log', '--no-merges', `--format=%H%x1f%an <%ae>%x1f%B${SEP}`, `${base}..${head}`], { encoding: 'utf8' }, ); diff --git a/scripts/check-denylist.mjs b/scripts/check-denylist.mjs index 3f248651d..74adcc65a 100644 --- a/scripts/check-denylist.mjs +++ b/scripts/check-denylist.mjs @@ -3,7 +3,7 @@ // // node scripts/check-denylist.mjs scan tracked + new files // node scripts/check-denylist.mjs --hash t print the hash entry for term t -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './lib/git.cjs'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { loadRules, scanText, sha, SKIP } from './lib/denylist.mjs'; @@ -15,7 +15,7 @@ if (args[0] === '--hash') { } const root = args[0] ?? '.'; const rules = loadRules(root); -const files = execFileSync('git', ['ls-files', '-co', '--exclude-standard'], { cwd: root, encoding: 'utf8' }) +const files = gitExecFileSync(['ls-files', '-co', '--exclude-standard'], { cwd: root, encoding: 'utf8' }) .split('\n') .filter((f) => f && !SKIP.has(f)); diff --git a/scripts/ci/affected.mjs b/scripts/ci/affected.mjs index f49c4a02f..af9cfdd0e 100644 --- a/scripts/ci/affected.mjs +++ b/scripts/ci/affected.mjs @@ -8,7 +8,7 @@ // node scripts/ci/affected.mjs --event pull_request --base the selected test files (test/x.test.ts), one per line // node scripts/ci/affected.mjs --event push every test file (any event but pull_request) // add --explain for why each one was selected (on stderr) -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../lib/git.cjs'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; @@ -96,7 +96,7 @@ export async function plan(dir, changed, added = new Set()) { return a.selectTests(g, changed, { isTest, forceAll, added }); } -const git = (dir, ...args) => execFileSync('git', args, { cwd: dir, encoding: 'utf8', maxBuffer: 64 << 20 }).split('\0').filter(Boolean); +const git = (dir, ...args) => gitExecFileSync(args, { cwd: dir, encoding: 'utf8', maxBuffer: 64 << 20 }).split('\0').filter(Boolean); function arg(args, name) { const i = args.indexOf(name); diff --git a/scripts/ci/installed-engine-config.mjs b/scripts/ci/installed-engine-config.mjs index 6f7223fa5..28cedf579 100644 --- a/scripts/ci/installed-engine-config.mjs +++ b/scripts/ci/installed-engine-config.mjs @@ -40,7 +40,8 @@ // A change that touches neither .worklane/ nor the pin has nothing to load: it prints so and exits 0, before fetching // any engine (the workflow's paths filter already skips most such PRs; this keeps the script right without it). // Exit 1: refused, or it couldn't check. -import { execFileSync, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; +import { gitExecFileSync, gitSpawnSync } from '../lib/git.cjs'; import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -77,7 +78,7 @@ class Refused extends Error {} const fail = (msg) => { throw new Refused(msg); }; -const git = (...a) => spawnSync('git', a, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024 }); +const git = (...a) => gitSpawnSync(a, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024 }); const gitText = (...a) => { const r = git(...a); return r.status === 0 ? r.stdout.trim() : null; @@ -442,7 +443,7 @@ async function loadWith({ commit, label: engine }, root, head) { const dir = mkdtempSync(join(tmpdir(), 'installed-engine-')); try { try { - const tar = execFileSync('git', ['archive', '--format=tar', commit, 'dist/src'], { maxBuffer: 512 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'] }); + const tar = gitExecFileSync(['archive', '--format=tar', commit, 'dist/src'], { maxBuffer: 512 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'] }); const x = spawnSync('tar', ['-x', '-C', dir], { input: tar, stdio: ['pipe', 'ignore', 'pipe'] }); if (x.status !== 0) throw new Error(lastLine(x.stderr) || `tar exited ${x.status}`); } catch (e) { diff --git a/scripts/ci/new-deps.mjs b/scripts/ci/new-deps.mjs index 33a20022a..0c48da331 100644 --- a/scripts/ci/new-deps.mjs +++ b/scripts/ci/new-deps.mjs @@ -16,6 +16,8 @@ // else from the event payload at $GITHUB_EVENT_PATH (pull_request.body). Exit 1 with each problem when any. import { execFileSync } from 'node:child_process'; import { readFileSync } from 'node:fs'; +import { devNull } from 'node:os'; +import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; export const SPEC_FIELDS = ['dependencies', 'optionalDependencies', 'peerDependencies']; @@ -142,12 +144,18 @@ export function check({ basePkg, headPkg, baseLock, headLock, body }) { return { changed: [...changed].map(([name, c]) => ({ name, versions: [...c.versions], licences: [...c.licences], why: c.why })), problems }; } -const git = (...a) => execFileSync('git', a, { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); +// The git runner's settings (#807; src/os/git.ts, scripts/lib/git.cjs): replace refs off and no graft file. Written +// out here because CI runs the base's copy of this file alone, from a temp dir (new-deps.yml), so it imports nothing +// of the repo's; test/git-runner.test.ts checks they are the runner's. +const GIT_FIXED_ARGS = ['--no-replace-objects']; +const GIT_ENV = { ...process.env, GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }; +const runGit = (args, opts) => execFileSync('git', [...GIT_FIXED_ARGS, ...args], { ...opts, env: GIT_ENV }); +const git = (...a) => runGit(a, { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); /** A JSON file at a revision; null when the revision doesn't have it. */ function jsonAt(rev, path) { let text; try { - text = execFileSync('git', ['show', `${rev}:${path}`], { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] }); + text = runGit(['show', `${rev}:${path}`], { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] }); } catch { git('rev-parse', '--verify', '--quiet', `${rev}^{commit}`); // a bad revision fails loudly; a missing file is null return null; diff --git a/scripts/ci/testrun-real-systemd.mjs b/scripts/ci/testrun-real-systemd.mjs index 62e61ff36..14579369c 100644 --- a/scripts/ci/testrun-real-systemd.mjs +++ b/scripts/ci/testrun-real-systemd.mjs @@ -14,7 +14,7 @@ import assert from 'node:assert/strict'; import { execFileSync, spawnSync } from 'node:child_process'; import { appendFileSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; import { connect } from 'node:net'; -import { homedir } from 'node:os'; +import { devNull, homedir } from 'node:os'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; @@ -56,7 +56,11 @@ const socketFromHere = (path) => new Promise((resolve) => { // A repo of this user's own: main, and a head commit on top of it. const repo = mkdtempSync(join(homedir(), 'repo-')); -const git = (...a) => execFileSync('git', ['-C', repo, '-c', 'user.name=ci', '-c', 'user.email=ci@example.com', ...a], { encoding: 'utf8', env: { PATH: '/usr/bin:/bin', HOME: homedir(), GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1' } }).trim(); +// The git runner's settings (#807; src/os/git.ts, scripts/lib/git.cjs): replace refs off and no graft file. Written +// out here because the job copies this file alone into the VM (ci-slow.yml); test/git-runner.test.ts checks them. +const GIT_FIXED_ARGS = ['--no-replace-objects']; +const GIT_ENV = { PATH: '/usr/bin:/bin', HOME: homedir(), GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1', GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }; +const git = (...a) => execFileSync('git', [...GIT_FIXED_ARGS, '-C', repo, '-c', 'user.name=ci', '-c', 'user.email=ci@example.com', ...a], { encoding: 'utf8', env: GIT_ENV }).trim(); git('init', '-q', '-b', 'main'); writeFileSync(join(repo, 'a.txt'), 'main\n'); mkdirSync(join(repo, 'art')); diff --git a/scripts/ci/updater-setup-root.mjs b/scripts/ci/updater-setup-root.mjs index 8fea55144..b92cb1cf0 100644 --- a/scripts/ci/updater-setup-root.mjs +++ b/scripts/ci/updater-setup-root.mjs @@ -26,6 +26,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, import { createRequire } from 'node:module'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; +import { commandExecFileSync, gitExecFileSync } from '../lib/git.cjs'; assert.equal(process.getuid(), 0, 'run this as root'); const coordinator = process.argv[2]; @@ -58,7 +59,8 @@ const owner = (f) => { const sha = (f) => (existsSync(f) ? createHash('sha256').update(readFileSync(f)).digest('hex') : 'absent'); const changes = () => readFileSync(CHANGES, 'utf8').trim().split('\n').map((l) => JSON.parse(l)); const pending = () => JSON.parse(readFileSync(PENDING, 'utf8')).pending; -const sh = (file, args, cwd = '/') => execFileSync(file, args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +// A caller's git (the command named git) through the git runner (#807). +const sh = (file, args, cwd = '/') => commandExecFileSync(file, args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); /** One run of the updater's unit, as its timer starts it; when it fails, its own journal says why. */ const updaterRun = () => { // This proof starts the unit many times in a few seconds, past systemd's default start limit (5 in 10 s); its timer @@ -164,7 +166,7 @@ release('v1.0.2', other.file); // The installed updater's own check, as root, against the machine's signers file. const updater = createRequire(import.meta.url)(UPDATER); let n = 0; -const verify = (tag) => updater.verifyRelease(origin, tag, SIGNERS, (file, args) => execFileSync(file, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }), join(work, `verify-${++n}.git`)); +const verify = (tag) => updater.verifyRelease(origin, tag, SIGNERS, (file, args) => commandExecFileSync(file, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }), join(work, `verify-${++n}.git`)); // ---- every other argument list: refused, and nothing changes const k = other.pub; @@ -405,9 +407,9 @@ const deps = { installed: () => installedSha, installedVersion: () => '1.0.2', // The release tags and their check: real, against the local repo, as root. - verifyRelease: (_url, tag, signers) => updater.verifyRelease(origin, tag, signers, (file, args) => execFileSync(file, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }), join(work, `run-verify-${++n}.git`)), + verifyRelease: (_url, tag, signers) => updater.verifyRelease(origin, tag, signers, (file, args) => commandExecFileSync(file, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }), join(work, `run-verify-${++n}.git`)), exec: (file, args) => { - if (file === 'git' && args[0] === 'ls-remote') return execFileSync('git', ['ls-remote', '--tags', origin, 'refs/tags/v*'], { encoding: 'utf8' }); + if (file === 'git' && args[0] === 'ls-remote') return gitExecFileSync(['ls-remote', '--tags', origin, 'refs/tags/v*'], { encoding: 'utf8' }); if (file === 'systemctl' && args[0] === 'list-units') return 'worklane-ci-standin.service loaded active running x\n'; if (file === 'systemctl' && args[0] === 'show') return 'ActiveState=active\nSubState=running\nNRestarts=0\nResult=success\n'; runLogs.push({ exec: [file, ...args].join(' ').slice(0, 80) }); diff --git a/scripts/ci/wizard-e2e.mjs b/scripts/ci/wizard-e2e.mjs index 44ca0383b..9b5e8f704 100644 --- a/scripts/ci/wizard-e2e.mjs +++ b/scripts/ci/wizard-e2e.mjs @@ -17,10 +17,10 @@ // browser asked for) and /automation.json (each step's "Without this page" line as the page showed it), for // the job's leak check. Nothing secret is ever printed: the key goes into the page from its file, never typed here. import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; +import { commandSpawnSync } from '../lib/git.cjs'; const [mode, dist, wizardUrl, keyFile, out, extra] = process.argv.slice(2); if (!['local', 'server', 'setup-loopback'].includes(mode) || !dist || !wizardUrl || !keyFile || !out || !extra) { @@ -116,8 +116,9 @@ async function firstRun(fresh = false) { return r.ok ? r.json() : { error: r.status }; })()`); } +// A caller's git (the command named git) through the git runner (#807). const sh = (cmd, args, o = {}) => { - const r = spawnSync(cmd, args, { encoding: 'utf8', ...o }); + const r = commandSpawnSync(cmd, args, { encoding: 'utf8', ...o }); return { status: r.status, out: (r.stdout ?? '').trim(), err: (r.stderr ?? '').trim() }; }; /** A file's owner and octal mode (through sudo on a server: the files are other users'). */ diff --git a/scripts/dev/compare-screenshots.mjs b/scripts/dev/compare-screenshots.mjs index 58aae4032..4a3df8edc 100644 --- a/scripts/dev/compare-screenshots.mjs +++ b/scripts/dev/compare-screenshots.mjs @@ -7,11 +7,11 @@ // Writes /before/*.png and /after/*.png (light and dark per page). // Stopped (Ctrl-C, a signal) or failing partway, it stops the step it is running, whose own clean-up // stops what that step started (the screenshot script's dashboards and Chrome), and removes its clones. -import { spawn, spawnSync } from 'node:child_process'; import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { commandSpawn, gitSpawnSync } from '../lib/git.cjs'; import { exitCleanup } from './children.mjs'; const arg = (name, fallback) => { @@ -24,11 +24,12 @@ const cleanup = exitCleanup(); const run = (cmd, args, cwd) => new Promise((ok, fail) => { // npm is npm.cmd on Windows, which only a shell resolves. - const p = cleanup.track(spawn(cmd, args, { cwd, stdio: 'inherit', shell: process.platform === 'win32' && cmd === 'npm' })); + // A caller's git (the command named git) through the git runner (#807). + const p = cleanup.track(commandSpawn(cmd, args, { cwd, stdio: 'inherit', shell: process.platform === 'win32' && cmd === 'npm' })); p.on('error', fail); p.on('exit', (code, signal) => (code === 0 ? ok() : fail(new Error(`${cmd} ${args.join(' ')} failed in ${cwd}${signal ? ` (${signal})` : ''}`)))); }); -const git = (...args) => spawnSync('git', args, { cwd: repo, encoding: 'utf8' }).stdout.trim(); +const git = (...args) => gitSpawnSync(args, { cwd: repo, encoding: 'utf8' }).stdout.trim(); await run('git', ['fetch', '-q', 'origin'], repo); const base = arg('--base', 'origin/main'); diff --git a/scripts/eval-roles.mjs b/scripts/eval-roles.mjs index ed8437452..df4a7b521 100644 --- a/scripts/eval-roles.mjs +++ b/scripts/eval-roles.mjs @@ -32,7 +32,7 @@ // [--fix-cap 10] // --write-held-out: write a held-out set for each role that has none (one case-writer run per role, from its // contract only); --score: score the current prompts once, no base (no --runs, no --fix). -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './lib/git.cjs'; import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -92,7 +92,7 @@ let noBase = false; if (!o.scoreOnly) { let baseSource; try { - baseSource = execFileSync('git', ['show', `${o.base}:src/roles.ts`], { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + baseSource = gitExecFileSync(['show', `${o.base}:src/roles.ts`], { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); } catch { console.error(`no src/roles.ts at ${o.base}`); // Without --spend the plan still prints (and the run stops there anyway). diff --git a/scripts/lib/git.cjs b/scripts/lib/git.cjs new file mode 100644 index 000000000..c48e3d958 --- /dev/null +++ b/scripts/lib/git.cjs @@ -0,0 +1,44 @@ +'use strict'; +// The one way scripts/ runs git (#807): the engine runner's settings (src/os/git.ts) for scripts, which run without a +// build. Every git process runs with replace refs off (`--no-replace-objects`, GIT_NO_REPLACE_OBJECTS=1) and reads no +// graft file (GIT_GRAFT_FILE names a path that can't exist), set last over the caller's environment. +// test/git-runner.test.ts checks these are the engine's settings, and fails on a git call in scripts/ that doesn't +// come through here. CommonJS, so a .cjs script can require it and a .mjs script can import it. +const { execFileSync, spawn, spawnSync } = require('node:child_process'); +const { devNull } = require('node:os'); +const { join } = require('node:path'); + +const GIT_FIXED_ARGS = Object.freeze(['--no-replace-objects']); +const GIT_FIXED_ENV = Object.freeze({ GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); + +/** `env` (or this process's environment), with the fixed settings on top. */ +const gitEnv = (env) => ({ ...(env || process.env), ...GIT_FIXED_ENV }); +/** The argument list a git process gets: the fixed arguments, then `args`. */ +const gitArgs = (args) => [...GIT_FIXED_ARGS, ...args]; +/** True for a command that runs git: `git`, or a path to it, in any case. */ +const isGitCommand = (cmd) => ['git', 'git.exe'].includes((String(cmd).split(/[\\/]/).pop() || '').toLowerCase()); + +/** spawnSync of git. */ +const gitSpawnSync = (args, options = {}) => spawnSync('git', gitArgs(args), { ...options, env: gitEnv(options.env) }); +/** execFileSync of git. */ +const gitExecFileSync = (args, options = {}) => execFileSync('git', gitArgs(args), { ...options, env: gitEnv(options.env) }); +/** spawn of git. */ +const gitSpawn = (args, options = {}) => spawn('git', gitArgs(args), { ...options, env: gitEnv(options.env) }); +/** spawnSync of a command named at run time: git (by name or path) with the settings, anything else as it is. */ +const commandSpawnSync = (cmd, args, options = {}) => (isGitCommand(cmd) ? spawnSync(cmd, gitArgs(args), { ...options, env: gitEnv(options.env) }) : spawnSync(cmd, args, options)); +/** execFileSync of a command named at run time: git (by name or path) with the settings, anything else as it is. */ +const commandExecFileSync = (cmd, args, options = {}) => (isGitCommand(cmd) ? execFileSync(cmd, gitArgs(args), { ...options, env: gitEnv(options.env) }) : execFileSync(cmd, args, options)); +/** spawn of a command named at run time: git (by name or path) with the settings, anything else as it is. */ +const commandSpawn = (cmd, args, options = {}) => (isGitCommand(cmd) ? spawn(cmd, gitArgs(args), { ...options, env: gitEnv(options.env) }) : spawn(cmd, args, options)); + +exports.GIT_FIXED_ARGS = GIT_FIXED_ARGS; +exports.GIT_FIXED_ENV = GIT_FIXED_ENV; +exports.gitEnv = gitEnv; +exports.gitArgs = gitArgs; +exports.isGitCommand = isGitCommand; +exports.gitSpawnSync = gitSpawnSync; +exports.gitExecFileSync = gitExecFileSync; +exports.gitSpawn = gitSpawn; +exports.commandSpawnSync = commandSpawnSync; +exports.commandExecFileSync = commandExecFileSync; +exports.commandSpawn = commandSpawn; diff --git a/scripts/lib/release-tree.mjs b/scripts/lib/release-tree.mjs index aafa674ab..8c8c78979 100644 --- a/scripts/lib/release-tree.mjs +++ b/scripts/lib/release-tree.mjs @@ -2,7 +2,7 @@ // dist/web/licenses.txt that wasn't the signed one, and it was published.) Compared by git blob id: a file's raw bytes // hashed as `git hash-object --no-filters` does, against the id `git ls-tree` lists for that path in the commit. No // clean filter, no line-ending conversion: equal ids mean equal bytes. Modes are not compared (npm normalises them). -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './git.cjs'; import { createHash } from 'node:crypto'; import { readdirSync, readFileSync } from 'node:fs'; import { join, relative, sep } from 'node:path'; @@ -31,7 +31,7 @@ export function filesUnder(base, dir = base) { /** The commit's files under `prefixes`: path -> blob id. */ export function committedFiles(commit, prefixes, cwd = process.cwd()) { - const raw = execFileSync('git', ['ls-tree', '-r', '-z', '--full-tree', commit, '--', ...prefixes], { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); + const raw = gitExecFileSync(['ls-tree', '-r', '-z', '--full-tree', commit, '--', ...prefixes], { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); const files = new Map(); for (const rec of raw.split('\0').filter(Boolean)) { const [meta, path] = [rec.slice(0, rec.indexOf('\t')), rec.slice(rec.indexOf('\t') + 1)]; diff --git a/scripts/pin-engine.mjs b/scripts/pin-engine.mjs index b14e63414..b7f0597fb 100644 --- a/scripts/pin-engine.mjs +++ b/scripts/pin-engine.mjs @@ -10,7 +10,7 @@ // - its commit is a release commit: one commit on top of a commit on origin/main, changing only dist/, carrying // dist/src/cli.js, with package.json at X.Y.Z (the shape scripts/release.mjs makes); // then writes ENGINE_TAG=vX.Y.Z and ENGINE_COMMIT= into scripts/bootstrap.sh, and nothing else. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './lib/git.cjs'; import { readFileSync, writeFileSync } from 'node:fs'; import { readPin, releaseProblem, writePin } from './lib/engine-pin.mjs'; import { canVerify } from './lib/signing.mjs'; @@ -28,7 +28,7 @@ const fail = (msg) => { if (!/^v\d+\.\d+\.\d+$/.test(tag ?? '')) fail('usage: pin-engine.mjs vX.Y.Z [--allowed-signers ] [--bootstrap ]'); const file = opt('bootstrap') ?? 'scripts/bootstrap.sh'; const signers = opt('allowed-signers'); -const git = (...a) => execFileSync('git', a, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const git = (...a) => gitExecFileSync(a, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); const ok = (...a) => { try { git(...a); @@ -69,7 +69,7 @@ const facts = { parents: parents.length, // --no-renames: a file moved into dist/ still names the path it left. -z, untrimmed: a valid UTF-8 name as itself; // any other has its bad bytes read as U+FFFD, so it starts with `dist/` only if its bytes do (#683). - changed: parents.length === 1 ? execFileSync('git', ['diff', '--name-only', '--no-renames', '-z', parents[0], commit], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).split('\0').filter(Boolean) : [], + changed: parents.length === 1 ? gitExecFileSync(['diff', '--name-only', '--no-renames', '-z', parents[0], commit], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).split('\0').filter(Boolean) : [], hasCli: ok('cat-file', '-e', `${commit}:dist/src/cli.js`), version: (() => { try { diff --git a/scripts/push-gate-hook.mjs b/scripts/push-gate-hook.mjs index 4f5f78927..3d89d18b3 100644 --- a/scripts/push-gate-hook.mjs +++ b/scripts/push-gate-hook.mjs @@ -18,7 +18,7 @@ // it is refused inside an agent run, and each use prints a loud line and is appended to HATCH_LOG in the git dir. import { execFileSync, spawnSync } from 'node:child_process'; import { appendFileSync, chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir, userInfo } from 'node:os'; +import { devNull, tmpdir, userInfo } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -38,8 +38,19 @@ class Refusal extends Error {} /** A refusal that first-time setup may answer with PUSH_GATE_BOOTSTRAP=1 (origin doesn't hold the gate yet). */ class NotOnOrigin extends Refusal {} +/** + * The git runner's settings (#807; src/os/git.ts, scripts/lib/git.cjs): replace refs off and no graft file, over the + * caller's environment. Written out here because this file runs on its own (setup-hooks copies it alone into the + * hooks dir), so it can't import them; test/git-runner.test.ts checks they are the runner's. The gate gets them too, + * in its environment, so every git it and gitleaks run has them, whichever gate origin holds. + */ +export const GIT_FIXED_ARGS = Object.freeze(['--no-replace-objects']); +export const GIT_FIXED_ENV = Object.freeze({ GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); +/** execFileSync of git, with the settings. Every git this file runs comes through here. */ +const runGit = (args, opts = {}) => execFileSync('git', [...GIT_FIXED_ARGS, ...args], { ...opts, env: { ...(opts.env || process.env), ...GIT_FIXED_ENV } }); + function git(args, opts = {}) { - return execFileSync('git', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024, ...opts }); + return runGit(args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024, ...opts }); } const tryGit = (args, opts) => { try { @@ -91,7 +102,7 @@ function extractFromRev(rev, label, dir) { throw new NotOnOrigin(`${label} has no ${path}.`); } mkdirSync(dirname(join(dir, path)), { recursive: true }); - writeFileSync(join(dir, path), execFileSync('git', ['show', `${rev}:${path}`], { maxBuffer: 64 * 1024 * 1024 })); + writeFileSync(join(dir, path), runGit(['show', `${rev}:${path}`], { maxBuffer: 64 * 1024 * 1024 })); } } @@ -227,7 +238,7 @@ function main(argv) { if (hookMode) noteIfHookIsOld(); // WHY: the gate reads its rules (and gitleaks its config) from the directory it runs in, so it runs in the // private copy; GIT_DIR points its git commands at this repo, and nothing else of the checkout is in reach. - const env = { ...process.env, GIT_DIR: gitDir }; + const env = { ...process.env, GIT_DIR: gitDir, ...GIT_FIXED_ENV }; for (const k of ['GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_PREFIX']) delete env[k]; const r = spawnSync(process.execPath, [join(dir, GATE_CODE[0]), ...argv], { cwd: dir, stdio: 'inherit', env }); if (r.error) throw r.error; diff --git a/scripts/push-gate.mjs b/scripts/push-gate.mjs index 0fd6e7a8f..e696df284 100644 --- a/scripts/push-gate.mjs +++ b/scripts/push-gate.mjs @@ -19,10 +19,19 @@ // email (a squash merge's DCO trailers) is dropped whole, name included. import { execFileSync, spawnSync } from 'node:child_process'; import { readFileSync } from 'node:fs'; +import { devNull } from 'node:os'; +import { join } from 'node:path'; import { loadRules, scanText, SKIP } from './lib/denylist.mjs'; const ZERO = /^0+$/; -const git = (...a) => execFileSync('git', a, { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); +// The git runner's settings (#807; src/os/git.ts, scripts/lib/git.cjs): replace refs off and no graft file, so a +// `git replace` or a graft in the repo can't swap the commits, trees or blobs scanned here. Written out here because +// the hook runs this file alone, fetched from origin (scripts/push-gate-hook.mjs GATE_CODE), so it imports nothing +// new; test/git-runner.test.ts checks they are the runner's. +const GIT_FIXED_ARGS = ['--no-replace-objects']; +const GIT_ENV = { ...process.env, GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }; +const runGit = (args, opts) => execFileSync('git', [...GIT_FIXED_ARGS, ...args], { ...opts, env: GIT_ENV }); +const git = (...a) => runGit(a, { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); function fail(why) { console.error(`push gate: REFUSED. ${why}`); @@ -69,7 +78,7 @@ function repoOwner() { if (fromEnv) return fromEnv[1].toLowerCase(); let url; try { - url = execFileSync('git', ['remote', 'get-url', 'origin'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); + url = runGit(['remote', 'get-url', 'origin'], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); } catch { return null; // no origin: no allowance } @@ -180,7 +189,8 @@ function scanTree(rules, tip) { function gitleaks(commits) { if (!commits.length) return; - const r = spawnSync('gitleaks', ['git', '--no-banner', '--redact', '--exit-code', '1', '--log-opts', `--no-walk ${commits.join(' ')}`, '.'], { encoding: 'utf8' }); + // gitleaks runs git itself: with the same settings, in its environment. + const r = spawnSync('gitleaks', ['git', '--no-banner', '--redact', '--exit-code', '1', '--log-opts', `--no-walk ${commits.join(' ')}`, '.'], { encoding: 'utf8', env: GIT_ENV }); if (r.error) fail(`gitleaks could not run (${r.error.code ?? r.error.message}); install it, the gate never skips the secret scan`); if (r.status === 1) fail(`gitleaks found a secret:\n${(r.stdout + r.stderr).trim().slice(-3000)}`); if (r.status !== 0) fail(`gitleaks exited ${r.status}:\n${r.stderr.trim().slice(-1000)}`); diff --git a/scripts/release.mjs b/scripts/release.mjs index 40d8a736e..f70589c90 100644 --- a/scripts/release.mjs +++ b/scripts/release.mjs @@ -22,6 +22,7 @@ // Pushing the tag runs .github/workflows/release.yml, which only STAGES the version on npm (after checking the tag's // signature against main's .github/release-signers); it goes live when the owner approves it on npmjs.com with 2FA. import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './lib/git.cjs'; import { existsSync, mkdtempSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -42,7 +43,7 @@ const fail = (msg) => { process.exit(1); }; if (!/^\d+\.\d+\.\d+$/.test(version ?? '')) fail('usage: release.mjs X.Y.Z | release.mjs changelog X.Y.Z [--repo owner/repo]'); -const git = (...a) => execFileSync('git', a, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const git = (...a) => gitExecFileSync(a, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); const gitConfig = (key) => { try { return git('config', '--get', key); diff --git a/scripts/rename.mjs b/scripts/rename.mjs index 9a2fe41c0..152782981 100644 --- a/scripts/rename.mjs +++ b/scripts/rename.mjs @@ -3,7 +3,7 @@ // npm run rename -- [--dry-run] // Reads the current name from src/brand.ts, then rewrites the three case // variants (Name, name, NAME) in every tracked or new non-ignored text file. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './lib/git.cjs'; import { readFileSync, writeFileSync } from 'node:fs'; // Files that mention the name as data, not as our identity. @@ -31,7 +31,7 @@ const pairs = [ [current, next], ]; -const files = execFileSync('git', ['ls-files', '-co', '--exclude-standard'], { encoding: 'utf8' }) +const files = gitExecFileSync(['ls-files', '-co', '--exclude-standard'], { encoding: 'utf8' }) .split('\n') .filter((f) => f && !SKIP.has(f)); diff --git a/scripts/setup-hooks.mjs b/scripts/setup-hooks.mjs index bc4859585..c61e9a20f 100644 --- a/scripts/setup-hooks.mjs +++ b/scripts/setup-hooks.mjs @@ -7,14 +7,14 @@ // branch being pushed can't change it; and the copy holds no rules of its own: it loads the gate and its rules // from origin's default branch on every push. The older `core.hooksPath=.githooks` setup ran the hook straight // from the checked-out branch, so this unsets it. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './lib/git.cjs'; import { chmodSync, copyFileSync, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; const MARKER = '# managed by setup-hooks: push gate'; const LAUNCHER = 'push-gate-hook.mjs'; -const git = (...a) => execFileSync('git', a, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const git = (...a) => gitExecFileSync(a, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); const tryGit = (...a) => { try { return git(...a); diff --git a/scripts/setup/engine-release.cjs b/scripts/setup/engine-release.cjs index 1a8bbf9bb..db4b39d64 100644 --- a/scripts/setup/engine-release.cjs +++ b/scripts/setup/engine-release.cjs @@ -12,8 +12,8 @@ // (never origin again), with origin's URL as its remote, checked out at by id. // It prints `verified : tag object , commit ` and exits 0, or names why and exits 1 with no checkout. 'use strict'; -const { execFileSync } = require('node:child_process'); const path = require('node:path'); +const { commandExecFileSync, gitExecFileSync } = require('../lib/git.cjs'); const { verifyRelease } = require('../machine/worklane-update.cjs'); const fail = (msg) => { @@ -26,7 +26,8 @@ if (!/^v[0-9]+\.[0-9]+\.[0-9]+$/.test(tag || '')) fail(`"${tag}" isn't a release if (!/^[0-9a-f]{40}$/.test(commit || '')) fail(`"${commit}" isn't a full commit id: a release is installed only at the commit scripts/bootstrap.sh pins`); if (!path.isAbsolute(signers)) fail(`the allowed signers file must be an absolute path, not ${signers}`); -const exec = (file, args) => execFileSync(file, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 300_000 }); +// git (the updater's verifyRelease runs it through exec) through the git runner (#807). +const exec = (file, args) => commandExecFileSync(file, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 300_000 }); const gitDir = path.join(work, 'verify.git'); const v = verifyRelease(url, tag, signers, exec, gitDir); if (!v.ok) fail(v.reason); @@ -34,7 +35,7 @@ if (v.commit !== commit) fail(`${tag} is signed and names itself, but points at // The source to build: the verified objects only, at the pinned commit by id. const src = path.join(work, 'src'); -const git = (...a) => exec('git', ['--no-replace-objects', ...a]).trim(); +const git = (...a) => gitExecFileSync(a, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 300_000 }).trim(); try { git('clone', '-q', '--no-checkout', gitDir, src); git('-C', src, 'remote', 'set-url', 'origin', url); diff --git a/scripts/verify-release-tag.mjs b/scripts/verify-release-tag.mjs index 164a6c2d2..7a0b72f95 100644 --- a/scripts/verify-release-tag.mjs +++ b/scripts/verify-release-tag.mjs @@ -14,7 +14,8 @@ // (default release-signer): `ssh-keygen -Y verify` on the tag's payload, as git itself checks it. Not // `git verify-tag`, which matches the signer to the tagger's email. // On success it prints `verified ` and exits 0. -import { execFileSync, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; +import { gitExecFileSync } from './lib/git.cjs'; import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -39,7 +40,7 @@ if (!commit) fail(usage); if (!/^[0-9a-f]{40}$/.test(commit)) fail(`--commit must be a full commit id, not "${commit}"`); if (!/^[\w.@-]+$/.test(principal)) fail(`a principal of letters, digits, ".", "@", "_" or "-", not "${principal}"`); -const git = (...a) => execFileSync('git', a, { stdio: ['ignore', 'pipe', 'pipe'] }); +const git = (...a) => gitExecFileSync(a, { stdio: ['ignore', 'pipe', 'pipe'] }); const gitText = (...a) => { try { return git(...a).toString('utf8').trim(); diff --git a/scripts/verify-release-tree.mjs b/scripts/verify-release-tree.mjs index 6db18b282..3881a3352 100644 --- a/scripts/verify-release-tree.mjs +++ b/scripts/verify-release-tree.mjs @@ -10,6 +10,7 @@ // (scripts/lib/release-tree.mjs): raw bytes, no line-ending conversion. // On success it prints `same as the signed tree: files at ` and exits 0. import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './lib/git.cjs'; import { existsSync, mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -26,7 +27,7 @@ const fail = (msg) => { if (!tgz || !commit || !existsSync(tgz)) fail('usage: verify-release-tree.mjs --commit '); let sha; try { - sha = execFileSync('git', ['rev-parse', '--verify', `${commit}^{commit}`], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + sha = gitExecFileSync(['rev-parse', '--verify', `${commit}^{commit}`], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); } catch { fail(`${commit} is not a commit here`); } diff --git a/src/auto-merge-resume.ts b/src/auto-merge-resume.ts index 750f3c5cb..ef96f4249 100644 --- a/src/auto-merge-resume.ts +++ b/src/auto-merge-resume.ts @@ -11,8 +11,9 @@ // Main's checks are JudgedChecks, so a pinned required check is green only from its pinned source. // Fail closed: a stop with no `by`, an unknown one, or an unreadable stop file waits for green main; main with no // required checks configured, a tip that can't be read, or checks that can't be read is not green. -import { spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; import type { JudgedChecks } from './check-source.js'; +import { gitSpawn } from './os/git.js'; import { killTree, spawnDetached } from './os/index.js'; import { requiredOutcomes } from './pr-watch.js'; @@ -81,9 +82,9 @@ export function remoteTip(repo: string, remote: string, branch: string, o: { tim }; // In its own process group, so a timeout kills git and everything it started (the transport: git-remote-https, // ssh, upload-pack), never leaving one holding a connection. - let child: ReturnType; + let child: ChildProcess; try { - child = spawn('git', ['ls-remote', remote, ref], { cwd: repo, detached: spawnDetached, stdio: ['ignore', 'pipe', 'ignore'] }); + child = gitSpawn(['ls-remote', remote, ref], { cwd: repo, detached: spawnDetached, stdio: ['ignore', 'pipe', 'ignore'] }); } catch { return finish(null); } diff --git a/src/check-pin-block.ts b/src/check-pin-block.ts index c45dc1554..291f6f3db 100644 --- a/src/check-pin-block.ts +++ b/src/check-pin-block.ts @@ -11,6 +11,7 @@ // - The block is valid under the engine's own schema, or it isn't offered: no duplicate key, no name the schema or // YAML can't take (those are comments). import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './os/git.js'; import { readFileSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { parse, parseDocument } from 'yaml'; @@ -58,7 +59,7 @@ export interface PinPlan { type Git = (repo: string, args: string[]) => string | null; const gitIn: Git = (repo, args) => { try { - return execFileSync('git', ['-C', repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 20_000, maxBuffer: 64 * 1024 * 1024 }); + return gitExecFileSync(['-C', repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 20_000, maxBuffer: 64 * 1024 * 1024 }); } catch { return null; } diff --git a/src/checkout-sync.ts b/src/checkout-sync.ts index dd163613b..28a13cf1b 100644 --- a/src/checkout-sync.ts +++ b/src/checkout-sync.ts @@ -2,7 +2,7 @@ // `git pull` in a terminal. This brings the checkout's default branch up to origin's, as a fast-forward only: never a // reset, a merge commit or a rebase, and never over the person's own work. Anything else is refused, saying why and // what to run instead. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from './os/git.js'; import { withToolPath } from './tool-path.js'; export interface GitRun { @@ -10,7 +10,7 @@ export interface GitRun { } export const systemGit = (root: string): GitRun => (args) => { - const r = spawnSync('git', args, { cwd: root, encoding: 'utf8', env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 120_000 }); + const r = gitSpawnSync(args, { cwd: root, encoding: 'utf8', env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 120_000 }); return { status: r.error ? null : r.status, stdout: r.stdout ?? '', stderr: r.error ? r.error.message : (r.stderr ?? '') }; }; diff --git a/src/checkout.ts b/src/checkout.ts index 8f55627d7..410137ef4 100644 --- a/src/checkout.ts +++ b/src/checkout.ts @@ -5,7 +5,7 @@ // don't read. A test-run tier that needs them says `full_checkout: true` (test_run.tiers). // // Non-cone sparse checkout ("/*" then "!//" per directory): everything, minus those directories. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './os/git.js'; import { mkdirSync, writeFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { z } from 'zod'; @@ -58,7 +58,7 @@ export function sparsePatterns(dirs: readonly string[]): string { return ['/*', ...dirs.map((d) => `!/${normalizeSparseDir(d)}/`)].join('\n') + '\n'; } -const git = (cwd: string, ...args: string[]) => execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const git = (cwd: string, ...args: string[]) => gitExecFileSync(args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); /** * Make the linked worktree at `path` (added with --no-checkout) sparse, leaving `dirs` out, and check its HEAD out. diff --git a/src/claims.ts b/src/claims.ts index 703e1788f..847372fae 100644 --- a/src/claims.ts +++ b/src/claims.ts @@ -7,7 +7,7 @@ // release = delete it, expecting our lease // Two coordinators (one per human) racing for the same issue: exactly one // push succeeds. Labels and assignees are cosmetic and follow the ref. -import { execFileSync, spawnSync } from 'node:child_process'; +import { gitExecFileSync, gitSpawnSync } from './os/git.js'; import { BRAND } from './brand.js'; export interface Lease { @@ -27,7 +27,7 @@ export interface ClaimOptions { export const claimRef = (issue: number) => `refs/${BRAND.cli}/claims/issue-${issue}`; function git(repo: string, args: string[], input?: string): string { - return execFileSync('git', args, { cwd: repo, encoding: 'utf8', input, stdio: ['pipe', 'pipe', 'pipe'] }).trim(); + return gitExecFileSync(args, { cwd: repo, encoding: 'utf8', input, stdio: ['pipe', 'pipe', 'pipe'] }).trim(); } /** A parentless commit holding the lease as its message. CAS needs no ancestry. */ @@ -39,7 +39,7 @@ function leaseCommit(repo: string, lease: Lease): string { /** Push `newSha` (or delete, when null) to the claim ref, only if it currently equals `expect` ('' = absent). */ function casPush(repo: string, remote: string, issue: number, expect: string, newSha: string | null): boolean { const ref = claimRef(issue); - const r = spawnSync('git', ['push', '--porcelain', `--force-with-lease=${ref}:${expect}`, remote, `${newSha ?? ''}:${ref}`], { cwd: repo, encoding: 'utf8' }); + const r = gitSpawnSync(['push', '--porcelain', `--force-with-lease=${ref}:${expect}`, remote, `${newSha ?? ''}:${ref}`], { cwd: repo, encoding: 'utf8' }); if (r.status === 0) return true; if (/\[rejected\]|stale info|already exists|fetch first/.test(r.stdout + r.stderr)) return false; throw new Error(`claim push failed: ${(r.stderr || r.stdout).trim().split('\n').pop()}`); diff --git a/src/cli.ts b/src/cli.ts index 366a11438..33820b7a5 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -71,6 +71,7 @@ import { sourceViewCommand } from './intake/source-view.js'; import { intakeCheckCommand } from './intake/check.js'; import { installService, stableNodePath, uninstallService } from './os/index.js'; import { ownEnv } from './os/own-env.js'; +import { gitExecFileSync } from './os/git.js'; import { projectStateDir } from './guardrails/context.js'; import { fileURLToPath } from 'node:url'; @@ -284,7 +285,7 @@ function option(args: string[], name: string): string | undefined { } function changedTestFiles(root: string, globs: string[], branch: string): string[] { - const base = execFileSync('git', ['merge-base', `origin/${branch}`, 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); + const base = gitExecFileSync(['merge-base', `origin/${branch}`, 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); // A renamed test file is an added one, and is checked (with renames on it was R, and skipped); a valid UTF-8 path as itself (#683). const files = changedNames(root, ['--diff-filter=AM', base]); const res = globs.map((g) => globToRegExp(g)); @@ -1011,8 +1012,8 @@ async function main(argv: string[]): Promise { const gh = i.credentials.github; const identity = i.commitIdentity ?? (gh.kind === 'app' ? await appBotIdentity({ appId: gh.app_id, installationId: gh.installation_id, keyPath: gh.key_path }) : { name: `${BRAND.cli}-${i.name}`, email: `${BRAND.cli}-${i.name}@users.noreply.invalid` }); const main = cfg.project.project.default_branch; - execFileSync('git', ['fetch', '-q', 'origin', main], { cwd: i.repo.path, stdio: 'inherit' }); - const base = execFileSync('git', ['rev-parse', `origin/${main}`], { cwd: i.repo.path, encoding: 'utf8' }).trim(); + gitExecFileSync(['fetch', '-q', 'origin', main], { cwd: i.repo.path, stdio: 'inherit' }); + const base = gitExecFileSync(['rev-parse', `origin/${main}`], { cwd: i.repo.path, encoding: 'utf8' }).trim(); const dryRun = flag(rest, '--dry-run'); const r = await tune({ wt: { repo: i.repo.path, root: cfg.tests.worktree.root, stateDir: i.stateDir, setup: cfg.tests.worktree.setup, ...(i.runAs ? { runAs: i.runAs } : {}), env: cfg.tests.env }, diff --git a/src/comment-policy.ts b/src/comment-policy.ts index aa28d278e..684088792 100644 --- a/src/comment-policy.ts +++ b/src/comment-policy.ts @@ -3,13 +3,13 @@ // function, checkComments, for the PR diff check (checkRange, scripts/check-comments.mjs) and the agents' post-edit // hook (checkWorkingFile, wired in #330's second part), so the two can't drift. Comments are found by a line // scanner per language (no parser dependency). -import { spawnSync } from 'node:child_process'; import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { parseDocument } from 'yaml'; import type { z } from 'zod'; import { BRAND } from './brand.js'; import { CommentPolicyConfig } from './config/schema.js'; +import { gitSpawnSync } from './os/git.js'; export type CommentPolicy = z.infer; @@ -492,7 +492,7 @@ export function addedLines(diff: string): Map> { // INVARIANT: no command a checkout's or user's git config names runs here (#330): fsmonitor off, no textconv or external diff. const git = (cwd: string, args: string[], timeoutMs?: number) => - spawnSync('git', ['-c', 'core.quotePath=false', '-c', 'core.fsmonitor=false', ...args], { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, ...(timeoutMs ? { timeout: timeoutMs } : {}) }); + gitSpawnSync(['-c', 'core.quotePath=false', '-c', 'core.fsmonitor=false', ...args], { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, ...(timeoutMs ? { timeout: timeoutMs } : {}) }); // INVARIANT: the hook and CI read the same new lines whatever the user's git config (prefixes, algorithm, context). const DIFF = [ ...['-c', 'diff.noprefix=false', '-c', 'diff.mnemonicPrefix=false', '-c', 'diff.interHunkContext=0', '-c', 'diff.suppressBlankEmpty=false'], diff --git a/src/config-change.ts b/src/config-change.ts index 0ebb1068c..39d11c0e8 100644 --- a/src/config-change.ts +++ b/src/config-change.ts @@ -3,6 +3,7 @@ // the same loader the coordinator uses, and proposed as a PR from a branch of their own; nothing is pushed to the // default branch, and nothing in the checkout is touched. import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from './os/git.js'; import { writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { parseDocument } from 'yaml'; @@ -75,7 +76,7 @@ export interface ConfigChange { type Git = (args: string[], cwd: string) => { status: number | null; stdout: string; stderr: string }; const realGit: Git = (args, cwd) => { - const r = spawnSync('git', args, { cwd, encoding: 'utf8' }); + const r = gitSpawnSync(args, { cwd, encoding: 'utf8' }); return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' }; }; diff --git a/src/config/live.ts b/src/config/live.ts index 0cd724148..28bd1dbea 100644 --- a/src/config/live.ts +++ b/src/config/live.ts @@ -4,7 +4,7 @@ // schema and checks as at start, and swapped in whole. A config that fails validation is never applied: the last // good one stays, and why is recorded. Settings read once at start (below) keep their running value until a restart. // Merges are still judged by the config at their own parent (main-watch.ts requiredAt): this is for everything else. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { BRAND } from '../brand.js'; import { ConfigInvalid, loadConfigAt, type Config, type ConfigError } from './load.js'; @@ -13,7 +13,7 @@ export const CONFIG_INCIDENT = 'config_invalid'; /** The git tree of the config directory at `rev`, or null (no such rev, or no config there). */ export function configTree(repo: string, rev: string): string | null { - const r = spawnSync('git', ['rev-parse', '-q', '--verify', `${rev}:${BRAND.configDir}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + const r = gitSpawnSync(['rev-parse', '-q', '--verify', `${rev}:${BRAND.configDir}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); return r.status === 0 ? r.stdout.trim() : null; } @@ -148,7 +148,7 @@ export class LiveConfig { } private sha(): string | null { - const r = spawnSync('git', ['rev-parse', '-q', '--verify', `${this.ref}^{commit}`], { cwd: this.o.repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + const r = gitSpawnSync(['rev-parse', '-q', '--verify', `${this.ref}^{commit}`], { cwd: this.o.repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); return r.status === 0 ? r.stdout.trim() : null; } } @@ -159,7 +159,7 @@ export class LiveConfig { */ export function startConfig(o: { repo: string; remote: string; fallback: Config; check?: (cfg: Config) => ConfigError[] }): { cfg: Config; tree: string | null; note: string | null } { const branch = o.fallback.project.project.default_branch; - spawnSync('git', ['fetch', '-q', o.remote, `+refs/heads/${branch}:refs/remotes/${o.remote}/${branch}`], { cwd: o.repo, stdio: 'ignore', timeout: 60_000 }); + gitSpawnSync(['fetch', '-q', o.remote, `+refs/heads/${branch}:refs/remotes/${o.remote}/${branch}`], { cwd: o.repo, stdio: 'ignore', timeout: 60_000 }); const ref = `refs/remotes/${o.remote}/${branch}`; const tree = configTree(o.repo, ref); // WHY (#715): whenever main's config can't run (none there, or invalid), the checkout's config as committed (the diff --git a/src/config/load.ts b/src/config/load.ts index 160cc50b8..25dd234d9 100644 --- a/src/config/load.ts +++ b/src/config/load.ts @@ -1,4 +1,4 @@ -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { parseDocument } from 'yaml'; @@ -68,7 +68,7 @@ export function loadConfig(root: string): Config { * schema and checks as loadConfig. Throws ConfigInvalid listing all errors. `dir` stays the checkout's. */ export function loadConfigAt(root: string, rev: string): Config { - const git = (...args: string[]) => spawnSync('git', args, { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 16 * 1024 * 1024 }); + const git = (...args: string[]) => gitSpawnSync(args, { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 16 * 1024 * 1024 }); if (git('rev-parse', '-q', '--verify', `${rev}:${BRAND.configDir}`).status !== 0) throw new ConfigInvalid([{ file: BRAND.configDir, path: '', message: `missing at ${rev}` }]); const show = (file: string) => { const r = git('show', `${rev}:${BRAND.configDir}/${file}`); diff --git a/src/coordinator.ts b/src/coordinator.ts index 8a204edbc..923f1224e 100644 --- a/src/coordinator.ts +++ b/src/coordinator.ts @@ -3,7 +3,8 @@ // step is decided from events plus GitHub, so a restart picks up where the // log says it was. Agents only ever propose: they commit on their own // branch in their own worktree; everything outward-facing happens here. -import { execFileSync, spawn, spawnSync } from 'node:child_process'; +import { spawn } from 'node:child_process'; +import { gitExecFileSync, gitSpawnSync } from './os/git.js'; import { createHash, randomBytes } from 'node:crypto'; import { cpSync, mkdirSync, readFileSync, existsSync, rmSync } from 'node:fs'; import { dirname, join } from 'node:path'; @@ -38,7 +39,7 @@ import { DEFAULT_HOTSPOTS, estimateFiles, fileIndex, holdReason, hotspotsIn, pic import { TaskGraph } from './coordinator/task-graph.js'; import { bodyHash, childBody, GOAL_LABEL, GOAL_SPLIT_LABEL, GOAL_SPLIT_SCHEMA, readGoalSplit, splitComment } from './goal-split.js'; import { changedFiles } from './merge-base.js'; -import { changedNames, diffStatText, rawChanges, untrackedNames } from './coordinator/change-files.js'; +import { changedNames, diffStatText, patchText, rawChanges, untrackedNames } from './coordinator/change-files.js'; import { importsOf, lightCheckPlan, lightCheckWait } from './light-check.js'; import { readRun, runsForIssue } from './run-record.js'; import { claimsOf, unbackedClaims, type CheckRecord } from './receipts.js'; @@ -508,7 +509,7 @@ export class Coordinator extends AutomationRuns { */ refreshConfig(): void { this.liveCfg ??= new LiveConfig({ repo: this.d.repo, remote: this.remote, branch: this.branch, ...(this.d.liveConfig?.tree !== undefined ? { tree: this.d.liveConfig.tree } : {}), ...(this.d.liveConfig?.check ? { check: this.d.liveConfig.check } : {}) }); - spawnSync('git', ['fetch', '-q', this.remote, `+refs/heads/${this.branch}:refs/remotes/${this.remote}/${this.branch}`], { cwd: this.d.repo, stdio: 'ignore', timeout: 60_000 }); + gitSpawnSync(['fetch', '-q', this.remote, `+refs/heads/${this.branch}:refs/remotes/${this.remote}/${this.branch}`], { cwd: this.d.repo, stdio: 'ignore', timeout: 60_000 }); const r = this.liveCfg.poll(this.repoCfg); if (r.kind === 'same') return; if (r.kind === 'invalid') { @@ -828,7 +829,7 @@ export class Coordinator extends AutomationRuns { this.poolBusy = true; this.poolRefill = (async () => { try { - spawnSync('git', ['fetch', '-q', this.remote, this.branch], { cwd: this.d.repo }); + gitSpawnSync(['fetch', '-q', this.remote, this.branch], { cwd: this.d.repo }); const base = this.git(this.d.repo, 'rev-parse', `${this.remote}/${this.branch}`); const wt = this.wt; const keyFiles = this.d.cfg.tests.worktree.setup_key; @@ -1518,7 +1519,8 @@ export class Coordinator extends AutomationRuns { } const added = files.reduce((s, f) => s + f.added, 0); const removed = files.reduce((s, f) => s + f.removed, 0); - const patchHash = createHash('sha256').update(this.git(path, 'diff', `${base}..${head}`)).digest('hex'); + // Over every file's lines, whatever the change's attributes say: a `-diff` file's new content is a new hash (#807). + const patchHash = createHash('sha256').update(patchText(path, [`${base}..${head}`])).digest('hex'); this.emit('change.proposed', { issue: n, branch: this.paths(n).branch, base, head, files: names, lines: added + removed, added, removed, patch_hash: patchHash }); return { files, tampered, patchHash }; } @@ -1788,10 +1790,10 @@ export class Coordinator extends AutomationRuns { const imports: Record = {}; if (mainChanged.length) { // -z, as prFiles: a valid UTF-8 path as itself, so an import of a file with an odd byte in its name still resolves. - const tree = new Set(execFileSync('git', ['ls-tree', '-r', '-z', '--name-only', pr.headSha], { cwd: this.d.repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 256 * 1024 * 1024 }).split('\0').filter(Boolean)); + const tree = new Set(gitExecFileSync(['ls-tree', '-r', '-z', '--name-only', pr.headSha], { cwd: this.d.repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 256 * 1024 * 1024 }).split('\0').filter(Boolean)); for (const f of prFiles) { if (!/\.((m|c)?(j|t)sx?|py)$/.test(f)) continue; - const r = spawnSync('git', ['show', `${pr.headSha}:${f}`], { cwd: this.d.repo, encoding: 'utf8' }); + const r = gitSpawnSync(['show', `${pr.headSha}:${f}`], { cwd: this.d.repo, encoding: 'utf8' }); if (r.status === 0) imports[f] = importsOf(f, r.stdout, tree); } } @@ -1839,7 +1841,7 @@ export class Coordinator extends AutomationRuns { // The monitor reads on the cheaper model routing gives a reflection (the short read-only run before a retry). const monitorModel = routeRun(this.d.cfg, 'reflection'); const show = (commit: string, path: string) => { - const r = spawnSync('git', ['show', `${commit}:${path}`], { cwd: this.d.repo, maxBuffer: 64 * 1024 * 1024 }); + const r = gitSpawnSync(['show', `${commit}:${path}`], { cwd: this.d.repo, maxBuffer: 64 * 1024 * 1024 }); return r.status === 0 ? r.stdout : null; }; return { @@ -1900,7 +1902,7 @@ export class Coordinator extends AutomationRuns { }); this.emit('run.cost', { issue: null, role: 'experimenter', model: r.model, usd: r.costUsd, turns: r.turns }); const files = o.hide.filter((h) => !h.includes('*')); - if (files.length) execFileSync('git', ['checkout', o.commit, '--', ...files], { cwd: path, stdio: 'ignore' }); + if (files.length) gitExecFileSync(['checkout', o.commit, '--', ...files], { cwd: path, stdio: 'ignore' }); return await then(r, path, lines.join('\n')); } finally { try { @@ -1930,7 +1932,7 @@ export class Coordinator extends AutomationRuns { }, bundle: (ref, commit) => { this.git(this.d.repo, 'update-ref', ref, commit); - const r = spawnSync('git', ['bundle', 'create', '-q', '-', ref], { cwd: this.d.repo, maxBuffer: 1024 * 1024 * 1024 }); + const r = gitSpawnSync(['bundle', 'create', '-q', '-', ref], { cwd: this.d.repo, maxBuffer: 1024 * 1024 * 1024 }); if (r.status !== 0) throw new Error(`git bundle ${ref}: ${r.stderr.toString().slice(0, 300)}`); return r.stdout; }, diff --git a/src/coordinator/base.ts b/src/coordinator/base.ts index 72c271e37..ac7f08b6c 100644 --- a/src/coordinator/base.ts +++ b/src/coordinator/base.ts @@ -1,7 +1,7 @@ // The coordinator's state and the helpers every part uses: its fields and constructor (the runner wrapper), // events, git, project commands, paths and task files, the governor, cost, asking, blocking and releasing. import type { Coordinator } from '../coordinator.js'; -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../os/git.js'; import { randomBytes } from 'node:crypto'; import { dirname, join } from 'node:path'; import { BRAND } from '../brand.js'; @@ -297,7 +297,7 @@ export abstract class CoordinatorBase { } protected git(cwd: string, ...args: string[]) { - return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + return gitExecFileSync(args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); } protected emit[0]>(type: T, payload: EventPayload) { diff --git a/src/coordinator/change-files.ts b/src/coordinator/change-files.ts index 3c7572b70..2bcb62847 100644 --- a/src/coordinator/change-files.ts +++ b/src/coordinator/change-files.ts @@ -25,7 +25,7 @@ // for a name that can't be handed back to git, from its blobs). review.ts flags a change that adds such a mark, since // GitHub and a checkout still honour it. import { Buffer, isUtf8 } from 'node:buffer'; -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import type { SizeFile } from '../pr-rules.js'; import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; @@ -57,7 +57,7 @@ function withGit(args: string[]): string[] { /** Run git; its stdout, untrimmed (a path may start or end with a space), or an Error saying why it failed. */ function run(cwd: string, args: string[], timeout?: number): string | Error { - const r = spawnSync('git', withGit(args), { cwd, encoding: 'utf8', maxBuffer: MAX_BUFFER, ...(timeout ? { timeout } : {}) }); + const r = gitSpawnSync(withGit(args), { cwd, encoding: 'utf8', maxBuffer: MAX_BUFFER, ...(timeout ? { timeout } : {}) }); if (r.error) return r.error; if (r.status !== 0) return new Error(`git ${args.join(' ').slice(0, 200)} failed: ${(r.stderr || `exit ${r.status}`).trim().slice(0, 500)}`); return r.stdout; @@ -65,7 +65,7 @@ function run(cwd: string, args: string[], timeout?: number): string | Error { /** run, with stdout as bytes: for output that holds paths. */ function runBytes(cwd: string, args: string[], timeout?: number): Buffer | Error { - const r = spawnSync('git', withGit(args), { cwd, encoding: 'buffer', maxBuffer: MAX_BUFFER, ...(timeout ? { timeout } : {}) }); + const r = gitSpawnSync(withGit(args), { cwd, encoding: 'buffer', maxBuffer: MAX_BUFFER, ...(timeout ? { timeout } : {}) }); if (r.error) return r.error; if (r.status !== 0) return new Error(`git ${args.join(' ').slice(0, 200)} failed: ${(r.stderr.toString('utf8') || `exit ${r.status}`).trim().slice(0, 500)}`); return r.stdout; @@ -168,6 +168,15 @@ export function diffStatText(cwd: string, rev: string): string { return must(run(cwd, ['diff', '--stat', '--no-renames', '--no-color', ...nameArgs([rev])])).trim(); } +/** + * `git diff ` as a patch, for a person, a prompt or a hash (#807): every file's lines whatever the attributes say + * (a `-diff` or `binary` mark, a textconv or an external diff), and every submodule link. `args` as changedNames takes + * them. Never parsed. + */ +export function patchText(cwd: string, args: string[]): string { + return must(run(cwd, ['diff', '--no-color', ...AS_TEXT, ...nameArgs(args)])); +} + /** `git show --stat --patch` of one commit, as text for a person. Never parsed. */ export function commitText(cwd: string, commit: string): string { return must(run(cwd, ['show', '--stat', '--patch', '--no-color', '--no-ext-diff', '--format=commit %H', ...nameArgs([commit])])); @@ -288,7 +297,7 @@ function treePaths(cwd: string, rev: string): Map { */ function attributeStates(cwd: string, rev: string, paths: Buffer[]): Map> | null { const input = Buffer.concat(paths.flatMap((p) => [p, Buffer.from([0])])); - const r = spawnSync('git', withGit(['check-attr', `--source=${rev}`, '-z', '--stdin', ...OUTCOME_ATTRIBUTES]), { cwd, input, encoding: 'buffer', maxBuffer: MAX_BUFFER, env: { ...process.env, GIT_ATTR_NOSYSTEM: '1' } }); + const r = gitSpawnSync(withGit(['check-attr', `--source=${rev}`, '-z', '--stdin', ...OUTCOME_ATTRIBUTES]), { cwd, input, encoding: 'buffer', maxBuffer: MAX_BUFFER, env: { ...process.env, GIT_ATTR_NOSYSTEM: '1' } }); if (r.error || r.status !== 0) return null; const out = new Map>(); // -z: "path NUL attribute NUL state NUL" for each path and attribute. @@ -344,7 +353,7 @@ const OUTCOME_SHOWN = 10; /** git's exit status and stdout, for the calls whose status is an answer (`cat-file -e`, `config --get-regexp`). */ function runStatus(cwd: string, args: string[]): { status: number | null; stdout: string } { - const r = spawnSync('git', withGit(args), { cwd, encoding: 'utf8', maxBuffer: MAX_BUFFER }); + const r = gitSpawnSync(withGit(args), { cwd, encoding: 'utf8', maxBuffer: MAX_BUFFER }); return { status: r.error ? null : r.status, stdout: r.stdout ?? '' }; } diff --git a/src/coordinator/checks-from-feedback.ts b/src/coordinator/checks-from-feedback.ts index bd3d0c659..e723eca30 100644 --- a/src/coordinator/checks-from-feedback.ts +++ b/src/coordinator/checks-from-feedback.ts @@ -3,7 +3,7 @@ // repo has sgconfig.yml) or a test, written by one run in a scratch worktree and checked before it is pushed; or an // issue, when the constraint (its evidence's files, or the draft's) is in a protected area. The PR is never recorded // as the harness's (no pr.opened) and its branch never auto-merges. See src/checks-from-feedback.ts. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import { BRAND } from '../brand.js'; @@ -64,7 +64,7 @@ export abstract class ChecksFromFeedback extends Optimizer { this.git(this.d.repo, 'fetch', '-q', this.remote, this.branch); const tip = this.git(this.d.repo, 'rev-parse', `${this.remote}/${this.branch}`); const dir = join(this.d.stateDir, 'feedback-checks', c.id); - spawnSync('git', ['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); + gitSpawnSync(['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); try { this.git(this.d.repo, 'worktree', 'add', '-q', '--detach', dir, tip); const mode = checkMode(dir); @@ -108,8 +108,8 @@ export abstract class ChecksFromFeedback extends Optimizer { await this.d.backlog.addLabels(pr.number, [CHECKS_LABEL]).catch(() => undefined); this.emit('feedback_check.proposed', { constraint: c.id, keys, kind: 'pr', number: pr.number, url: pr.url, mode: mode.kind, why: a.summary.slice(0, 1000) }); } finally { - spawnSync('git', ['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); - spawnSync('git', ['worktree', 'prune'], { cwd: this.d.repo }); + gitSpawnSync(['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); + gitSpawnSync(['worktree', 'prune'], { cwd: this.d.repo }); } } diff --git a/src/coordinator/conversation.ts b/src/coordinator/conversation.ts index a551b2ca3..a1ea8065a 100644 --- a/src/coordinator/conversation.ts +++ b/src/coordinator/conversation.ts @@ -4,7 +4,7 @@ // intake role; shows the draft; and files it `ready` only on an explicit yes from the person who posted it or an // admin (mayFileReady). Until then the draft is only in the coordinator's private store. Once filed, the thread // gets the issue's progress from the events: planned, PR opened, merged, live. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../os/git.js'; import { existsSync, readFileSync, rmSync, statSync } from 'node:fs'; import { dirname, join } from 'node:path'; import type { EventPayload } from '../events/types.js'; @@ -280,7 +280,7 @@ export abstract class Conversation extends Channel { private includes(deploy: string, sha: string): boolean { if (deploy === sha) return true; try { - execFileSync('git', ['merge-base', '--is-ancestor', sha, deploy], { cwd: this.d.repo, stdio: 'ignore' }); + gitExecFileSync(['merge-base', '--is-ancestor', sha, deploy], { cwd: this.d.repo, stdio: 'ignore' }); return true; } catch { return false; diff --git a/src/coordinator/decisions.ts b/src/coordinator/decisions.ts index e21c032b7..f60a6922f 100644 --- a/src/coordinator/decisions.ts +++ b/src/coordinator/decisions.ts @@ -1,5 +1,5 @@ // Decisions: held plans, refused hosts, and the owner's answers on issues. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { randomBytes } from 'node:crypto'; import { rmSync } from 'node:fs'; import { join } from 'node:path'; @@ -65,7 +65,7 @@ export abstract class Decisions extends Recovery { } const review: ReviewConfig = this.d.cfg.review ?? DEFAULT_REVIEW; // The plan gate: always (the default) holds every plan for the owner; risky only one the merge policy's checks hold. - const reasons = gateReasons(this.d.cfg.project.plan_gate, planHoldReasons(plan, { review, moneyPaths: this.moneyPaths(review.money_path_source), scope: lockScopeOf(this.d), existsAtBase: (d) => spawnSync('git', ['cat-file', '-e', `${base}:${d}`], { cwd: this.d.repo }).status === 0 })); + const reasons = gateReasons(this.d.cfg.project.plan_gate, planHoldReasons(plan, { review, moneyPaths: this.moneyPaths(review.money_path_source), scope: lockScopeOf(this.d), existsAtBase: (d) => gitSpawnSync(['cat-file', '-e', `${base}:${d}`], { cwd: this.d.repo }).status === 0 })); await this.d.backlog.comment(n, planComment(plan, owner, reasons)); const commentsAt = (await this.d.backlog.comments(n)).length; if (reasons.length) { @@ -203,15 +203,15 @@ export abstract class Decisions extends Recovery { const file = `${BRAND.configDir}/guardrails.yaml`; this.git(this.d.repo, 'fetch', '-q', this.remote, this.branch); const base = this.git(this.d.repo, 'rev-parse', `${this.remote}/${this.branch}`); - const shown = spawnSync('git', ['show', `${base}:${file}`], { cwd: this.d.repo, encoding: 'utf8' }); + const shown = gitSpawnSync(['show', `${base}:${file}`], { cwd: this.d.repo, encoding: 'utf8' }); if (shown.status !== 0) return { pr: null, detail: `${file} isn't on ${this.branch}, so no change was proposed` }; const next = withAllowedHost(shown.stdout, host); if (next === null) return { pr: null, detail: `${host} is already allowed on ${this.branch}` }; - const blob = spawnSync('git', ['hash-object', '-w', '--stdin'], { cwd: this.d.repo, encoding: 'utf8', input: next }).stdout.trim(); + const blob = gitSpawnSync(['hash-object', '-w', '--stdin'], { cwd: this.d.repo, encoding: 'utf8', input: next }).stdout.trim(); const index = join(this.d.stateDir, `allow-${randomBytes(3).toString('hex')}.index`); const env = { ...process.env, GIT_INDEX_FILE: index, GIT_AUTHOR_NAME: this.identity.name, GIT_AUTHOR_EMAIL: this.identity.email, GIT_COMMITTER_NAME: this.identity.name, GIT_COMMITTER_EMAIL: this.identity.email }; const g = (args: string[], input?: string) => { - const r = spawnSync('git', args, { cwd: this.d.repo, encoding: 'utf8', env, ...(input !== undefined ? { input } : {}) }); + const r = gitSpawnSync(args, { cwd: this.d.repo, encoding: 'utf8', env, ...(input !== undefined ? { input } : {}) }); if (r.status !== 0) throw new Error(`git ${args[0]} failed: ${(r.stderr || '').trim().slice(-300)}`); return r.stdout.trim(); }; diff --git a/src/coordinator/fix-runs.ts b/src/coordinator/fix-runs.ts index bc5d30e52..536dff809 100644 --- a/src/coordinator/fix-runs.ts +++ b/src/coordinator/fix-runs.ts @@ -1,7 +1,7 @@ // CI fix runs and conflict fix runs on the harness's own PRs. import { localDoneWhen } from '../testrun/local.js'; import type { Coordinator } from '../coordinator.js'; -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { randomBytes } from 'node:crypto'; import { BRAND } from '../brand.js'; import type { ReviewConfig } from '../config/schema.js'; @@ -314,7 +314,7 @@ export abstract class FixRuns extends Landing { const why = s.no_change_needed ?? `it made no commit (${s.summary})`; return await give(trigger.kind === 'ci' ? `the fix run found the failure isn't caused by this change: ${why}` : `the fix run changed nothing for the review: ${why}`); } - if (spawnSync('git', ['merge-base', '--is-ancestor', pr.headSha, head], { cwd: path }).status !== 0) return await give('the fix run rewrote the branch history instead of adding commits'); + if (gitSpawnSync(['merge-base', '--is-ancestor', pr.headSha, head], { cwd: path }).status !== 0) return await give('the fix run rewrote the branch history instead of adding commits'); const change = await this.inspect(n, path, claimed.base, head, repro); if ('rejected' in change) return await give(`the fix was rejected: ${change.rejected}`); const checks = await this.verify(n, path, head, doneWhen, repro); @@ -422,7 +422,7 @@ export abstract class FixRuns extends Landing { const baseSha = this.git(this.d.repo, 'rev-parse', `${this.remote}/${baseRef}`); // The branch already contains its base: GitHub's merge check is behind. Look again soon instead. this.git(this.d.repo, 'fetch', '-q', this.remote, `+refs/heads/${pr.head}:refs/remotes/${this.remote}/${pr.head}`); - if (spawnSync('git', ['merge-base', '--is-ancestor', baseSha, pr.headSha], { cwd: this.d.repo }).status === 0) { + if (gitSpawnSync(['merge-base', '--is-ancestor', baseSha, pr.headSha], { cwd: this.d.repo }).status === 0) { this.prPolled.set(number, Date.now() - (this.d.prPollMs ?? 60_000) + MERGEABLE_RECHECK_MS); return; } @@ -476,7 +476,7 @@ export abstract class FixRuns extends Landing { /** git, returning null instead of throwing (a path missing at a commit, an unmerged index entry). */ protected gitOrNull(cwd: string, ...args: string[]): string | null { - const r = spawnSync('git', args, { cwd, encoding: 'utf8' }); + const r = gitSpawnSync(args, { cwd, encoding: 'utf8' }); return r.status === 0 ? r.stdout.replace(/\n$/, '') : null; } @@ -562,7 +562,7 @@ export abstract class FixRuns extends Landing { if (this.gitOrNull(path, 'rev-parse', 'HEAD^1') !== pr.headSha || this.gitOrNull(path, 'rev-parse', 'HEAD^2') !== baseSha) return await give('the fix run did not finish the merge commit of the branch and its base (it rewrote history, added commits, or left the merge uncommitted)'); // Exactly as committed (no trimming): the comparison is line for line, the last line included. const committed = (f: string) => { - const r = spawnSync('git', ['show', `HEAD:${f}`], { cwd: path, encoding: 'utf8' }); + const r = gitSpawnSync(['show', `HEAD:${f}`], { cwd: path, encoding: 'utf8' }); return r.status === 0 ? r.stdout : null; }; const resolved = Object.fromEntries(files.map((f) => [f, committed(f)])); diff --git a/src/coordinator/intake.ts b/src/coordinator/intake.ts index 9cd15e12e..8b7b923cd 100644 --- a/src/coordinator/intake.ts +++ b/src/coordinator/intake.ts @@ -1,5 +1,5 @@ // Work intake (docs/intake.md): pulling the sources, triage, filing or asking, replies, and recovery after a restart. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../os/git.js'; import { createHash } from 'node:crypto'; import { readFileSync, existsSync, rmSync, statSync } from 'node:fs'; import { dirname, join } from 'node:path'; @@ -225,7 +225,7 @@ export abstract class Intake extends QuickReplies { if (rep?.result !== 'fixed_on_main' || !rep.commit) return t; const { commit, ...rest } = rep; try { - execFileSync('git', ['merge-base', '--is-ancestor', commit, `refs/remotes/${this.remote}/${this.branch}`], { cwd: this.d.repo, stdio: 'ignore', timeout: 10_000 }); + gitExecFileSync(['merge-base', '--is-ancestor', commit, `refs/remotes/${this.remote}/${this.branch}`], { cwd: this.d.repo, stdio: 'ignore', timeout: 10_000 }); return t; } catch { return { ...t, reproduction: rest }; @@ -462,7 +462,7 @@ export abstract class Intake extends QuickReplies { const includes = (deploy: string, sha: string) => { if (deploy === sha) return true; try { - execFileSync('git', ['merge-base', '--is-ancestor', sha, deploy], { cwd: this.d.repo, stdio: 'ignore' }); + gitExecFileSync(['merge-base', '--is-ancestor', sha, deploy], { cwd: this.d.repo, stdio: 'ignore' }); return true; } catch { return false; diff --git a/src/coordinator/landing.ts b/src/coordinator/landing.ts index b8dada770..4a6318333 100644 --- a/src/coordinator/landing.ts +++ b/src/coordinator/landing.ts @@ -1,5 +1,5 @@ // Landing: the land queue and batches, the gates before landing, opening PRs, and what follows a landing (deploy). -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { randomBytes } from 'node:crypto'; import { BRAND } from '../brand.js'; import { type DoneWhenList } from '../backlog/types.js'; @@ -166,9 +166,9 @@ export abstract class Landing extends Decisions { } for (const q of batch) { const claimed = this.events(q.issue).filter((e) => e.type === 'issue.claimed').at(-1)!.payload as EventPayload<'issue.claimed'>; - const pick = spawnSync('git', ['cherry-pick', `${claimed.base}..${q.head}`], { cwd: path, encoding: 'utf8' }); + const pick = gitSpawnSync(['cherry-pick', `${claimed.base}..${q.head}`], { cwd: path, encoding: 'utf8' }); if (pick.status !== 0) { - spawnSync('git', ['cherry-pick', '--abort'], { cwd: path }); + gitSpawnSync(['cherry-pick', '--abort'], { cwd: path }); result(q.issue, 'conflict', null, `does not apply cleanly on ${tip.slice(0, 8)}${applied.length ? ` after ${applied.map((a) => `#${a.issue}`).join(', ')}` : ''}; never guessing at conflicts`); await this.block(q.issue, ownerOf(q.issue), `conflicts with ${this.branch}; needs a rebase`); continue; diff --git a/src/coordinator/learnings.ts b/src/coordinator/learnings.ts index f90952211..dccae20ec 100644 --- a/src/coordinator/learnings.ts +++ b/src/coordinator/learnings.ts @@ -1,6 +1,6 @@ // WHY: item 33 wiring (src/learnings.ts): notes when a task ends, and an owner's promote filed as an AGENTS.md PR // built from the default branch with git plumbing (no worktree, no agent), labelled review:L3, never merged here. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { randomBytes } from 'node:crypto'; import { mkdirSync, rmSync } from 'node:fs'; import { join } from 'node:path'; @@ -44,15 +44,15 @@ export abstract class Learnings extends ChecksFromFeedback { private async fileAgentsMdPr(fp: string, line: string, by: string): Promise<{ pr: { url: string; number: number } | null; detail: string }> { this.git(this.d.repo, 'fetch', '-q', this.remote, this.branch); const base = this.git(this.d.repo, 'rev-parse', `${this.remote}/${this.branch}`); - const shown = spawnSync('git', ['show', `${base}:${AGENTS_MD}`], { cwd: this.d.repo, encoding: 'utf8' }); + const shown = gitSpawnSync(['show', `${base}:${AGENTS_MD}`], { cwd: this.d.repo, encoding: 'utf8' }); const next = withLearning(shown.status === 0 ? shown.stdout : null, line); if (next === null) return { pr: null, detail: `the line is in ${AGENTS_MD} on ${this.branch} already` }; - const blob = spawnSync('git', ['hash-object', '-w', '--stdin'], { cwd: this.d.repo, encoding: 'utf8', input: next }).stdout.trim(); + const blob = gitSpawnSync(['hash-object', '-w', '--stdin'], { cwd: this.d.repo, encoding: 'utf8', input: next }).stdout.trim(); mkdirSync(this.d.stateDir, { recursive: true }); const index = join(this.d.stateDir, `learning-${randomBytes(3).toString('hex')}.index`); const env = { ...process.env, GIT_INDEX_FILE: index, GIT_AUTHOR_NAME: this.identity.name, GIT_AUTHOR_EMAIL: this.identity.email, GIT_COMMITTER_NAME: this.identity.name, GIT_COMMITTER_EMAIL: this.identity.email }; const g = (args: string[], input?: string) => { - const r = spawnSync('git', args, { cwd: this.d.repo, encoding: 'utf8', env, ...(input !== undefined ? { input } : {}) }); + const r = gitSpawnSync(args, { cwd: this.d.repo, encoding: 'utf8', env, ...(input !== undefined ? { input } : {}) }); if (r.status !== 0) throw new Error(`git ${args[0]} failed: ${(r.stderr || '').trim().slice(-300)}`); return r.stdout.trim(); }; diff --git a/src/coordinator/merge-flow.ts b/src/coordinator/merge-flow.ts index 7d94dd06f..a8963b90e 100644 --- a/src/coordinator/merge-flow.ts +++ b/src/coordinator/merge-flow.ts @@ -4,7 +4,7 @@ import { startRefusal, TestRunHeld } from '../testrun/gate.js'; import { CI_NONE_MAIN_WATCH, ciIsNone, requiredFor } from '../ci-none.js'; import { PinnedChecks, type JudgedChecks } from '../check-source.js'; import type { Coordinator } from '../coordinator.js'; -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { readFileSync, existsSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { BRAND } from '../brand.js'; @@ -19,7 +19,7 @@ import { instructionTargets } from '../skilleval.js'; import { checkedPush } from '../push-check.js'; import { conflictTrigger } from '../conflicts.js'; import { abortMerge, mergeBaseInto } from '../merge-base.js'; -import { changedNamesOrNull } from './change-files.js'; +import { changedNamesOrNull, patchText } from './change-files.js'; import { lightOutcome } from '../light-check.js'; import { readiness, requiredOutcomes, type Readiness } from '../pr-watch.js'; import { checksFor, effectiveClass, riskClassOf, rulesetHold } from '../pr-rules.js'; @@ -123,12 +123,19 @@ export abstract class MergeFlow extends FixRuns { return git('rev-parse', '--verify', '-q', `${this.remote}/${this.branch}`); }, parent: (sha) => git('rev-parse', '--verify', '-q', `${sha}^1`), - isAncestor: (a, b) => spawnSync('git', ['merge-base', '--is-ancestor', a, b], { cwd: this.d.repo }).status === 0, + isAncestor: (a, b) => gitSpawnSync(['merge-base', '--is-ancestor', a, b], { cwd: this.d.repo }).status === 0, firstParents: (from, to) => (git('rev-list', '--first-parent', '--reverse', `${from}..${to}`) ?? '').split('\n').filter(Boolean), subject: (sha) => git('log', '-1', '--format=%s', sha) ?? '', // A rename as both its paths and a valid UTF-8 path as itself, so a locked category on either one holds (#683). files: (sha) => changedNamesOrNull(this.d.repo, [`${sha}^1`, sha]) ?? [], - diff: (sha) => git('diff', `${sha}^1`, sha) ?? '', + // The culprit's patch as text whatever its attributes say (#807): a `-diff` file shows its lines, not "Binary files differ". + diff: (sha) => { + try { + return patchText(this.d.repo, [`${sha}^1`, sha]); + } catch { + return ''; + } + }, }, config: () => this.d.cfg.project.fix_forward, required: () => this.requiredChecks(), @@ -307,7 +314,7 @@ export abstract class MergeFlow extends FixRuns { // A merge into the default branch (by anyone) is when PRs start conflicting: look at all of them now. if (this.watchedPrs().length && now - this.tipCheckedAt >= TIP_CHECK_MS) { this.tipCheckedAt = now; - const r = spawnSync('git', ['ls-remote', this.remote, `refs/heads/${this.branch}`], { cwd: this.d.repo, encoding: 'utf8' }); + const r = gitSpawnSync(['ls-remote', this.remote, `refs/heads/${this.branch}`], { cwd: this.d.repo, encoding: 'utf8' }); const tip = r.status === 0 ? (r.stdout.split(/\s/)[0] ?? '') : ''; if (tip && tip !== this.defaultTip) { if (this.defaultTip) this.prPolled.clear(); @@ -447,7 +454,7 @@ export abstract class MergeFlow extends FixRuns { if (!attempt) return; if (pr.state === 'merged' && pr.mergeCommit) this.git(this.d.repo, 'fetch', '-q', this.remote, this.branch); const commit = (sha: string) => { - if (spawnSync('git', ['merge-base', '--is-ancestor', sha, `${this.remote}/${this.branch}`], { cwd: this.d.repo }).status !== 0) return null; + if (gitSpawnSync(['merge-base', '--is-ancestor', sha, `${this.remote}/${this.branch}`], { cwd: this.d.repo }).status !== 0) return null; const out = this.gitOrNull(this.d.repo, 'log', '-1', '--format=%P%n%B', sha, '--'); if (out === null) return null; const [parents = '', ...message] = out.split('\n'); @@ -547,7 +554,7 @@ export abstract class MergeFlow extends FixRuns { const files = this.changeFiles(this.d.repo, base, pr.headSha); const labels = (this.events(n).filter((e) => e.type === 'issue.seen').at(-1)?.payload as { labels?: string[] } | undefined)?.labels ?? []; const level = computeLevel({ files, labels, moneyPaths: this.moneyPaths(review.money_path_source), scope: lockScopeOf(this.d) }, review); - const exists = (p: string) => spawnSync('git', ['cat-file', '-e', `${base}:${p}`], { cwd: this.d.repo }).status === 0; + const exists = (p: string) => gitSpawnSync(['cat-file', '-e', `${base}:${p}`], { cwd: this.d.repo }).status === 0; const newTopLevel = [...new Set(files.map((f) => f.path.split('/')).filter((s) => s.length > 1).map((s) => s[0]!))].filter((d) => !exists(d)).map((d) => `${d}/`); // The verdict on this head, or on the head the harness's merge-ins of the default branch were made on. const judged = carriedHead(mergeIns, number, pr.headSha); @@ -1173,15 +1180,15 @@ export abstract class MergeFlow extends FixRuns { this.emit('coordinator.error', { instance: this.d.instance, where: `revert ${name}`, kind: 'error', message: why.slice(0, 500) }); return { why: why.slice(0, 500) }; }; - spawnSync('git', ['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); + gitSpawnSync(['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); try { this.git(this.d.repo, 'worktree', 'add', '-q', '--detach', dir, tip); const parents = this.git(this.d.repo, 'rev-list', '--parents', '-n', '1', sha).split(' ').length - 1; // A range: newest first (git revert's order), into one commit (no pre-commit or commit-msg hook, as git revert itself). const rv = range - ? spawnSync('git', [...as, 'revert', '--no-commit', `${range[0]}^..${range.at(-1)}`], { cwd: dir, encoding: 'utf8' }) - : spawnSync('git', [...as, 'revert', ...(parents > 1 ? ['-m', '1'] : []), '--no-edit', sha], { cwd: dir, encoding: 'utf8' }); - const done = rv.status === 0 && range ? spawnSync('git', [...as, 'commit', '-q', '--no-verify', '-m', `${title}\n\nThis reverts commits ${range[0]}^..${range.at(-1)}.`], { cwd: dir, encoding: 'utf8' }) : rv; + ? gitSpawnSync([...as, 'revert', '--no-commit', `${range[0]}^..${range.at(-1)}`], { cwd: dir, encoding: 'utf8' }) + : gitSpawnSync([...as, 'revert', ...(parents > 1 ? ['-m', '1'] : []), '--no-edit', sha], { cwd: dir, encoding: 'utf8' }); + const done = rv.status === 0 && range ? gitSpawnSync([...as, 'commit', '-q', '--no-verify', '-m', `${title}\n\nThis reverts commits ${range[0]}^..${range.at(-1)}.`], { cwd: dir, encoding: 'utf8' }) : rv; if (done.status !== 0) { const lines = `${done.stdout ?? ''}\n${done.stderr ?? ''}`.split('\n').map((l) => l.trim()).filter(Boolean); const conflicts = lines.filter((l) => l.startsWith('CONFLICT')); @@ -1194,8 +1201,8 @@ export abstract class MergeFlow extends FixRuns { const pr = await this.d.backlog.openPr(branch, this.branch, title, body, { headSha: head }); return { url: pr.url, number: pr.number }; } finally { - spawnSync('git', ['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); - spawnSync('git', ['worktree', 'prune'], { cwd: this.d.repo }); + gitSpawnSync(['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); + gitSpawnSync(['worktree', 'prune'], { cwd: this.d.repo }); } } @@ -1301,9 +1308,9 @@ export abstract class MergeFlow extends FixRuns { */ private revertRange(sha: string, through: string, commits: number): string[] | null { const tip = `${this.remote}/${this.branch}`; - if (spawnSync('git', ['merge-base', '--is-ancestor', through, tip], { cwd: this.d.repo }).status !== 0) return null; + if (gitSpawnSync(['merge-base', '--is-ancestor', through, tip], { cwd: this.d.repo }).status !== 0) return null; // Parents before children whatever the commit dates say, so only the checks below decide. - const r = spawnSync('git', ['rev-list', '--topo-order', '--parents', '--reverse', `${sha}^..${through}`], { cwd: this.d.repo, encoding: 'utf8' }); + const r = gitSpawnSync(['rev-list', '--topo-order', '--parents', '--reverse', `${sha}^..${through}`], { cwd: this.d.repo, encoding: 'utf8' }); if (r.status !== 0) return null; const rows = r.stdout.split('\n').filter(Boolean).map((l) => l.split(' ')); if (rows.length < 2 || rows.length > Math.min(commits, 250) || rows[0]![0] !== sha || rows.at(-1)![0] !== through) return null; @@ -1313,7 +1320,7 @@ export abstract class MergeFlow extends FixRuns { /** Whether `a` is `b` or an ancestor of it, in the clone. */ private isAncestor(a: string, b: string) { - return spawnSync('git', ['merge-base', '--is-ancestor', a, b], { cwd: this.d.repo }).status === 0; + return gitSpawnSync(['merge-base', '--is-ancestor', a, b], { cwd: this.d.repo }).status === 0; } /** @@ -1374,7 +1381,7 @@ export abstract class MergeFlow extends FixRuns { } catch { who = null; } - const s = spawnSync('git', ['log', '-1', '--format=%s', sha], { cwd: this.d.repo, encoding: 'utf8' }); + const s = gitSpawnSync(['log', '-1', '--format=%s', sha], { cwd: this.d.repo, encoding: 'utf8' }); const subject = redactString(s.status === 0 ? s.stdout.trim() : '').slice(0, 300); this.emit('main_watch.direct_push', { sha, url, kind: push.kind, pusher: who?.login ?? null, pusher_from: who?.from ?? null, subject, alert: push.alert }); await this.directPushOutbound(); @@ -1439,7 +1446,7 @@ export abstract class MergeFlow extends FixRuns { } catch { // already gone } - spawnSync('git', ['branch', '-D', `${BRAND.cli}/combined-${n}`], { cwd: this.d.repo }); + gitSpawnSync(['branch', '-D', `${BRAND.cli}/combined-${n}`], { cwd: this.d.repo }); this.emit('light_check.finished', { issue: n, number, head: pr.headSha, main_sha: mainSha, overlap, outcome, wait_ms: Math.max(0, Date.now() - t0), detail }); this.prPolled.delete(number); // decide on the next pass, not after the poll interval } diff --git a/src/coordinator/pr-batch.ts b/src/coordinator/pr-batch.ts index 9277d63e3..74c9cec2e 100644 --- a/src/coordinator/pr-batch.ts +++ b/src/coordinator/pr-batch.ts @@ -11,7 +11,7 @@ // back in the queue), and nothing is merged twice. import { TestRunHeld } from '../testrun/gate.js'; import { GPU_NO_REPEAT, gpuRunsNotToRepeat } from '../testrun/gpu.js'; -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { BRAND } from '../brand.js'; import type { CommitCheck, PullRequest } from '../backlog/types.js'; import type { JudgedChecks } from '../check-source.js'; @@ -623,7 +623,7 @@ export class PrBatcher { } private git(...args: string[]) { - return spawnSync('git', args, { cwd: this.d.repo, encoding: 'utf8' }); + return gitSpawnSync(args, { cwd: this.d.repo, encoding: 'utf8' }); } private gitOk(...args: string[]): string { diff --git a/src/coordinator/pr-hygiene.ts b/src/coordinator/pr-hygiene.ts index c1c9f58d5..52ebb45c4 100644 --- a/src/coordinator/pr-hygiene.ts +++ b/src/coordinator/pr-hygiene.ts @@ -2,7 +2,7 @@ // over the open PRs this instance opened (watchedPrs: a person's PR is never among them). The git work happens in a // scratch worktree of the coordinator's clone; the only push is checkedPush to the PR's own branch, pinned to the // head it read (never over someone else's push, never the default branch). Nothing runs under an emergency stop. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { join } from 'node:path'; import { BRAND } from '../brand.js'; import type { PullRequest } from '../backlog/types.js'; @@ -111,8 +111,8 @@ export abstract class PrHygiene extends Gardener { const { issue: n, number } = w; const dir = join(this.d.stateDir, 'pr-hygiene', `pr-${number}`); const clean = () => { - spawnSync('git', ['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); - spawnSync('git', ['worktree', 'prune'], { cwd: this.d.repo }); + gitSpawnSync(['worktree', 'remove', '--force', dir], { cwd: this.d.repo }); + gitSpawnSync(['worktree', 'prune'], { cwd: this.d.repo }); }; clean(); try { diff --git a/src/coordinator/test-runs.ts b/src/coordinator/test-runs.ts index 665a48a00..0372eaa35 100644 --- a/src/coordinator/test-runs.ts +++ b/src/coordinator/test-runs.ts @@ -3,7 +3,7 @@ // new run, and the superseded one is cancelled. Full tiers run per batch of approved PRs (PR mode, Track PB): the // harness's, and people's PRs from this listing (merge-flow.ts batchPeoplesPrs): a mapped admin's or writer's once its // fast checks pass; a bot's or an unmapped author's only once an admin approved it. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../os/git.js'; import { BRAND } from '../brand.js'; import { changedNames } from './change-files.js'; import type { EventLog } from '../events/log.js'; @@ -154,7 +154,7 @@ export class PrTestRuns { private fetchPr(pr: OpenPr): string { if (this.d.fetchPr) return this.d.fetchPr(pr); const ref = `refs/${BRAND.cli}/test-run/${pr.number}`; - const git = (...a: string[]) => execFileSync('git', ['-C', this.d.repo, ...a], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 180_000 }).trim(); + const git = (...a: string[]) => gitExecFileSync(['-C', this.d.repo, ...a], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 180_000 }).trim(); git('fetch', '-q', this.d.remote, `+refs/pull/${pr.number}/head:${ref}`, `+refs/heads/${pr.base}:refs/remotes/${this.d.remote}/${pr.base}`); return git('rev-parse', `${ref}^{commit}`); } diff --git a/src/dashboard.ts b/src/dashboard.ts index 318aff4d1..897ac22af 100644 --- a/src/dashboard.ts +++ b/src/dashboard.ts @@ -14,6 +14,7 @@ import { PINS_FILE } from './experiments/store.js'; import { handleHook, hooksGatewayToken } from './automations/webhook.js'; import { AUTOMATIONS_FILE, AutomationsPolicy, limitsInForce } from './automations/schema.js'; import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from './os/git.js'; import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http'; import { createHash, randomBytes, timingSafeEqual } from 'node:crypto'; import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; @@ -1259,7 +1260,7 @@ export function startDashboard(opts: DashboardOptions): Promise<{ server: Server }; /** The test files the repo's runner can name (the first-task form's picker): tracked files that look like tests. */ const testFiles = (): string[] => { - const r = spawnSync('git', ['ls-files'], { cwd: opts.root, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); + const r = gitSpawnSync(['ls-files'], { cwd: opts.root, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); if (r.status !== 0) return []; return r.stdout.split('\n').filter((f) => /(^|\/)(tests?|__tests__|spec)\/.+\.[a-z]+$|\.(test|spec)\.[cm]?[jt]sx?$|_test\.(go|py|rb)$|(^|\/)test_[^/]+\.py$/.test(f)).slice(0, 300); }; @@ -2246,7 +2247,7 @@ export function startDashboard(opts: DashboardOptions): Promise<{ server: Server if (url.pathname === '/api/first-run/checks-seen' && req.method === 'GET') { if (!isWriter(who.role)) return json(res, 403, { error: writersOnly('the first-run checklist is', who) }); const branch = opts.cfg.project.project.default_branch; - const sha = spawnSync('git', ['rev-parse', `origin/${branch}`], { cwd: opts.root, encoding: 'utf8' }).stdout.trim(); + const sha = gitSpawnSync(['rev-parse', `origin/${branch}`], { cwd: opts.root, encoding: 'utf8' }).stdout.trim(); if (!/^[0-9a-f]{40}$/.test(sha)) return json(res, 200, { checks: [], error: `origin/${branch} isn't fetched here` }); const gh = actionDeps().gh; const runs = gh(['api', `repos/${repo}/commits/${sha}/check-runs`, '--jq', '.check_runs[].name']); diff --git a/src/decision-detail.ts b/src/decision-detail.ts index a4e87a1c2..180639bab 100644 --- a/src/decision-detail.ts +++ b/src/decision-detail.ts @@ -9,7 +9,7 @@ // - a viewer gets only the fields VIEWER_DECISION_FIELDS allows (src/viewer-events.ts), never the context; // - a writer gets the context too, as the agent runs it comes from (mayReadRuns), minus the owner-only parts; // - the owners also get the owner-only parts: raw check output, the tamper guard's flags, and the diff. -import { execFile } from 'node:child_process'; +import { gitExecFile } from './os/git.js'; import { BRAND } from './brand.js'; import { DEFAULT_REVIEW } from './coordinator/shared.js'; import { isOwner, isWriter, mayAnswerKind, type Role } from './dashboard-auth.js'; @@ -348,9 +348,14 @@ export function decisionDetail(id: string, d: DetailDeps): DecisionDetail | null export async function decisionDiff(root: string, base: string, head: string): Promise<{ diff: string; truncated: boolean } | { diff: null; why: string }> { if (!SHA.test(base) || !SHA.test(head)) return { diff: null, why: 'no commits recorded for this change' }; const MAX = 512 * 1024; + // Every file's lines whatever the change's attributes say (#807): a file its .gitattributes marks `-diff` or + // `binary` (or gives a textconv or an external diff) shows as it is, not as "Binary files differ"; the user's + // attributes file is not read. Output past the buffer is shown cut, like output past MAX. + const args = ['-c', 'core.attributesFile=/dev/null', 'diff', '--no-color', '--text', '--no-textconv', '--no-ext-diff', '--ignore-submodules=none', `${base}..${head}`]; return new Promise((done) => { - execFile('git', ['diff', '--no-color', '--no-ext-diff', `${base}..${head}`], { cwd: root, env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: 'utf8' }, (err, stdout, stderr) => { - if (err) return done({ diff: null, why: `not in this checkout's git: ${(stderr || err.message).trim().split('\n')[0]}` }); + gitExecFile(args, { cwd: root, env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: 'utf8' }, (err, stdout, stderr) => { + if (err && !(err.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' && stdout)) return done({ diff: null, why: `not in this checkout's git: ${(stderr || err.message).trim().split('\n')[0]}` }); + if (err) return done({ diff: redactString(stdout.slice(0, MAX)), truncated: true }); const truncated = stdout.length > MAX; done({ diff: redactString(truncated ? stdout.slice(0, MAX) : stdout), truncated }); }); diff --git a/src/demo.ts b/src/demo.ts index f879215dd..ec1813f57 100644 --- a/src/demo.ts +++ b/src/demo.ts @@ -1,7 +1,8 @@ // A self-contained demo: a copy of the example project with a local "GitHub" // remote and a file backlog, worked by the real coordinator with scripted // agents, so the event log (and the dashboard) shows every state for real. -import { execFileSync, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; +import { gitExecFileSync } from './os/git.js'; import { cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -17,7 +18,7 @@ import { childEnv } from './os/index.js'; import { DEFAULT_COMMIT_IDENTITY, FakeRunner, type RunRequest } from './runner.js'; import { writeSlotFile } from './slot-file.js'; -const git = (cwd: string, ...a: string[]) => execFileSync('git', a, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const git = (cwd: string, ...a: string[]) => gitExecFileSync(a, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); function commitAll(cwd: string, msg: string) { git(cwd, 'add', '-A'); diff --git a/src/deploy-gate/wire.ts b/src/deploy-gate/wire.ts index f6131a573..153de383c 100644 --- a/src/deploy-gate/wire.ts +++ b/src/deploy-gate/wire.ts @@ -1,6 +1,6 @@ // The deploy gate's real git and GitHub calls, and setting it up at coordinator start (only with tests.yaml // deploy_gate, and only once the instance's test runs are set up: the gate runs its tiers through them). -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { existsSync, readFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { join } from 'node:path'; @@ -21,7 +21,7 @@ type Fetch = typeof fetch; /** The gate's git over the coordinator's clone: both branches fetched (the deploy branch only when it exists). */ export function gateGit(repo: string, remote: string, main: string, deploy: string): GateGit { - const git = (...a: string[]) => spawnSync('git', a, { cwd: repo, encoding: 'utf8', timeout: 180_000, maxBuffer: 64 * 1024 * 1024 }); + const git = (...a: string[]) => gitSpawnSync(a, { cwd: repo, encoding: 'utf8', timeout: 180_000, maxBuffer: 64 * 1024 * 1024 }); return { fetch: () => { const ls = git('ls-remote', remote, `refs/heads/${deploy}`); diff --git a/src/direct-land.ts b/src/direct-land.ts index cad0aacd7..3f2225082 100644 --- a/src/direct-land.ts +++ b/src/direct-land.ts @@ -5,7 +5,7 @@ // and refuses to start unless GitHub enforces the branch's protection; a check that errors refuses too; // - every direct land asks again just before its push, and pushes nothing unless the answer is "enforced". // A remote that is a local path (a bare repo on disk: the demo, the tests) has no protection to ask about: allowed. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from './os/git.js'; import { existsSync } from 'node:fs'; import type { EventLog } from './events/log.js'; import type { EventPayload, StoredEvent } from './events/types.js'; @@ -15,7 +15,7 @@ export type Protection = Pick; /** Whether `remote` in `repo` is a path on this machine (or file://), not a host that could protect a branch. */ export function localRemote(repo: string, remote: string): boolean { - const r = spawnSync('git', ['remote', 'get-url', remote], { cwd: repo, encoding: 'utf8' }); + const r = gitSpawnSync(['remote', 'get-url', remote], { cwd: repo, encoding: 'utf8' }); const url = r.status === 0 ? r.stdout.trim() : ''; if (!url) return false; return url.startsWith('file://') || existsSync(url); diff --git a/src/doctor.ts b/src/doctor.ts index c6881d3b3..3ec810f40 100644 --- a/src/doctor.ts +++ b/src/doctor.ts @@ -3,6 +3,7 @@ // that can't run reports why, and a feature the setup's mode can't have is // listed as off, with why. import { execFileSync, spawnSync } from 'node:child_process'; +import { gitExecFileSync } from './os/git.js'; import { existsSync, lstatSync, readFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; import { agentRecordWarning } from './setup-steps.js'; @@ -210,7 +211,7 @@ export function doctor(rootArg: string, opts: { agentshield?: boolean; /** A pol if (!gl) add('gitleaks', 'fail', `not on PATH or in ${toolBin()}: commit and transcript scans cannot run`); else add('gitleaks', 'ok', execFileSync(gl, ['version'], { encoding: 'utf8' }).trim()); try { - const common = execFileSync('git', ['rev-parse', '--git-common-dir'], { cwd: root, encoding: 'utf8' }).trim(); + const common = gitExecFileSync(['rev-parse', '--git-common-dir'], { cwd: root, encoding: 'utf8' }).trim(); const hook = join(resolve(root, common), 'hooks', 'pre-commit'); if (existsSync(hook) && readFileSync(hook, 'utf8').includes(GIT_HOOK_MARKER)) add('pre-commit', 'ok', 'gitleaks pre-commit installed'); else add('pre-commit', 'warn', `not installed (\`${BRAND.cli} install --git-hooks\`); the land queue still re-scans`); diff --git a/src/experiments/view.ts b/src/experiments/view.ts index 706bd8799..9c45440b5 100644 --- a/src/experiments/view.ts +++ b/src/experiments/view.ts @@ -3,7 +3,7 @@ // Who may do what is here too, so the page and the tests share one rule: owners start, resume, approve, decline and // archive; writers (and owners) pause and stop a running job, the safe direction; viewers nothing (403, no nav item). import { createHash } from 'node:crypto'; -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../os/git.js'; import type { Role } from '../dashboard-auth.js'; import type { EventLog } from '../events/log.js'; import type { EventPayload, StoredEvent } from '../events/types.js'; @@ -51,7 +51,7 @@ export const noExperimentsHere = (instance: string | undefined) => const show = (repo: string) => (commit: string, path: string) => { try { - return execFileSync('git', ['show', `${commit}:${path}`], { cwd: repo, stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 64 * 1024 * 1024 }); + return gitExecFileSync(['show', `${commit}:${path}`], { cwd: repo, stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 64 * 1024 * 1024 }); } catch { return null; } @@ -67,7 +67,7 @@ const diffOf = (repo: string, commit: string | null): string[] => { }; const tipOf = (site: ExpSite) => { try { - return execFileSync('git', ['rev-parse', `origin/${site.branch}`], { cwd: site.repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); + return gitExecFileSync(['rev-parse', `origin/${site.branch}`], { cwd: site.repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); } catch { return null; } diff --git a/src/harness-evals/replay.ts b/src/harness-evals/replay.ts index 803720e2b..c004a7c0d 100644 --- a/src/harness-evals/replay.ts +++ b/src/harness-evals/replay.ts @@ -4,7 +4,7 @@ // scratch clone is deleted when its replay ends. The engine is the one running the replay (the harness under test); // the task's project config is its own at the base, except the harness config taken from the harness under test // (HARNESS_CONFIG_PATHS), so a change to that config is what the replay measures. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../os/git.js'; import { randomBytes } from 'node:crypto'; import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; @@ -56,7 +56,7 @@ export class BudgetRunner implements AgentRunner { } } -const git = (cwd: string, ...a: string[]) => execFileSync('git', a, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 600_000 }).trim(); +const git = (cwd: string, ...a: string[]) => gitExecFileSync(a, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 600_000 }).trim(); /** The engine's own identity on the scratch config commit, as an agent session commits. */ function commit(cwd: string, msg: string) { diff --git a/src/harness-evals/schedule.ts b/src/harness-evals/schedule.ts index 1b2b92bc3..8a3d9e44c 100644 --- a/src/harness-evals/schedule.ts +++ b/src/harness-evals/schedule.ts @@ -1,7 +1,7 @@ // When the harness eval suite runs: on this machine, from the nightly window (tests.yaml nightly_at), at most once a // day; weekly, and when the engine running here changed src/ since the last run. Which engine is running comes from // the engine itself: its git commit, or the commit its install directory is named after (/opt//). -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { existsSync, realpathSync } from 'node:fs'; import { basename, join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -15,7 +15,7 @@ const SHA = /^[0-9a-f]{40}$/; /** The commit of the engine running this code, or null when it can't tell. */ export function engineSha(root: string = ENGINE_ROOT): string | null { if (existsSync(join(root, '.git'))) { - const r = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); + const r = gitSpawnSync(['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); const sha = r.status === 0 ? r.stdout.trim() : ''; if (SHA.test(sha)) return sha; } @@ -31,7 +31,7 @@ export function engineSha(root: string = ENGINE_ROOT): string | null { export function srcChanged(from: string, to: string, clones: string[]): boolean | null { for (const cwd of clones) { if (!existsSync(join(cwd, '.git'))) continue; - const r = spawnSync('git', ['diff', '--quiet', from, to, '--', 'src/'], { cwd, stdio: 'ignore' }); + const r = gitSpawnSync(['diff', '--quiet', from, to, '--', 'src/'], { cwd, stdio: 'ignore' }); if (r.status === 0) return false; if (r.status === 1) return true; } diff --git a/src/init.ts b/src/init.ts index 376a17373..915f49b43 100644 --- a/src/init.ts +++ b/src/init.ts @@ -12,7 +12,8 @@ // agent's Claude token and the checkout). Init never installs tools there: that stays the owner's step. // // What local mode can't do is listed plainly (unavailableIn), in doctor and on the dashboard, never left to fail. -import { spawn, spawnSync } from 'node:child_process'; +import { spawn } from 'node:child_process'; +import { commandSpawnSync } from './os/git.js'; import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; import { delimiter, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -653,7 +654,8 @@ export function systemIo(o: { interactive: boolean }): InitIo { return { which: (cmd) => findTool(cmd), run: (cmd, args, opt = {}) => { - const r = spawnSync(cmd, args, { encoding: 'utf8', timeout: 60_000, env: probeEnv(opt), ...(opt.cwd ? { cwd: opt.cwd } : {}) }); + // A caller's git (the command named git) through the git runner (#807). + const r = commandSpawnSync(cmd, args, { encoding: 'utf8', timeout: 60_000, env: probeEnv(opt), ...(opt.cwd ? { cwd: opt.cwd } : {}) }); return { status: r.error ? null : r.status, stdout: r.stdout ?? '', stderr: r.error ? r.error.message : (r.stderr ?? '') }; }, serviceManagerEnv: () => serviceManagerEnv(MANAGER_LOGIN_VARS), diff --git a/src/install.ts b/src/install.ts index e10a1dd43..30fb6c581 100644 --- a/src/install.ts +++ b/src/install.ts @@ -6,7 +6,7 @@ // checkout's .claude/settings.json (keeping the project's own entries), for // local runs without an instance. Re-running is safe: our entries are // recognized by a marker and replaced, never duplicated. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './os/git.js'; import { chmodSync, cpSync, existsSync, mkdirSync, readFileSync, readlinkSync, renameSync, writeFileSync } from 'node:fs'; import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -201,7 +201,7 @@ function readJson(path: string): Settings { export function gitRemoteRepo(root: string): string | null { try { // WHY trim: git ends its answer with a newline; the shared parser (src/managed-apps.ts) takes a URL as given. - return githubRepoOf(execFileSync('git', ['remote', 'get-url', 'origin'], { cwd: root, encoding: 'utf8' }).trim()); + return githubRepoOf(gitExecFileSync(['remote', 'get-url', 'origin'], { cwd: root, encoding: 'utf8' }).trim()); } catch { return null; } @@ -345,14 +345,14 @@ export function installGitHooks(root: string, cfg: Config, engineCli = fileURLTo * `branches` (the config's, when there is one) and whatever origin/HEAD names at push time. */ export function installCheckoutPrePush(root: string, branches: string[], engineCli?: string, node: string = stableNodePath().path): string { - const common = execFileSync('git', ['rev-parse', '--git-common-dir'], { cwd: root, encoding: 'utf8' }).trim(); + const common = gitExecFileSync(['rev-parse', '--git-common-dir'], { cwd: root, encoding: 'utf8' }).trim(); // WHY (#468): the checkout's hook runs node by Homebrew's stable bin/node (or this node's own path), as the service does. return installPrePushHook(join(resolve(root, common), 'hooks'), { branches, node, ...(engineCli ? { engine: engineCli } : {}) }); } /** gitleaks pre-commit hook in the repo's common hooks dir, chaining any existing hook. */ export function installGitHook(root: string): string { - const common = execFileSync('git', ['rev-parse', '--git-common-dir'], { cwd: root, encoding: 'utf8' }).trim(); + const common = gitExecFileSync(['rev-parse', '--git-common-dir'], { cwd: root, encoding: 'utf8' }).trim(); const hooksDir = join(resolve(root, common), 'hooks'); mkdirSync(hooksDir, { recursive: true }); const hook = join(hooksDir, 'pre-commit'); diff --git a/src/main-watch.ts b/src/main-watch.ts index 7b74e62e2..1d15b945e 100644 --- a/src/main-watch.ts +++ b/src/main-watch.ts @@ -31,7 +31,7 @@ // The required_checks a commit is judged by are the ones in force when it landed: config.yaml as committed at its // first parent, read with git. Not the config loaded at startup: the clone's working tree isn't updated as the default // branch moves, so a check dropped since would still be required of every later merge, and never report. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from './os/git.js'; import { createHash, randomBytes } from 'node:crypto'; import { existsSync, mkdirSync, readFileSync, renameSync } from 'node:fs'; import { dirname, join } from 'node:path'; @@ -615,7 +615,7 @@ export function subjectPrNumber(subject: string): number | null { } export function repoGit(repo: string, remote: string, branch: string): WatchGit { - const git = (...a: string[]) => spawnSync('git', a, { cwd: repo, encoding: 'utf8' }); + const git = (...a: string[]) => gitSpawnSync(a,{ cwd: repo, encoding: 'utf8' }); return { tip: () => { git('fetch', '-q', remote, `+refs/heads/${branch}:refs/remotes/${remote}/${branch}`); @@ -936,7 +936,7 @@ export interface AcceptGit extends WatchGit { /** AcceptGit over a clone (the instance's repo): the tip is fetched as the watch fetches it. */ export function acceptGit(repo: string, remote: string, branch: string): AcceptGit { - const git = (...a: string[]) => spawnSync('git', a, { cwd: repo, encoding: 'utf8' }); + const git = (...a: string[]) => gitSpawnSync(a,{ cwd: repo, encoding: 'utf8' }); return { ...repoGit(repo, remote, branch), resolve: (ref) => { diff --git a/src/merge-base.ts b/src/merge-base.ts index fdb7b4abd..77c009a20 100644 --- a/src/merge-base.ts +++ b/src/merge-base.ts @@ -2,13 +2,13 @@ // coordinator owns: merge a base into the checked-out branch and report what // conflicted. Merge commits only here (the default strategy); conflict // markers use diff3 style so a resolver also sees the common ancestor. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from './os/git.js'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; import { changedNames, changedNamesOrNull, presentNames } from './coordinator/change-files.js'; import { isNonUtf8Path } from './review.js'; -const git = (cwd: string, ...a: string[]) => spawnSync('git', a, { cwd, encoding: 'utf8' }); +const git = (cwd: string, ...a: string[]) => gitSpawnSync(a, { cwd, encoding: 'utf8' }); /** A blob's id at `rev:path` (or the index's, `:path`), or null when there is none. */ const blob = (cwd: string, spec: string) => { const r = git(cwd, 'rev-parse', '--verify', '-q', spec); diff --git a/src/merge-policy-weakening.ts b/src/merge-policy-weakening.ts index 28723fd9e..8653ff82d 100644 --- a/src/merge-policy-weakening.ts +++ b/src/merge-policy-weakening.ts @@ -60,6 +60,9 @@ export const CHECK_TABLES: readonly CheckTable[] = [ { path: 'test/raw-check-reads.ts', table: 'JUDGED_CASTS_ALLOWED', kind: 'never-grow' }, { path: 'test/raw-check-reads.ts', table: 'MAKERS_ALLOWED', kind: 'never-grow' }, { path: 'test/owner-terminal-rerun.test.ts', table: 'FIRST_TIME_ONLY', kind: 'never-grow' }, + // The git runner scan's exceptions (#807): a file let start git outside the runner, and the runners it skips. + { path: 'test/git-runner.test.ts', table: 'OUTSIDE_THE_RUNNER', kind: 'never-grow' }, + { path: 'test/git-runner.test.ts', table: 'RUNNERS', kind: 'never-grow' }, { path: 'scripts/lib/denylist.mjs', table: 'SKIP', kind: 'never-grow' }, { path: 'src/guardrails/engine.ts', table: 'AGENT_EDITABLE', kind: 'never-grow' }, // The lints' rules and bad fixtures, and the guards' lists: taking one out is a loosening. @@ -124,6 +127,7 @@ export const LINT_TESTS: readonly string[] = [ 'test/web-buttons.test.ts', 'test/superfan.test.ts', 'test/claude-accounts-retired.test.ts', + 'test/git-runner.test.ts', ]; /** diff --git a/src/nightly.ts b/src/nightly.ts index 16978337d..cf116eaf9 100644 --- a/src/nightly.ts +++ b/src/nightly.ts @@ -2,7 +2,7 @@ // baseline) plus any extra nightly tiers (e.g. mutation testing), queued // behind the machine-wide full-run lock, the project's idle probe and the // load gate, so they never overlap another full run. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './os/git.js'; import { fileURLToPath } from 'node:url'; import { BRAND } from './brand.js'; import type { Config } from './config/load.js'; @@ -17,8 +17,8 @@ export function queueBaselineRun(root: string, cfg: Config, eventsDb: string, ac const fmt = cfg.tests.failures; if (!fmt) throw new Error('tests.yaml needs a failures: { section, item } format to record a baseline'); const branch = cfg.project.project.default_branch; - execFileSync('git', ['fetch', '-q', 'origin', branch], { cwd: root }); - const sha = execFileSync('git', ['rev-parse', `origin/${branch}`], { cwd: root, encoding: 'utf8' }).trim(); + gitExecFileSync(['fetch', '-q', 'origin', branch], { cwd: root }); + const sha = gitExecFileSync(['rev-parse', `origin/${branch}`], { cwd: root, encoding: 'utf8' }).trim(); const state = projectStateDir(root); const name = `baseline-${sha.slice(0, 8)}`; const wt = { repo: root, root: cfg.tests.worktree.root, stateDir: state, setup: cfg.tests.worktree.setup, ...(runAs ? { runAs } : {}) }; diff --git a/src/os/git.ts b/src/os/git.ts new file mode 100644 index 000000000..a5dfb01f2 --- /dev/null +++ b/src/os/git.ts @@ -0,0 +1,112 @@ +// The one way the engine runs git (#807). +// +// WHY: refs/replace/* and $GIT_DIR/info/grafts live in the repository, and every worktree of a repo shares them. An +// agent in its worktree can write either, and then every git the engine runs on that repo reads the agent's stand-in +// instead of the object asked for: a `git replace` of a pushed commit hid a refused file from the push limits (1 +// problem became 0, measured on #802's head), and a graft can give a commit other parents, so a range reads as other +// commits. Both are on by default in git. +// +// INVARIANT: every git process the engine starts goes through a function here, and each one runs with replace refs +// off (`--no-replace-objects` before the subcommand, and GIT_NO_REPLACE_OBJECTS=1, which git passes to every git it +// runs in turn) and with no graft file (GIT_GRAFT_FILE names a path that can't exist, so $GIT_DIR/info/grafts is +// never read; `--no-replace-objects` alone still honours grafts, measured on git 2.43). The settings go on last, over +// whatever environment the caller passes, so no caller can turn them off. test/git-runner.test.ts fails on a git call +// in src/ or scripts/ that doesn't come through here (or scripts/lib/git.cjs, the same settings for scripts). +// +// LIMIT: a shallow clone's `.git/shallow` is still read (it is how a shallow clone works, and git writes it itself). +// A program the engine runs that runs git itself (gitleaks) gets these settings only when it is handed gitEnv(). +import { execFile, execFileSync, spawn, spawnSync } from 'node:child_process'; +import type { + ChildProcess, + ChildProcessByStdio, + ExecFileException, + ExecFileOptions, + ExecFileOptionsWithStringEncoding, + ExecFileSyncOptions, + ExecFileSyncOptionsWithBufferEncoding, + ExecFileSyncOptionsWithStringEncoding, + SpawnOptions, + SpawnOptionsWithStdioTuple, + SpawnSyncOptions, + SpawnSyncOptionsWithBufferEncoding, + SpawnSyncOptionsWithStringEncoding, + SpawnSyncReturns, + StdioNull, + StdioPipe, +} from 'node:child_process'; +import { devNull } from 'node:os'; +import { join } from 'node:path'; +import type { Readable } from 'node:stream'; + +/** Before every git subcommand: replace refs off for this git (git also sets GIT_NO_REPLACE_OBJECTS for the gits it runs). */ +export const GIT_FIXED_ARGS: readonly string[] = Object.freeze(['--no-replace-objects']); + +/** + * On every git process, over the caller's environment. GIT_GRAFT_FILE is a path under the null device, which can't + * exist (ENOTDIR on Linux and macOS), so git reads no graft file and says nothing about it. + */ +export const GIT_FIXED_ENV: Readonly> = Object.freeze({ + GIT_NO_REPLACE_OBJECTS: '1', + GIT_GRAFT_FILE: join(devNull, 'no-grafts'), +}); + +/** `env` (or this process's environment), with the fixed settings on top. */ +export function gitEnv(env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return { ...(env ?? process.env), ...GIT_FIXED_ENV }; +} + +/** The argument list a git process gets: the fixed arguments, then `args`. */ +export function gitArgs(args: readonly string[]): string[] { + return [...GIT_FIXED_ARGS, ...args]; +} + +/** True for a command that runs git: `git`, or a path to it (`/usr/bin/git`, `C:\...\git.exe`), in any case. */ +export function isGitCommand(cmd: string): boolean { + const name = (cmd.split(/[\\/]/).pop() ?? '').toLowerCase(); + return name === 'git' || name === 'git.exe'; +} + +/** spawnSync of git. */ +export function gitSpawnSync(args: readonly string[], options: SpawnSyncOptionsWithStringEncoding): SpawnSyncReturns; +export function gitSpawnSync(args: readonly string[], options?: SpawnSyncOptionsWithBufferEncoding): SpawnSyncReturns; +export function gitSpawnSync(args: readonly string[], options?: SpawnSyncOptions): SpawnSyncReturns; +export function gitSpawnSync(args: readonly string[], options: SpawnSyncOptions = {}): SpawnSyncReturns { + return spawnSync('git', gitArgs(args), { ...options, env: gitEnv(options.env) }); +} + +/** execFileSync of git. */ +export function gitExecFileSync(args: readonly string[], options: ExecFileSyncOptionsWithStringEncoding): string; +export function gitExecFileSync(args: readonly string[], options?: ExecFileSyncOptionsWithBufferEncoding): Buffer; +export function gitExecFileSync(args: readonly string[], options?: ExecFileSyncOptions): string | Buffer; +export function gitExecFileSync(args: readonly string[], options: ExecFileSyncOptions = {}): string | Buffer { + return execFileSync('git', gitArgs(args), { ...options, env: gitEnv(options.env) }); +} + +/** spawn of git. */ +export function gitSpawn(args: readonly string[], options: SpawnOptionsWithStdioTuple): ChildProcessByStdio; +export function gitSpawn(args: readonly string[], options: SpawnOptionsWithStdioTuple): ChildProcessByStdio; +export function gitSpawn(args: readonly string[], options: SpawnOptions): ChildProcess; +export function gitSpawn(args: readonly string[], options: SpawnOptions): ChildProcess { + return spawn('git', gitArgs(args), { ...options, env: gitEnv(options.env) }); +} + +/** execFile of git, its output as text. */ +export function gitExecFile( + args: readonly string[], + options: ExecFileOptionsWithStringEncoding, + callback: (error: ExecFileException | null, stdout: string, stderr: string) => void, +): ChildProcess { + const o: ExecFileOptions = { ...options, env: gitEnv(options.env) }; + return execFile('git', gitArgs(args), o, (error, stdout, stderr) => callback(error, String(stdout), String(stderr))); +} + +/** + * spawnSync of a command a caller names at run time (a probe's or a stand-in-able runner's `run(cmd, args)`): git, + * by name or path, through gitSpawnSync, so a runner that takes any command can't run git without the settings; + * anything else as it is. + */ +export function commandSpawnSync(cmd: string, args: readonly string[], options: SpawnSyncOptionsWithStringEncoding): SpawnSyncReturns { + return isGitCommand(cmd) + ? spawnSync(cmd, gitArgs(args), { ...options, env: gitEnv(options.env) }) + : spawnSync(cmd, args, options); +} diff --git a/src/plugins/fetch.ts b/src/plugins/fetch.ts index a30cec228..d2297f2be 100644 --- a/src/plugins/fetch.ts +++ b/src/plugins/fetch.ts @@ -4,6 +4,7 @@ // DynamicUser) is the root side's job (scripts/setup/plugins.sh); this module is the part that is the same wherever it // runs, and it never executes a file it fetched. import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../os/git.js'; import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -43,7 +44,7 @@ export function gitSafety(allowFile: boolean, pin?: string): string[] { return ['-c', 'http.followRedirects=false', ...(pin ? ['-c', `http.curloptResolve=${pin}`] : []), '-c', 'core.hooksPath=/dev/null', '-c', 'protocol.allow=never', '-c', 'protocol.https.allow=always', ...(allowFile ? ['-c', 'protocol.file.allow=always'] : []), '-c', 'submodule.recurse=false', '-c', 'core.symlinks=true', '-c', 'advice.detachedHead=false']; } function git(cwd: string, home: string, args: string[], allowFile: boolean, pin?: string): string { - return execFileSync('git', [...gitSafety(allowFile, pin), ...args], { + return gitExecFileSync([...gitSafety(allowFile, pin), ...args], { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], diff --git a/src/push-check.ts b/src/push-check.ts index 358923757..b523a49f8 100644 --- a/src/push-check.ts +++ b/src/push-check.ts @@ -5,9 +5,9 @@ // leaves a SKILL.md that fails the skill lint. // The coordinator also runs pushProblems on each change before verifying it, // so a doomed change is caught before a long test run. -import { execFileSync, spawnSync } from 'node:child_process'; import { sizeFiles } from './coordinator/change-files.js'; import { globToRegExp } from './guardrails/glob.js'; +import { gitExecFileSync, gitSpawnSync } from './os/git.js'; import { shownPath } from './path-text.js'; import { lintSkill, SKILL_SECTIONS } from './skill-lint.js'; import { defaultBranchRefusal, type PushGuard } from './push-guard.js'; @@ -23,7 +23,7 @@ export const DEFAULT_PUSH_LIMITS: PushLimits = { max_file_mb: 10, max_changed_li const WORKFLOWS = '.github/workflows/'; const MB = 1024 * 1024; -const git = (cwd: string, args: string[], input?: string) => execFileSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * MB, stdio: ['pipe', 'pipe', 'pipe'], ...(input !== undefined ? { input } : {}) }); +const git = (cwd: string, args: string[], input?: string) => gitExecFileSync(args, { cwd, encoding: 'utf8', maxBuffer: 256 * MB, stdio: ['pipe', 'pipe', 'pipe'], ...(input !== undefined ? { input } : {}) }); const list = (paths: string[], n = 10) => `${paths.slice(0, n).join(', ')}${paths.length > n ? ` and ${paths.length - n} more` : ''}`; /** Changed paths in a problem's text, each as shownPath shows it (#683): `rev-list --objects` prints a name as it is. */ @@ -69,7 +69,7 @@ export function pushProblems(cwd: string, base: string, head: string, limits: Pu // Skills the change adds or edits must pass the skill lint as they stand at head. const badSkills: string[] = []; for (const p of touched.filter((t) => t === 'SKILL.md' || t.endsWith('/SKILL.md'))) { - const r = spawnSync('git', ['show', `${head}:${p}`], { cwd, encoding: 'utf8', maxBuffer: 16 * MB }); + const r = gitSpawnSync(['show',`${head}:${p}`], { cwd, encoding: 'utf8', maxBuffer: 16 * MB }); if (r.status !== 0) continue; // deleted by the change const lint = lintSkill(r.stdout); if (lint.length) badSkills.push(`${shownPath(p)} (${lint.join('; ')})`); @@ -94,7 +94,7 @@ export function checkedPush(o: { cwd: string; remote: string; base: string; head const refused = pushProblems(o.cwd, o.base, o.head, o.limits); if (refused.length) return { ok: false, refused }; const lease = o.expect ? [`--force-with-lease=${o.ref}:${o.expect}`] : []; - const push = spawnSync('git', ['push', ...lease, o.remote, `${o.force ? '+' : ''}${o.head}:${o.ref}`], { cwd: o.cwd, encoding: 'utf8' }); + const push = gitSpawnSync(['push',...lease, o.remote, `${o.force ? '+' : ''}${o.head}:${o.ref}`], { cwd: o.cwd, encoding: 'utf8' }); if (push.status !== 0) return { ok: false, error: (push.stderr || '').trim() }; return { ok: true }; } diff --git a/src/review.ts b/src/review.ts index 80f64b414..b6f75039f 100644 --- a/src/review.ts +++ b/src/review.ts @@ -1,7 +1,7 @@ // Risk-based review levels, computed by code from what a change touches. // The highest matching level wins; a failed or low-confidence evaluator // verdict bumps it up one; an agent can raise its own level, never lower it. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from './os/git.js'; import { posix } from 'node:path'; import type { ReviewConfig } from './config/schema.js'; import { globToRegExp } from './guardrails/glob.js'; @@ -326,6 +326,9 @@ export const SELF_MODIFICATION_PARTS: { what: string; globs: string[] }[] = [ { what: 'the merge policy', globs: ['src/merge-policy*', 'src/coordinator/merge-flow*', 'src/pr-rules*'] }, { what: 'the main watch', globs: ['src/main-watch*', 'src/github/main-watch-api*'] }, { what: 'the push guard', globs: ['src/push-guard*', 'src/push-check*'] }, + // The one way the engine and its scripts run git, with replace refs and grafts off (#807): every reader above reads + // through it. + { what: 'the git runner', globs: ['src/os/git.ts', 'scripts/lib/git.cjs'] }, { what: 'the updater', globs: ['src/updates*', `scripts/machine/${BRAND.cli}-update*`] }, // src/path-text.ts holds the odd-name tests computeLevel raises on (moved out of src/review.ts, #683), so it stays locked with it. { what: 'the review levels', globs: [`${BRAND.configDir}/review.yaml`, 'src/review.ts', 'src/path-text.ts'] }, @@ -386,6 +389,8 @@ export const SELF_MODIFICATION_PARTS: { what: string; globs: string[] }[] = [ 'test/managed-apps.test.ts', // The merge reconciliation's cases: only the engine's own App's merge is reconciled, never another's (#733). 'test/merge-idempotent.test.ts', + // The replace-ref and graft routes, and the scan that every git call goes through the git runner (#807). + 'test/git-runner.test.ts', ], }, { what: 'the release procedure and the design rules', globs: ['docs/releasing.md', 'docs/design.md'] }, @@ -515,7 +520,7 @@ export interface LevelResult { export function loadMoneyPaths(repo: string, src: ReviewConfig['money_path_source'], rev: string): RegExp[] { if (!src) return []; const file = posix.normalize(src.file).replace(/^\.\//, ''); - const shown = spawnSync('git', ['show', `${rev}:${file}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 16 * 1024 * 1024 }); + const shown = gitSpawnSync(['show',`${rev}:${file}`], { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 16 * 1024 * 1024 }); if (shown.status !== 0) throw new Error(`money_path_source: ${file} isn't readable at ${rev}: ${(shown.stderr || shown.error?.message || '').trim().split('\n')[0]}`); let text = shown.stdout; if (src.pattern) { diff --git a/src/scan/secrets.ts b/src/scan/secrets.ts index 80f0e8d99..cf30aca79 100644 --- a/src/scan/secrets.ts +++ b/src/scan/secrets.ts @@ -4,6 +4,7 @@ import { spawn, spawnSync } from 'node:child_process'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { gitEnv } from '../os/git.js'; import { findTool } from '../tool-path.js'; export interface Finding { @@ -49,7 +50,7 @@ export function scanCommit(cwd: string, includeUnstaged: boolean, binary = findT if (!binary) return { status: 'unavailable', error: 'gitleaks not found on PATH' }; const findings: Finding[] = []; for (const args of includeUnstaged ? [['--staged'], []] : [['--staged']]) { - const r = spawnSync(binary, ['git', '--pre-commit', ...args, '--redact', '--no-banner', '--exit-code', String(LEAKS_EXIT), '.'], { cwd, encoding: 'utf8', timeout: 120_000 }); + const r = spawnSync(binary, ['git', '--pre-commit', ...args, '--redact', '--no-banner', '--exit-code', String(LEAKS_EXIT), '.'], { cwd, encoding: 'utf8', timeout: 120_000, env: gitEnv() }); if (r.error) return { status: 'unavailable', error: r.error.message }; if (r.status === LEAKS_EXIT) findings.push({ rule: 'see gitleaks output', file: '(staged changes)', line: 0 }); else if (r.status !== 0) return { status: 'unavailable', error: `gitleaks exited ${r.status}` }; @@ -78,7 +79,9 @@ function scanAsync(binary: string, args: (report: string) => string[], cwd: stri const dir = mkdtempSync(join(tmpdir(), 'scan-')); const report = join(dir, 'report.json'); return new Promise((resolve) => { - const child = spawn(binary, args(report), { ...(cwd ? { cwd } : {}), stdio: ['ignore', 'ignore', 'pipe'] }); + // gitleaks runs git itself (`git log -p`): with the git runner's settings, so a replace ref or a graft can't swap + // the commits it reads (#807). + const child = spawn(binary, args(report), { ...(cwd ? { cwd } : {}), stdio: ['ignore', 'ignore', 'pipe'], env: gitEnv() }); let err = ''; child.stderr.on('data', (d: Buffer) => (err = (err + d.toString()).slice(-2000))); const done = (r: ScanResult) => { diff --git a/src/server-auto-config.ts b/src/server-auto-config.ts index 30f21c29d..bf2114802 100644 --- a/src/server-auto-config.ts +++ b/src/server-auto-config.ts @@ -8,7 +8,7 @@ // doesn't stop the coordinator; // - there is no terminal: open questions are recorded, and answered with `instance auto-config --answer`. // Its state is the coordinator's own (auto-config.json in the instance's state dir); each run prints one JSON line. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from './os/git.js'; import { existsSync, mkdirSync, readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { parseDocument } from 'yaml'; @@ -143,7 +143,7 @@ export async function serverAutoConfig(name: string, o: { answers?: Record { - const p = spawnSync('git', args, { cwd, encoding: 'utf8', env: { ...process.env, ...identity, GIT_TERMINAL_PROMPT: '0', [`${BRAND.envPrefix}_INSTANCE`]: name }, timeout: 300_000 }); + const p = gitSpawnSync(args, { cwd, encoding: 'utf8', env: { ...process.env, ...identity, GIT_TERMINAL_PROMPT: '0', [`${BRAND.envPrefix}_INSTANCE`]: name }, timeout: 300_000 }); return { status: p.error ? null : p.status, stdout: p.stdout ?? '', stderr: p.error ? p.error.message : (p.stderr ?? '') }; }, gh: (args) => gh(args), diff --git a/src/setup-actions.ts b/src/setup-actions.ts index 3268cd4ef..dbcc3bed1 100644 --- a/src/setup-actions.ts +++ b/src/setup-actions.ts @@ -1,7 +1,8 @@ // WHY: the first-run page fixes some steps itself (Fix kind 'action'). Each action shows its plan before anything // happens and runs the CLI's own code for the step; the dashboard runs it for owners only, confirmed and audited. import { join } from 'node:path'; -import { execFileSync, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; +import { commandSpawnSync, gitExecFileSync } from './os/git.js'; import { BRAND } from './brand.js'; import type { Config } from './config/load.js'; import { refreshFingerprints, type RefreshResult } from './guardrails/context.js'; @@ -54,7 +55,7 @@ export function systemActionDeps(): ActionDeps { } const gitCommonHooks = (root: string) => { - const common = execFileSync('git', ['rev-parse', '--git-common-dir'], { cwd: root, encoding: 'utf8' }).trim(); + const common = gitExecFileSync(['rev-parse', '--git-common-dir'], { cwd: root, encoding: 'utf8' }).trim(); return join(common.startsWith('/') ? common : join(root, common), 'hooks'); }; @@ -147,7 +148,8 @@ export const ACTIONS: Record = { }), run: async (c) => { for (const cmd of defaultBranchCommands(c)) { - const r = spawnSync(cmd[0]!, cmd.slice(1), { cwd: c.root, encoding: 'utf8', timeout: 120_000, env: withToolPath() }); + // The git commands below (defaultBranchCommands) through the git runner (#807). + const r = commandSpawnSync(cmd[0]!, cmd.slice(1), { cwd: c.root, encoding: 'utf8', timeout: 120_000, env: withToolPath() }); if (r.status !== 0) return { ok: false, detail: `${cmd.join(' ')} failed: ${(r.stderr || r.error?.message || '').trim().split('\n')[0]}` }; } return { ok: true, detail: `fetched origin/${c.cfg.project.project.default_branch}; origin/HEAD set` }; diff --git a/src/setup-steps.ts b/src/setup-steps.ts index 38b38cbca..97dff0279 100644 --- a/src/setup-steps.ts +++ b/src/setup-steps.ts @@ -4,7 +4,7 @@ // and it checks them with these detectors), `doctor` (one line per step) and the dashboard's first-run page // (GET /api/first-run). A detector only looks: it runs read-only commands (`claude auth status`, `gh auth status`, // `git rev-parse`, the service manager's status) and reads files; it never changes anything. -import { spawnSync } from 'node:child_process'; +import { commandSpawnSync } from './os/git.js'; import { Worker } from 'node:worker_threads'; import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; @@ -96,7 +96,8 @@ export function systemProbe(o: { managerEnv?: ManagerEnv } = {}): SetupProbe { return { which: (cmd) => findTool(cmd), run: (cmd, args, o = {}) => { - const r = spawnSync(cmd, args, { encoding: 'utf8', timeout: 30_000, env: probeEnv(o), ...(o.cwd ? { cwd: o.cwd } : {}) }); + // A caller's git (the command named git) through the git runner (#807). + const r = commandSpawnSync(cmd, args, { encoding: 'utf8', timeout: 30_000, env: probeEnv(o), ...(o.cwd ? { cwd: o.cwd } : {}) }); return { status: r.error ? null : r.status, stdout: r.stdout ?? '', stderr: r.error ? r.error.message : (r.stderr ?? '') }; }, serviceRunning, diff --git a/src/skill-lint.ts b/src/skill-lint.ts index 4f9f83b34..1ee3c9f53 100644 --- a/src/skill-lint.ts +++ b/src/skill-lint.ts @@ -1,6 +1,6 @@ // Skill lint: every SKILL.md has the six sections a skill needs to be used and checked, each with something in it. // Validate (how to check the result) matters most: a skill whose result can't be checked is a guess. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './os/git.js'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; @@ -38,7 +38,7 @@ export function lintSkill(text: string): string[] { /** Every tracked SKILL.md in a git checkout (forward-slash paths, relative to `root`). */ export function trackedSkills(root: string): string[] { - return execFileSync('git', ['ls-files', '-z', '--', '*SKILL.md'], { cwd: root, encoding: 'utf8' }) + return gitExecFileSync(['ls-files', '-z', '--', '*SKILL.md'], { cwd: root, encoding: 'utf8' }) .split('\0') .filter((p) => p === 'SKILL.md' || p.endsWith('/SKILL.md')); } diff --git a/src/starter-ci.ts b/src/starter-ci.ts index ef04009ab..2932b9b82 100644 --- a/src/starter-ci.ts +++ b/src/starter-ci.ts @@ -7,7 +7,7 @@ // design (src/setup/app-manifest.ts, scripts/setup/app-permissions.mjs); this only drafts text and builds a link. // Once the workflow ran on the default branch, its check is offered for required_checks as an ordinary config // change the person reviews and proposes (src/config-change.ts), never written here. -import { spawnSync } from 'node:child_process'; +import { commandSpawnSync } from './os/git.js'; import { BRAND } from './brand.js'; import { detectProject, NO_TESTS, type Detection } from './auto-config.js'; import { GITHUB_WEB } from './github-hosts.js'; @@ -166,7 +166,8 @@ export interface StarterCiView extends StarterCi { type Run = (cmd: string, args: string[], cwd: string) => { status: number | null; stdout: string; stderr: string }; const realRun: Run = (cmd, args, cwd) => { - const r = spawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: 60_000 }); + // A caller's git (the command named git) through the git runner (#807). + const r = commandSpawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: 60_000 }); return { status: r.error ? null : r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' }; }; diff --git a/src/testrun/cache.ts b/src/testrun/cache.ts index 2f492df87..1ee17062d 100644 --- a/src/testrun/cache.ts +++ b/src/testrun/cache.ts @@ -1,7 +1,7 @@ // Cache keys: the sha256 of the files that pin what a cache holds (the lockfile), read at the run's commit from the // coordinator's clone. A changed lockfile is a new key and so a new, empty cache, never a stale one; a key file the // commit doesn't have is part of the key too ("missing"), so adding one later changes it. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../os/git.js'; import { createHash } from 'node:crypto'; import type { TestRunCache } from './config.js'; @@ -10,7 +10,7 @@ export function cacheKey(repo: string, sha: string, files: readonly string[]): s for (const f of files) { h.update(`${f}\0`); try { - h.update(execFileSync('git', ['-C', repo, 'show', `${sha}:${f}`], { maxBuffer: 256 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] })); + h.update(gitExecFileSync(['-C', repo, 'show', `${sha}:${f}`], { maxBuffer: 256 * 1024 * 1024, stdio: ['ignore', 'pipe', 'ignore'] })); } catch { h.update('\0missing\0'); } diff --git a/src/testrun/host.ts b/src/testrun/host.ts index ad40d69b3..743e691a8 100644 --- a/src/testrun/host.ts +++ b/src/testrun/host.ts @@ -13,6 +13,7 @@ import type { Readable } from 'node:stream'; import { BRAND } from '../brand.js'; import { killTree, shellCommand, spawnDetached } from '../os/index.js'; import { ownEnv } from '../os/own-env.js'; +import { gitArgs, gitEnv } from '../os/git.js'; import { SPARSE_MAX, sparseDirProblem } from '../checkout.js'; import { MAX_CONNECTIONS, PINNED_IMAGE } from './config.js'; import { HELPER_PROTOCOL } from './helper.js'; @@ -543,7 +544,8 @@ export function localHost(base: string, o: { docker?: string; path?: string } = const w = await run('sh', ['-c', 'cat > "$1"', '_', file], { stdin: bundle, timeoutMs: 600_000 }); if (w.code !== 0) throw fail('create', `receiving run ${id}'s bundle`, w); const [bash, args] = shellCommand(cloneScript()); - const env = { PATH: o.path ?? ownEnv().get('PATH') ?? '/usr/bin:/bin', HOME: join(dir, 'home'), LANG: 'C.UTF-8' }; + // The clone script runs git: with the git runner's settings, as every engine git does (#807). + const env = gitEnv({ PATH: o.path ?? ownEnv().get('PATH') ?? '/usr/bin:/bin', HOME: join(dir, 'home'), LANG: 'C.UTF-8' }); mkdirSync(join(base, 'store'), { recursive: true, mode: 0o700 }); const r = await run(bash, [...args, '_', file, join(dir, 'work'), refs.branch, refs.head, refs.base, refs.headSha, refs.baseSha, join(base, 'store'), ...refs.exclude], { env, cwd: dir, timeoutMs: 600_000 }); if (r.code !== 0) throw fail('create', `creating run ${id}'s workspace`, r); @@ -551,7 +553,7 @@ export function localHost(base: string, o: { docker?: string; path?: string } = }, async storeTips() { if (!existsSync(join(base, 'store', 'HEAD'))) return []; - const r = await run('git', [`--git-dir=${join(base, 'store')}`, 'for-each-ref', '--format=%(objectname)', 'refs/keep/'], { env: { PATH: o.path ?? ownEnv().get('PATH') ?? '/usr/bin:/bin', GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' }, timeoutMs: 60_000 }); + const r = await run('git', gitArgs([`--git-dir=${join(base, 'store')}`, 'for-each-ref', '--format=%(objectname)', 'refs/keep/']), { env: gitEnv({ PATH: o.path ?? ownEnv().get('PATH') ?? '/usr/bin:/bin', GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' }), timeoutMs: 60_000 }); if (r.code !== 0) throw fail('store-tips', 'reading the object store', r); return parseTips(r.stdout); }, diff --git a/src/testrun/measure-cli.ts b/src/testrun/measure-cli.ts index eb48159e8..254de94ca 100644 --- a/src/testrun/measure-cli.ts +++ b/src/testrun/measure-cli.ts @@ -3,7 +3,7 @@ // INVARIANT: the host is always the sandbox host (no flag picks another; only a test passes MeasureIo.host), and no // checks dependency, token or App key is loaded here: a measure can't post a check. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from '../os/git.js'; import { join } from 'node:path'; import { BRAND } from '../brand.js'; import { EventLog } from '../events/log.js'; @@ -68,7 +68,7 @@ export async function measureCommand(instance: string | undefined, sub: string | const given = args.sha ?? defaultBranch(i.repo.path)?.sha ?? ''; plan = planMeasure(config, { sha: given, tiers: args.tiers, sets: args.sets }); try { - plan.sha = execFileSync('git', ['-C', i.repo.path, 'rev-parse', '--verify', '-q', `${plan.sha}^{commit}`], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); + plan.sha = gitExecFileSync(['-C', i.repo.path, 'rev-parse', '--verify', '-q', `${plan.sha}^{commit}`], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); } catch { throw new MeasureRefused(`${plan.sha} is not a commit in ${i.repo.path}; nothing was started`); } diff --git a/src/testrun/predates.ts b/src/testrun/predates.ts index 4efbf6a0a..7d014551e 100644 --- a/src/testrun/predates.ts +++ b/src/testrun/predates.ts @@ -7,7 +7,7 @@ // Only then is anything here called a predates; an up-to-date head that lacks a script deleted it itself. // Every git question is asked of the coordinator's clone (the head and the default branch are fetched there), and // any that can't be answered counts as "no": this never turns a run into a predates on a guess. -import { spawnSync } from 'node:child_process'; +import { gitSpawnSync } from '../os/git.js'; import { parse } from 'yaml'; import { BRAND } from '../brand.js'; import { defaultBranch } from './workspace.js'; @@ -25,7 +25,7 @@ export interface Predates { } function git(repo: string, ...args: string[]): { ok: boolean; out: string } { - const r = spawnSync('git', ['-C', repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 60_000, maxBuffer: 16 * 1024 * 1024 }); + const r = gitSpawnSync(['-C', repo, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 60_000, maxBuffer: 16 * 1024 * 1024 }); return { ok: r.status === 0, out: r.stdout ?? '' }; } @@ -71,7 +71,7 @@ export function configBehind(repo: string, sha: string): string | null { if (!base) return null; const c = configCommit(repo, base.sha); if (!c) return null; - const r = spawnSync('git', ['-C', repo, 'merge-base', '--is-ancestor', c, sha], { stdio: 'ignore', timeout: 60_000 }); + const r = gitSpawnSync(['-C', repo, 'merge-base', '--is-ancestor', c, sha], { stdio: 'ignore', timeout: 60_000 }); // 1: not an ancestor. Anything else (0, or an error) is no evidence of an old head. return r.status === 1 ? c : null; } diff --git a/src/testrun/select.ts b/src/testrun/select.ts index 1bd997df8..af449666b 100644 --- a/src/testrun/select.ts +++ b/src/testrun/select.ts @@ -7,9 +7,9 @@ // // The graph is read from the coordinator's clone at the run's commit (git ls-tree, git cat-file), never from the // workspace: the sandboxed test user owns that. -import { spawnSync } from 'node:child_process'; import { builtinModules } from 'node:module'; import { BRAND } from '../brand.js'; +import { gitSpawnSync } from '../os/git.js'; import { buildGraph, CODE_EXT, selectTests, type Plan } from './affected.js'; import type { TestRunConfig } from './config.js'; @@ -52,7 +52,7 @@ function packageCheck(pkgJson: string | null): (spec: string) => boolean { }; } -const git = (repo: string, args: string[], input?: string) => spawnSync('git', ['-C', repo, ...args], { input, maxBuffer: 512 * 1024 * 1024, timeout: 120_000 }); +const git = (repo: string, args: string[], input?: string) => gitSpawnSync(['-C', repo, ...args], { input, maxBuffer: 512 * 1024 * 1024, timeout: 120_000 }); /** Each file's contents at `sha`, through one git cat-file --batch. A file over 1 MB reads as empty (built output). */ export function readAt(repo: string, sha: string, files: readonly string[]): Map { diff --git a/src/testrun/workspace.ts b/src/testrun/workspace.ts index 5e56e3da0..1919f58e1 100644 --- a/src/testrun/workspace.ts +++ b/src/testrun/workspace.ts @@ -13,13 +13,13 @@ // // Directories the project leaves out (tests.yaml checkout.sparse_exclude) are left out of the workspace too (sparse // checkout), unless a tier of the run sets full_checkout. -import { execFileSync, spawn } from 'node:child_process'; import { Readable } from 'node:stream'; import { BRAND } from '../brand.js'; +import { gitExecFileSync, gitSpawn } from '../os/git.js'; import { safeProjectEnv } from '../project-env.js'; import type { RunDatabase, RunHost } from './host.js'; -const git = (repo: string, ...a: string[]) => execFileSync('git', ['-C', repo, ...a], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 120_000 }).trim(); +const git = (repo: string, ...a: string[]) => gitExecFileSync(['-C', repo, ...a], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 120_000 }).trim(); /** The default branch's name and commit in the coordinator's clone: origin/HEAD, else origin/main, else main. */ export function defaultBranch(repo: string): { name: string; sha: string } | null { @@ -44,7 +44,7 @@ export function defaultBranch(repo: string): { name: string; sha: string } | nul export function sharedTips(repo: string, tips: readonly string[]): string[] { const wanted = [...new Set(tips.filter((t) => /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/.test(t)))]; if (!wanted.length) return []; - const out = execFileSync('git', ['-C', repo, 'cat-file', '--batch-check=%(objectname) %(objecttype)'], { input: wanted.map((t) => `${t}\n`).join(''), encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 120_000 }); + const out = gitExecFileSync(['-C', repo, 'cat-file', '--batch-check=%(objectname) %(objecttype)'], { input: wanted.map((t) => `${t}\n`).join(''), encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'], timeout: 120_000 }); return out .split('\n') .map((l) => l.split(' ')) @@ -83,7 +83,7 @@ export function treeBundle(repo: string, sha: string, id: string, have: readonly const refs = [...(lacks(repo, full, have) ? [[head, full] as const] : []), ...(lacks(repo, base.sha, have) ? [[baseRef, base.sha] as const] : [])]; if (!refs.length) return { stream: Readable.from([]), branch: base.name, headSha: full, baseSha: base.sha, refs: 0, done: Promise.resolve() }; for (const [r, c] of refs) git(repo, 'update-ref', r, c); - const child = spawn('git', ['-C', repo, 'bundle', 'create', '--quiet', '-', ...refs.map(([r]) => r), ...(have.length ? ['--not', ...have] : [])], { stdio: ['ignore', 'pipe', 'pipe'] }); + const child = gitSpawn(['-C', repo, 'bundle', 'create', '--quiet', '-', ...refs.map(([r]) => r), ...(have.length ? ['--not', ...have] : [])], { stdio: ['ignore', 'pipe', 'pipe'] }); let err = ''; child.stderr.on('data', (d: Buffer) => (err += d.toString())); const cleanup = () => { diff --git a/src/tune.ts b/src/tune.ts index 0c7b98b63..68c506552 100644 --- a/src/tune.ts +++ b/src/tune.ts @@ -1,7 +1,7 @@ // `tune`: profile a project's test suite on this machine (a fresh worktree on main, set up as the agent user; the // suite at 1, 2, 4 and all cores' workers; time and peak worker memory) and propose tests.yaml's `cores.max` and // `{cores}` as a PR, under the usual push limits. Nothing changes until it is reviewed and merged. -import { execFileSync, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; import { existsSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { freemem } from 'node:os'; import { join } from 'node:path'; @@ -12,6 +12,7 @@ import type { TestsConfig } from './config/schema.js'; import { CORES_PLACEHOLDER, envWithCores, withCores } from './cores.js'; import { cpuCount, peakMemoryMb, projectCommand, withPeakMemory, writeGroupOnly, type ProjectRunAs } from './os/index.js'; import { machineStats } from './os/stats.js'; +import { gitExecFileSync } from './os/git.js'; import { checkedPush, type PushLimits } from './push-check.js'; import { recommendWorkers, WORKER_VAR, type ProfileRun, type TuneProfile } from './tuning.js'; import { createWorktree, removeWorktree, type WorktreeOptions } from './worktrees.js'; @@ -126,7 +127,7 @@ export async function tune(o: TuneOptions): Promise { const after = applyTuning(before, rec.workers, setting); if (after === before) return { profile, recommendation: rec, changed: false }; writeFileSync(file, after); - const git = (...a: string[]) => execFileSync('git', a, { cwd: path, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + const git = (...a: string[]) => gitExecFileSync(a, { cwd: path, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); git('add', '--', `${BRAND.configDir}/tests.yaml`); git('-c', `user.name=${o.identity.name}`, '-c', `user.email=${o.identity.email}`, 'commit', '-q', '-m', `Test workers: at most ${rec.workers}, from a profile on this machine`, '-m', rec.why); const head = git('rev-parse', 'HEAD'); diff --git a/src/worktree-pool.ts b/src/worktree-pool.ts index 628a7a30f..25328e8f5 100644 --- a/src/worktree-pool.ts +++ b/src/worktree-pool.ts @@ -7,7 +7,7 @@ // When main moves and the files that pin the install (worktree.setup_key) didn't change, a pooled worktree // follows main by a checkout in place, keeping what its setup installed, so a merge doesn't empty the pool. It is // still set up on main only and handed to one task, never back: no task's change reaches another's dependencies. -import { execFileSync } from 'node:child_process'; +import { gitExecFileSync } from './os/git.js'; import { createHash } from 'node:crypto'; import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; @@ -29,7 +29,7 @@ export interface PoolEntry { export const POOL_MIN_FREE_GB = 10; const poolFile = (o: WorktreeOptions) => join(o.stateDir, 'worktree-pool.json'); -const git = (cwd: string, ...args: string[]) => execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const git = (cwd: string, ...args: string[]) => gitExecFileSync(args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); export function readPool(o: WorktreeOptions): PoolEntry[] { try { diff --git a/src/worktrees.ts b/src/worktrees.ts index bed8cc2ec..ba71ba7d6 100644 --- a/src/worktrees.ts +++ b/src/worktrees.ts @@ -1,7 +1,8 @@ // Worktrees the coordinator creates and owns. It never lists, touches or // removes a worktree it didn't create: names carry our prefix AND must be in // our ownership record, so other sessions' worktrees are always safe. -import { execFileSync, spawn, spawnSync } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; +import { gitExecFileSync, gitSpawnSync } from './os/git.js'; import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, rmdirSync, unlinkSync, writeFileSync } from 'node:fs'; import { join, resolve } from 'node:path'; import { BRAND } from './brand.js'; @@ -38,7 +39,7 @@ function setOwned(o: WorktreeOptions, list: string[]) { writeGroupOnly(ownedFile(o), JSON.stringify([...new Set(list)], null, 2)); } -const git = (cwd: string, ...args: string[]) => execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const git = (cwd: string, ...args: string[]) => gitExecFileSync(args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); /** * What Claude Code's sandbox leaves in a worktree: on Linux it protects paths by mounting over them, so a missing one @@ -167,8 +168,8 @@ export function createWorktree(o: WorktreeOptions, name: string, branch: string, export function removeWorktree(o: WorktreeOptions, name: string): boolean { const path = worktreePath(o, name); if (!owned(o).includes(path)) throw new Error(`refusing to remove ${path}: not created by ${BRAND.cli}`); - spawnSync('git', ['worktree', 'remove', '--force', path], { cwd: o.repo, encoding: 'utf8' }); - spawnSync('git', ['worktree', 'prune'], { cwd: o.repo }); + gitSpawnSync(['worktree', 'remove', '--force', path], { cwd: o.repo, encoding: 'utf8' }); + gitSpawnSync(['worktree', 'prune'], { cwd: o.repo }); const gone = !existsSync(path) && !git(o.repo, 'worktree', 'list', '--porcelain').includes(`worktree ${path}\n`); if (gone) setOwned(o, owned(o).filter((p) => p !== path)); return gone; diff --git a/test/git-runner.test.ts b/test/git-runner.test.ts new file mode 100644 index 000000000..352a8904e --- /dev/null +++ b/test/git-runner.test.ts @@ -0,0 +1,287 @@ +// One git runner for every git call the engine makes (#807). refs/replace/* and $GIT_DIR/info/grafts are in the +// repository, which every worktree of it shares, so an agent can write either; every git the engine runs there then +// reads the agent's stand-in. A `git replace` of the pushed commit hid a refused file from the push limits (1 problem +// became 0), and a graft that makes the base reach the head empties the range. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { devNull, tmpdir } from 'node:os'; +import { join, relative } from 'node:path'; +import { ReviewConfig } from '../src/config/schema.js'; +import { DEFAULT_PUSH_LIMITS, pushProblems, type PushLimits } from '../src/push-check.js'; +import { computeLevel, SELF_MODIFICATION, selfModificationPart } from '../src/review.js'; +import { findTool } from '../src/tool-path.js'; +import { repoRoot } from './helpers.js'; + +const git = (cwd: string, ...a: string[]) => execFileSync('git', a, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const limits = (over: Partial = {}): PushLimits => ({ ...DEFAULT_PUSH_LIMITS, ...over }); + +/** A repo with one commit (base) and a change on it (head) that adds a refused file and a big one. */ +function repo() { + const work = mkdtempSync(join(tmpdir(), 'git-runner-')); + git(work, 'init', '-q', '-b', 'main'); + for (const [k, v] of [['user.email', 'bot@example.com'], ['user.name', 'bot'], ['commit.gpgsign', 'false']]) git(work, 'config', k!, v!); + writeFileSync(join(work, 'README.md'), 'hello\n'); + git(work, 'add', 'README.md'); + git(work, 'commit', '-q', '-m', 'init'); + const base = git(work, 'rev-parse', 'HEAD'); + mkdirSync(join(work, 'keys')); + writeFileSync(join(work, 'keys', 'secret.pem'), 'not a real key\n'); + writeFileSync(join(work, 'weights.bin'), 'x'.repeat(4096)); + git(work, 'add', '.'); + git(work, 'commit', '-q', '-m', 'add a key and a big file'); + return { work, base, head: git(work, 'rev-parse', 'HEAD') }; +} + +const RULES = limits({ refuse_paths: ['**/*.pem'], max_file_mb: 0.002 }); + +test('a replace ref on the pushed commit doesn\'t hide a refused file or a big one from the push limits', () => { + const r = repo(); + const before = pushProblems(r.work, r.base, r.head, RULES); + assert.equal(before.length, 2, before.join('\n')); + // The decoy: the same parent, base's tree (no key, no big file). + const decoy = git(r.work, 'commit-tree', `${r.base}^{tree}`, '-p', r.base, '-m', 'docs'); + git(r.work, 'replace', r.head, decoy); + // git itself now reads the decoy. + assert.equal(git(r.work, 'log', '--format=', '--name-only', `${r.base}..${r.head}`), ''); + const after = pushProblems(r.work, r.base, r.head, RULES); + assert.equal(after.length, 2, after.join('\n')); + assert.match(after.join('\n'), /keys\/secret\.pem/); + assert.match(after.join('\n'), /weights\.bin/); +}); + +test('a graft that makes the base reach the head doesn\'t empty the range the push limits read', () => { + const r = repo(); + // $GIT_DIR/info/grafts: the base commit now has the head as a parent, so base..head is empty to git. + const grafts = join(r.work, '.git', 'info', 'grafts'); + writeFileSync(grafts, `${r.base} ${r.head}\n`); + assert.equal(execFileSync('git', ['rev-list', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }), ''); + const after = pushProblems(r.work, r.base, r.head, RULES); + assert.equal(after.length, 2, after.join('\n')); + assert.match(after.join('\n'), /keys\/secret\.pem/); + assert.match(after.join('\n'), /weights\.bin/); +}); + +test('gitleaks, which runs git itself, gets the settings: a replace ref on the leaky commit doesn\'t hide its secret from the change scan', { skip: !findTool('gitleaks') && 'gitleaks not installed' }, async () => { + const { scanRange } = await import('../src/scan/secrets.js'); + const r = repo(); + writeFileSync(join(r.work, 'c.js'), `export const token = '${'ghp_' + 'a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8'}';\n`); + git(r.work, 'add', 'c.js'); + git(r.work, 'commit', '-q', '-m', 'leaky change'); + const leaky = git(r.work, 'rev-parse', 'HEAD'); + assert.equal((await scanRange(r.work, `${r.head}..${leaky}`)).status, 'leaks'); + const decoy = git(r.work, 'commit-tree', `${r.head}^{tree}`, '-p', r.head, '-m', 'docs'); + git(r.work, 'replace', leaky, decoy); + assert.equal((await scanRange(r.work, `${r.head}..${leaky}`)).status, 'leaks'); +}); + +test('the push gate script (scripts/push-gate.mjs), which runs alone, scans the real commits under a replace ref', { skip: !findTool('gitleaks') && 'gitleaks not installed' }, () => { + const r = repo(); + const hash = (t: string) => createHash('sha256').update(t).digest('hex'); + const real = JSON.parse(readFileSync(join(repoRoot, '.denylist.json'), 'utf8')) as { patterns: string[] }; + writeFileSync(join(r.work, '.denylist.json'), JSON.stringify({ hashed: [hash('secretproject')], patterns: real.patterns })); + git(r.work, 'add', '.denylist.json'); + git(r.work, 'commit', '-q', '-m', 'rules'); + const base = git(r.work, 'rev-parse', 'HEAD'); + writeFileSync(join(r.work, 'notes.md'), 'deployed by secretproject\n'); + git(r.work, 'add', 'notes.md'); + git(r.work, 'commit', '-q', '-m', 'add notes'); + const head = git(r.work, 'rev-parse', 'HEAD'); + const gate = () => spawnSync(process.execPath, [join(repoRoot, 'scripts', 'push-gate.mjs'), '--range', `${base}..${head}`], { cwd: r.work, encoding: 'utf8' }); + const before = gate(); + assert.notEqual(before.status, 0, 'refused before the replace'); + assert.match(before.stderr, /notes\.md/); + const decoy = git(r.work, 'commit-tree', `${base}^{tree}`, '-p', base, '-m', 'add notes'); + git(r.work, 'replace', head, decoy); + const after = gate(); + assert.notEqual(after.status, 0, after.stderr); + assert.match(after.stderr, /REFUSED[\s\S]*notes\.md/); +}); + +// ---- The runner itself. + +test('the runner\'s settings go on last: a caller\'s environment can\'t turn replace refs or grafts back on', async () => { + const { GIT_FIXED_ENV, gitEnv, gitExecFileSync, gitSpawnSync } = await import('../src/os/git.js'); + const r = repo(); + // A graft file of the caller's choosing (the base reaches the head): the runner's GIT_GRAFT_FILE wins. + const grafts = join(r.work, 'grafts'); + writeFileSync(grafts, `${r.base} ${r.head}\n`); + const env = { ...process.env, GIT_GRAFT_FILE: grafts }; + assert.equal(execFileSync('git', ['rev-list', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env, stdio: ['ignore', 'pipe', 'ignore'] }), ''); + const names = (out: string) => out.split('\n').filter(Boolean).sort(); + const want = ['keys/secret.pem', 'weights.bin']; + assert.deepEqual(names(gitExecFileSync(['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env })), want); + assert.deepEqual(names(gitSpawnSync(['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env }).stdout), want); + assert.deepEqual({ ...gitEnv({ GIT_GRAFT_FILE: grafts, GIT_NO_REPLACE_OBJECTS: '0' }) }, { ...GIT_FIXED_ENV }); + // Replace refs: `--no-replace-objects` is in the argument list, which no environment takes back. + const decoy = git(r.work, 'commit-tree', `${r.base}^{tree}`, '-p', r.base, '-m', 'docs'); + git(r.work, 'replace', r.head, decoy); + assert.deepEqual(names(gitExecFileSync(['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env: {} })), want); +}); + +test('a runner that takes any command (a probe\'s run) gives git the settings, by name or path, and leaves others as they are', { skip: process.platform === 'win32' && 'sh' }, async () => { + const { commandSpawnSync, isGitCommand } = await import('../src/os/git.js'); + const r = repo(); + const decoy = git(r.work, 'commit-tree', `${r.base}^{tree}`, '-p', r.base, '-m', 'docs'); + git(r.work, 'replace', r.head, decoy); + const gitPath = execFileSync('sh', ['-c', 'command -v git'], { encoding: 'utf8' }).trim(); + for (const cmd of ['git', gitPath]) { + const out = commandSpawnSync(cmd, ['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8' }).stdout; + assert.match(out, /keys\/secret\.pem/, cmd); + } + assert.ok(isGitCommand('git') && isGitCommand('/usr/bin/git') && isGitCommand('C:\\Git\\cmd\\git.exe') && isGitCommand('GIT')); + assert.ok(!isGitCommand('gitleaks') && !isGitCommand('/usr/bin/legit') && !isGitCommand('gh')); + const argv = commandSpawnSync(process.execPath, ['-e', 'console.log(JSON.stringify(process.argv.slice(1)))', 'a'], { encoding: 'utf8' }); + assert.deepEqual(JSON.parse(argv.stdout), ['a']); +}); + +test('scripts/lib/git.cjs (scripts run without a build) has the engine runner\'s settings, and applies them last', async () => { + const engine = await import('../src/os/git.js'); + const scripts = createRequire(import.meta.url)(join(repoRoot, 'scripts', 'lib', 'git.cjs')) as typeof engine; + assert.deepEqual([...scripts.GIT_FIXED_ARGS], [...engine.GIT_FIXED_ARGS]); + assert.deepEqual({ ...scripts.GIT_FIXED_ENV }, { ...engine.GIT_FIXED_ENV }); + // What the files that run on their own write out (ITS_OWN, below) is these values. + assert.deepEqual([...engine.GIT_FIXED_ARGS], ['--no-replace-objects']); + assert.deepEqual({ ...engine.GIT_FIXED_ENV }, { GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); + const r = repo(); + const decoy = git(r.work, 'commit-tree', `${r.base}^{tree}`, '-p', r.base, '-m', 'docs'); + git(r.work, 'replace', r.head, decoy); + writeFileSync(join(r.work, '.git', 'info', 'grafts'), `${r.base} ${r.head}\n`); + const env = { ...process.env, GIT_GRAFT_FILE: join(r.work, '.git', 'info', 'grafts') }; + assert.match(scripts.gitExecFileSync(['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env }), /keys\/secret\.pem/); + assert.match(scripts.gitSpawnSync(['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env }).stdout, /keys\/secret\.pem/); + assert.match(scripts.commandSpawnSync('git', ['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8' }).stdout, /keys\/secret\.pem/); +}); + +// ---- No git call in src/ or scripts/ bypasses the runner. + +/** + * Where a source text starts git itself: `git` (or a path to it, or git.exe) as a quoted string in an argument or array + * position or assigned to a name (`spawnSync('git', …)`, `run('git', …)`, `['git', 'fetch']`, `const bin = 'git'`), or + * a shell line that starts with git (`exec('git log')`, `['-c', 'git log']`). A comparison (`=== 'git'`) or an object + * key's value (`type: 'git'`) isn't one. A backstop next to the runner, not a proof: a name built at run time + * (`'gi' + 't'`) gets past it. + */ +function gitStarts(text: string): number { + const patterns = [ + /(?:[([,]|(?])=)\s*(['"`])(?:[^'"`\s]*[\\/])?git(?:\.exe)?\1/gi, + /\b(?:exec|execSync)\s*\(\s*(['"`])\s*git\s/g, + /(['"`])-c\1\s*,\s*(['"`])\s*git\s/g, + ]; + return patterns.reduce((n, re) => n + (text.match(re)?.length ?? 0), 0); +} + +const RUNNERS = new Set(['src/os/git.ts', 'scripts/lib/git.cjs']); + +/** + * A file that runs on its own (copied alone, or fetched alone from origin) can't import a runner: it holds the + * settings itself, written out exactly so, and the test below ties them to the runner's values. + */ +const ITS_OWN = [/const GIT_FIXED_ARGS = (?:Object\.freeze\()?\['--no-replace-objects'\]/, /GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join\(devNull, 'no-grafts'\)/]; +/** A runner that takes any command (`run(cmd, args)`), and so gives git the settings by its name. */ +const BY_NAME = (fn: string, call: string) => [new RegExp(`${fn}\\(${call}`)]; + +/** + * Files that start git outside the runners, each with how many times (exactly: one more is a new call) and why. Each + * is a call through a runner that takes any command and gives git the settings by its name (commandSpawnSync), a + * file that runs on its own and holds the settings itself, or text that names git without starting it. + */ +const OUTSIDE_THE_RUNNER: Record = { + // Through a runner that takes any command: its real one spawns through commandSpawnSync (src/os/git.ts). + 'src/init.ts': { n: 6, why: "`c.io.run('git', ...)` and `io.run('git', ...)`: systemIo().run, here, spawns through commandSpawnSync; and `manual.includes('git')`, a hint's text", must: BY_NAME('commandSpawnSync', 'cmd, args, ') }, + 'src/setup-steps.ts': { n: 7, why: "`k.p.run('git', ...)`: systemProbe().run, here, spawns through commandSpawnSync; and REQUIRED_TOOLS, two lists of tool names", must: BY_NAME('commandSpawnSync', 'cmd, args, ') }, + 'src/setup-mode.ts': { n: 3, why: "originPlan's two commands and `base.run('git', ...)`: each run by an InitIo's run, systemIo() in src/init.ts (commandSpawnSync)" }, + 'src/setup-actions.ts': { n: 3, why: "defaultBranchCommands' two commands, run here through commandSpawnSync; and `manual.includes('git')`, a hint's text", must: BY_NAME('commandSpawnSync', 'cmd\\[0\\]!, cmd\\.slice\\(1\\), ') }, + 'src/starter-ci.ts': { n: 3, why: "`run('git', ...)`: realRun, here, spawns through commandSpawnSync", must: BY_NAME('commandSpawnSync', 'cmd, args, ') }, + 'src/testrun/host.ts': { n: 1, why: "`run('git', gitArgs(...), { env: gitEnv(...) })`: the async runner, given the settings by hand", must: [/run\('git', gitArgs\(\[/, /env: gitEnv\(\{ PATH/] }, + // Text that names git without starting it. + 'src/scan/secrets.ts': { n: 2, why: "gitleaks' `git` subcommand; gitleaks runs git itself, so it gets gitEnv()", must: [/'--exit-code', String\(LEAKS_EXIT\), '\.'\], \{ cwd, encoding: 'utf8', timeout: 120_000, env: gitEnv\(\) \}/, /stdio: \['ignore', 'ignore', 'pipe'\], env: gitEnv\(\) \}/] }, + 'src/review.ts': { n: 2, why: "`'git'`: the name of a way to read a .gitattributes line" }, + 'src/plugins/approval.ts': { n: 1, why: "`z.literal('git')`: a plugin source's type" }, + 'src/updater-setup.ts': { n: 1, why: 'a comment quoting an allowed-signers line (namespaces="git")' }, + 'src/os/index.ts': { n: 2, why: "`which('git')` and the `Git` folder: where Git for Windows' bash is, never run as git" }, + 'scripts/verify-release-tag.mjs': { n: 2, why: "ssh-keygen's `-n git`: the signature namespace" }, + // Runs on its own, with the settings written out. + 'scripts/push-gate-hook.mjs': { n: 1, why: 'runGit: setup-hooks copies this file alone into the hooks dir; it hands the gate the settings in its environment too', must: [...ITS_OWN, /\{ \.\.\.process\.env, GIT_DIR: gitDir, \.\.\.GIT_FIXED_ENV \}/] }, + 'scripts/push-gate.mjs': { n: 2, why: "runGit, and gitleaks' `git` subcommand (with GIT_ENV): the hook runs this file alone, fetched from origin (GATE_CODE)", must: [...ITS_OWN, /\{ encoding: 'utf8', env: GIT_ENV \}/] }, + 'scripts/ci/new-deps.mjs': { n: 1, why: "runGit: CI runs the base's copy of this file alone, from a temp dir (new-deps.yml)", must: ITS_OWN }, + 'scripts/ci/testrun-real-systemd.mjs': { n: 1, why: 'its git: the job copies this file alone into the VM (ci-slow.yml)', must: [/const GIT_FIXED_ARGS = \['--no-replace-objects'\]/, /GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join\(devNull, 'no-grafts'\)/, /execFileSync\('git', \[\.\.\.GIT_FIXED_ARGS, /] }, + 'scripts/machine/worklane-update.cjs': { + n: 6, + why: "the root updater, installed alone in /usr/local/libexec: its reads of the verification clone carry VERIFY_GIT_ARGS (replace refs off); that clone is root's own, which no agent can write a graft into; the two ls-remote calls read no object", + must: [/^const VERIFY_GIT_ARGS = \['--no-replace-objects'\];$/m], + }, + // CI and dev scripts, through a runner that takes any command: commandSpawnSync / commandExecFileSync / commandSpawn (scripts/lib/git.cjs). + 'scripts/ci/updater-setup-root.mjs': { n: 1, why: "`sh('git', ...)`: sh, here, runs through commandExecFileSync", must: BY_NAME('commandExecFileSync', 'file, args, ') }, + 'scripts/ci/wizard-e2e.mjs': { n: 5, why: "`sh('git', ...)`: sh, here, runs through commandSpawnSync", must: BY_NAME('commandSpawnSync', 'cmd, args, ') }, + 'scripts/dev/compare-screenshots.mjs': { n: 4, why: "`run('git', ...)`: run, here, spawns through commandSpawn", must: BY_NAME('commandSpawn', 'cmd, args, ') }, +}; + +function sources(): string[] { + const files: string[] = []; + const walk = (d: string) => { + for (const e of readdirSync(d, { withFileTypes: true })) { + if (e.isDirectory()) walk(join(d, e.name)); + else if (/\.(ts|mts|cts|mjs|cjs|js)$/.test(e.name)) files.push(join(d, e.name)); + } + }; + walk(join(repoRoot, 'src')); + walk(join(repoRoot, 'scripts')); + return files; +} + +test('every git call in src/ and scripts/ goes through the runner (src/os/git.ts, scripts/lib/git.cjs)', () => { + const bad: string[] = []; + const seen = new Set(); + for (const f of sources()) { + const rel = relative(repoRoot, f).split('\\').join('/'); + if (RUNNERS.has(rel)) continue; + const text = readFileSync(f, 'utf8'); + const n = gitStarts(text); + if (!n) continue; + seen.add(rel); + const allowed = OUTSIDE_THE_RUNNER[rel]; + if (!allowed) bad.push(`${rel}: starts git ${n} time(s) (not on the list)`); + else if (allowed.n !== n) bad.push(`${rel}: starts git ${n} time(s), listed ${allowed.n}`); + else for (const re of allowed.must ?? []) if (!re.test(text)) bad.push(`${rel}: no ${re}`); + } + for (const rel of Object.keys(OUTSIDE_THE_RUNNER)) if (!seen.has(rel)) bad.push(`${rel}: listed, but starts git no more (take it off the list)`); + assert.deepEqual(bad, [], 'run git through src/os/git.ts (gitSpawnSync, gitExecFileSync, gitSpawn, gitExecFile, commandSpawnSync) or scripts/lib/git.cjs'); +}); + +test('the scan catches each way of starting git, and not text that only names it', () => { + const caught = [ + "spawnSync('git', ['log'])", + 'execFileSync("git", ["show", x])', + 'spawn(`git`, args, {})', + "execFile('git', a, o, cb)", + "run('/usr/bin/git', ['fetch'])", + "spawnSync('C:\\\\Git\\\\cmd\\\\git.exe', [])", + "spawnSync('GIT', [])", + "const commands = [['git', 'fetch', '-q']];", + "const bin = 'git';", + "exec('git log -1', cb)", + 'execSync(`git status`)', + "spawnSync('sh', ['-c', 'git log'])", + "spawnSync(\n 'git',\n ['log'],\n)", + ]; + for (const c of caught) assert.equal(gitStarts(c), 1, c); + const named = ["if (bin === 'git') return;", "basename(a[0]) !== 'git'", "{ type: 'git', url }", "// run git log", "'gitleaks'", "spawnSync('legit', [])", "'git log' + x", "the git"]; + for (const c of named) assert.equal(gitStarts(c), 0, c); +}); + +test('the runners and these tests are locked: a change to any of them alone is self-modification, L3, never L0', () => { + const cfg = ReviewConfig.parse({ version: 1, levels: { L0_auto: { when: ['docs-only', 'tests-only'], max_lines: 200 }, L1_evaluator: { when: [] }, L2_notify: { when: [] }, L3_human: { when: [] } } }); + for (const [path, part] of [['test/git-runner.test.ts', 'the tests of the harness controls'], ['src/os/git.ts', 'the git runner'], ['scripts/lib/git.cjs', 'the git runner']] as const) { + const r = computeLevel({ files: [{ path, added: 1, removed: 1 }], labels: [], moneyPaths: [], scope: undefined }, cfg); + assert.deepEqual([r.level, r.categories[SELF_MODIFICATION], selfModificationPart(path)], ['L3', [path], part], path); + } +}); + +test('the runners the scan skips are the two runner files, and each starts git itself', () => { + assert.deepEqual([...RUNNERS].sort(), ['scripts/lib/git.cjs', 'src/os/git.ts']); + for (const rel of RUNNERS) assert.ok(gitStarts(readFileSync(join(repoRoot, rel), 'utf8')) > 0, rel); +}); diff --git a/test/review.test.ts b/test/review.test.ts index f5e02063c..d72013fc5 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -171,7 +171,7 @@ const GUARD_TREE = [ 'src/events/types.ts', 'src/experiments/paths.ts', 'src/governor-holds.ts', 'src/guard/chat.ts', 'src/guard/intake.ts', 'src/guard/research.ts', 'src/guardrails/context.ts', 'src/guardrails/engine.ts', 'src/guardrails/fingerprint.ts', 'src/guardrails/glob.ts', 'src/guardrails/push-hook.ts', 'src/guardrails/shell.ts', 'src/held-out-paths.ts', 'src/hook-imports.ts', - 'src/hook-main.ts', 'src/hook.ts', 'src/model-routing.ts', 'src/network.ts', 'src/optimize.ts', 'src/os/index.ts', + 'src/hook-main.ts', 'src/hook.ts', 'src/model-routing.ts', 'src/network.ts', 'src/optimize.ts', 'src/os/git.ts', 'src/os/index.ts', 'src/os/own-env.ts', 'src/os/system-paths.ts', 'src/os/system-root-guard.ts', 'src/path-text.ts', 'src/preview/config.ts', 'src/project-env.ts', 'src/provider-hosts.ts', 'src/providers.ts', 'src/quiet-sqlite.ts', 'src/review.ts', 'src/roles.ts', 'src/routing.ts', 'src/scan/secrets.ts', 'src/slot-file.ts', 'src/stage-names.ts', 'src/testrun/affected.ts', 'src/testrun/config.ts', From e99ea51f094397926432cbed7abdbb1ba12c6bdc Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Tue, 20 Oct 2026 22:44:48 -0500 Subject: [PATCH 02/18] git runner: auto-merge-resume keeps its header comment in the build; the test header says what it measures (#807) With a type-only import first, tsc dropped auto-merge-resume.ts's leading comment from its build (measured: 1,428 bytes gone from dist/src/auto-merge-resume.js). The value import goes first again. test/git-runner.test.ts's header now gives the counts its tests measure (2 problems became 0, for each route). Signed-off-by: Jayden Vasquez --- src/auto-merge-resume.ts | 2 +- test/git-runner.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/auto-merge-resume.ts b/src/auto-merge-resume.ts index 9ba891a18..fa3303c1e 100644 --- a/src/auto-merge-resume.ts +++ b/src/auto-merge-resume.ts @@ -12,9 +12,9 @@ // Main's checks are JudgedChecks, so a pinned required check is green only from its pinned source. // Fail closed: a stop with no `by`, an unknown one, or an unreadable stop file waits for green main; main with no // required checks configured, a tip that can't be read, or checks that can't be read is not green. +import { gitSpawn } from './os/git.js'; import type { ChildProcess } from './os/child-process.js'; import type { JudgedChecks } from './check-source.js'; -import { gitSpawn } from './os/git.js'; import { killTree, spawnDetached } from './os/index.js'; import { requiredOutcomes } from './pr-watch.js'; diff --git a/test/git-runner.test.ts b/test/git-runner.test.ts index 57b4a4d22..014afcef5 100644 --- a/test/git-runner.test.ts +++ b/test/git-runner.test.ts @@ -1,7 +1,7 @@ // One git runner for every git call the engine makes (#807). refs/replace/* and $GIT_DIR/info/grafts are in the // repository, which every worktree of it shares, so an agent can write either; every git the engine runs there then -// reads the agent's stand-in. A `git replace` of the pushed commit hid a refused file from the push limits (1 problem -// became 0), and a graft that makes the base reach the head empties the range. +// reads the agent's stand-in. A `git replace` of the pushed commit hid a refused file and a big one from the push +// limits (2 problems became 0), and a graft that makes the base reach the head emptied the range (2 became 0). import { test } from 'node:test'; import assert from 'node:assert/strict'; import { execFileSync, spawnSync } from 'node:child_process'; From a8d9c98ccab2aa8aede80e0cd0265ad23987f6ff Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Tue, 20 Oct 2026 22:47:52 -0500 Subject: [PATCH 03/18] changelog: every git the engine runs ignores replace refs and grafts (#927) The release note for the git runner: what an upgrade changes (a new patch_hash for the same change, the optional setup:hooks rerun) and the guardrail it adds. Test (fails without): none, release notes only; test/changelog*.test.ts pass with it. Signed-off-by: Jayden Vasquez --- changelog.d/927-git-runner.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 changelog.d/927-git-runner.md diff --git a/changelog.d/927-git-runner.md b/changelog.d/927-git-runner.md new file mode 100644 index 000000000..c7d774bba --- /dev/null +++ b/changelog.d/927-git-runner.md @@ -0,0 +1,16 @@ +### Upgrading (from v0.5.2) + +- **Every git the engine runs ignores replace refs and grafts (#927).** An agent could write `refs/replace/*` or + `.git/info/grafts` in the shared repo and make the push limits, the secret scan and the review read other commits. + Nothing to do; a change's recorded `patch_hash` now covers every file's lines, so it differs from an older engine's + for the same change. Run `npm run setup:hooks` once to get the push-gate hook that also hands the gate these + settings (the old hook keeps working). + +### Guardrail changes: what each one does + +- **The git runner (#927):** `src/os/git.ts` (and `scripts/lib/git.cjs` for scripts) starts every git with replace + refs off and no graft file, over whatever environment the caller passes; gitleaks gets the same settings. A test + fails on a git call in `src/` or `scripts/` that doesn't come through it, and the runner and that test wait for the + owner. +- **What the owner sees (#927):** the decision diff, the fix-forward culprit diff and the patch hash show a file's + lines even when the change's `.gitattributes` marks it `-diff` or `binary`. From fee5bb19ffc7e5c506fe62891af32e5045492cf8 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Tue, 20 Oct 2026 23:38:18 -0500 Subject: [PATCH 04/18] test: fixtures follow the git runner (#927) CI was red at a8d9c98c on six files that expected the old argv or copied fixed file lists: - intake-coordinator expects `git --no-replace-objects merge-base ...`; - testrun-measure's pinned import list for measure-cli swaps ../os/child-process.js for ../os/git.js (a swap, not a loosening: os/git.ts starts its processes only through os/child-process.ts); - compare-screenshots, eval-spend-gate, engine-release and installed-engine-config copy scripts/lib/git.cjs beside the scripts they copy. Test (fails without): the six files, red in run 39540570972; 59 of 59 pass with this. Signed-off-by: Jayden Vasquez --- test/compare-screenshots.test.ts | 3 +++ test/engine-release.test.ts | 2 +- test/eval-spend-gate.test.ts | 4 +++- test/installed-engine-config.test.ts | 2 +- test/intake-coordinator.test.ts | 3 ++- test/testrun-measure.test.ts | 2 +- 6 files changed, 11 insertions(+), 5 deletions(-) diff --git a/test/compare-screenshots.test.ts b/test/compare-screenshots.test.ts index af2a0c926..d69755592 100644 --- a/test/compare-screenshots.test.ts +++ b/test/compare-screenshots.test.ts @@ -33,6 +33,9 @@ function fakeRepo(): string { git(origin, 'init', '-q', '--bare', '-b', 'main'); mkdirSync(join(repo, 'scripts', 'dev'), { recursive: true }); for (const f of ['compare-screenshots.mjs', 'children.mjs']) copyFileSync(join(repoRoot, 'scripts', 'dev', f), join(repo, 'scripts', 'dev', f)); + // The git runner compare-screenshots.mjs imports (#807). + mkdirSync(join(repo, 'scripts', 'lib'), { recursive: true }); + copyFileSync(join(repoRoot, 'scripts', 'lib', 'git.cjs'), join(repo, 'scripts', 'lib', 'git.cjs')); writeFileSync( join(repo, 'scripts', 'dev', 'screenshots.mjs'), [ diff --git a/test/engine-release.test.ts b/test/engine-release.test.ts index dbcaec0f6..5cf1fc3d9 100644 --- a/test/engine-release.test.ts +++ b/test/engine-release.test.ts @@ -97,7 +97,7 @@ function fixture() { // updater beside them, and its own .github/release-signers: the trusted list. const checkout = join(dir, 'checkout'); sh(dir, 'git', ['clone', '-q', origin, checkout]); - for (const f of ['scripts/setup/engine.sh', 'scripts/setup/lib.sh', 'scripts/setup/engine-release.cjs', 'scripts/machine/worklane-update.cjs']) { + for (const f of ['scripts/setup/engine.sh', 'scripts/setup/lib.sh', 'scripts/setup/engine-release.cjs', 'scripts/machine/worklane-update.cjs', 'scripts/lib/git.cjs']) { if (!existsSync(join(repoRoot, f))) continue; mkdirSync(join(checkout, f, '..'), { recursive: true }); copyFileSync(join(repoRoot, f), join(checkout, f)); diff --git a/test/eval-spend-gate.test.ts b/test/eval-spend-gate.test.ts index 30a7e0b8f..0a4003fa4 100644 --- a/test/eval-spend-gate.test.ts +++ b/test/eval-spend-gate.test.ts @@ -160,8 +160,10 @@ test('eval-roles.mjs: a dist/ whose eval-roles.js has no spend option still stop join(root, 'dist', 'src', 'eval-roles.js'), `export * from ${JSON.stringify(real('eval-roles.js'))};\nimport { evalRolesOptions as current } from ${JSON.stringify(real('eval-roles.js'))};\nexport const evalRolesOptions = (argv) => { const o = current(argv); delete o.spend; return o; };\n`, ); - mkdirSync(join(root, 'scripts')); + mkdirSync(join(root, 'scripts', 'lib'), { recursive: true }); writeFileSync(join(root, 'scripts', 'eval-roles.mjs'), readFileSync(evalRoles, 'utf8')); + // The git runner the script imports (#807). + writeFileSync(join(root, 'scripts', 'lib', 'git.cjs'), readFileSync(join(repoRoot, 'scripts', 'lib', 'git.cjs'), 'utf8')); for (const d of ['node_modules', 'src', 'templates']) symlinkSync(join(repoRoot, d), join(root, d)); const r = run(join(root, 'scripts', 'eval-roles.mjs'), ['--score', '--roles', 'feedback-checks'], m.env, root); assert.equal(r.status, 2, r.out); diff --git a/test/installed-engine-config.test.ts b/test/installed-engine-config.test.ts index 7b31b6151..56300eb1a 100644 --- a/test/installed-engine-config.test.ts +++ b/test/installed-engine-config.test.ts @@ -184,7 +184,7 @@ const labelEvent = (event: 'labeled' | 'unlabeled', login: string, at: string, n const workflow = () => readFileSync(join(repoRoot, '.github', 'workflows', 'installed-engine-config.yml'), 'utf8'); /** The check's scripts as this checkout has them, to commit into a fixture's main. */ const checkScripts = () => - Object.fromEntries(['scripts/ci/installed-engine-config.mjs', 'scripts/lib/engine-pin.mjs', 'scripts/verify-release-tag.mjs'].map((f) => [f, readFileSync(join(repoRoot, f), 'utf8')])); + Object.fromEntries(['scripts/ci/installed-engine-config.mjs', 'scripts/lib/engine-pin.mjs', 'scripts/verify-release-tag.mjs', 'scripts/lib/git.cjs'].map((f) => [f, readFileSync(join(repoRoot, f), 'utf8')])); const refused = (r: { code: number | null; out: string }, why: RegExp) => { assert.equal(r.code, 1, r.out); diff --git a/test/intake-coordinator.test.ts b/test/intake-coordinator.test.ts index c94877f4e..e18a93c84 100644 --- a/test/intake-coordinator.test.ts +++ b/test/intake-coordinator.test.ts @@ -177,7 +177,8 @@ test('intake (#333): a bug already fixed on main is answered naming the fixing c onMain = git(t.repo, 'rev-parse', 'HEAD'); git(t.repo, 'update-ref', 'refs/remotes/origin/main', onMain); const spawned = await spawnsDuring(() => t.c.checkIntake()); - const check = (id: string) => `execFileSync git merge-base --is-ancestor ${id} refs/remotes/origin/main`; + // Through the git runner (#807): replace refs off. + const check = (id: string) => `execFileSync git --no-replace-objects merge-base --is-ancestor ${id} refs/remotes/origin/main`; assert.deepEqual(spawned, [check(onMain), check('0a1b2c3d4e5f')], 'the only process: one read-only git check per fixing commit, on the trusted checkout'); assert.equal(t.discord.replies.length, 3); assert.match(t.discord.replies[0]!.text, new RegExp(`already fixed.*\\b${onMain}\\b`), 'confirmed on main: named'); diff --git a/test/testrun-measure.test.ts b/test/testrun-measure.test.ts index f9b2433ca..a2e6ad00a 100644 --- a/test/testrun-measure.test.ts +++ b/test/testrun-measure.test.ts @@ -268,7 +268,7 @@ test('no check, no PR: a measure records only testrun.measure, so the PR runs, t const names = [...reach].map((f) => f.slice(join(repoRoot, 'src').length + 1)); for (const bad of ['testrun/checks.ts', 'testrun/index.ts', 'testrun/setup.ts', 'github-app.ts', 'coordinator/test-runs.ts', 'coordinator/pr-batch.ts']) assert.ok(!names.includes(bad), `measure.ts reaches ${bad}`); const cli = readFileSync(join(repoRoot, 'src', 'testrun', 'measure-cli.ts'), 'utf8'); - assert.deepEqual([...cli.matchAll(/^import [^'"]*['"]([^'"]+)['"]/gm)].map((m) => m[1]).sort(), ['../brand.js', '../events/log.js', '../instance.js', '../os/child-process.js', './host.js', './measure.js', './workspace.js', 'node:path']); + assert.deepEqual([...cli.matchAll(/^import [^'"]*['"]([^'"]+)['"]/gm)].map((m) => m[1]).sort(), ['../brand.js', '../events/log.js', '../instance.js', '../os/git.js', './host.js', './measure.js', './workspace.js', 'node:path']); }); test('the helper sees exactly what a PR run of the tier sends it: the same calls, command and env, the measured value aside', { skip: !hasPerl && 'perl for the flock holder' }, async () => { From a1ff009105217da61ad7cf65a18cbe9d11946cf9 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Tue, 20 Oct 2026 23:38:27 -0500 Subject: [PATCH 05/18] push limits: a shallow cut inside the pushed range is refused (#927) A `shallow` file lists commits whose parents git treats as missing, and any agent can write one into the shared repo. With the head's parent listed, pushProblems read 0 problems for a range that added a refused file; deleted again before the push, the push carried that file to the remote (measured by #927's final check). shallowCut refuses a range such a commit cuts, before and after the walks and once more right before the push in checkedPush; a shallow file it can't read is a refusal too. Only a cut inside the range is refused: the harness-eval replay's clone is shallow at the task's base (measured), and a depth-limited clone pushes changes above its cut as before. Not --shallow-file: git's internal. The git-runner scan's comment lists its measured misses, and the release note names the shallow refusal. Test (fails without): test/push-shallow.test.ts, 3 of 4 fail at a8d9c98c (a hand-written cut: 0 problems; the delete-between race; an unreadable shallow file); the depth-limited clone control passes either way. Signed-off-by: Jayden Vasquez --- changelog.d/927-git-runner.md | 3 ++ src/push-check.ts | 34 ++++++++++++- test/git-runner.test.ts | 7 ++- test/push-shallow.test.ts | 94 +++++++++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 test/push-shallow.test.ts diff --git a/changelog.d/927-git-runner.md b/changelog.d/927-git-runner.md index c7d774bba..60a043f11 100644 --- a/changelog.d/927-git-runner.md +++ b/changelog.d/927-git-runner.md @@ -12,5 +12,8 @@ refs off and no graft file, over whatever environment the caller passes; gitleaks gets the same settings. A test fails on a git call in `src/` or `scripts/` that doesn't come through it, and the runner and that test wait for the owner. +- **A shallow cut in a pushed range is refused (#927):** a `.git/shallow` (a shallow clone's, or one written by hand) + that cuts the range being pushed made the push limits read only part of it. The push is now refused, and the + message names `git fetch --unshallow`. A shallow clone still pushes changes that lie wholly above its cut. - **What the owner sees (#927):** the decision diff, the fix-forward culprit diff and the patch hash show a file's lines even when the change's `.gitattributes` marks it `-diff` or `binary`. diff --git a/src/push-check.ts b/src/push-check.ts index b523a49f8..9600dbd93 100644 --- a/src/push-check.ts +++ b/src/push-check.ts @@ -5,6 +5,8 @@ // leaves a SKILL.md that fails the skill lint. // The coordinator also runs pushProblems on each change before verifying it, // so a doomed change is caught before a long test run. +import { existsSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; import { sizeFiles } from './coordinator/change-files.js'; import { globToRegExp } from './guardrails/glob.js'; import { gitExecFileSync, gitSpawnSync } from './os/git.js'; @@ -29,9 +31,35 @@ const list = (paths: string[], n = 10) => `${paths.slice(0, n).join(', ')}${path /** Changed paths in a problem's text, each as shownPath shows it (#683): `rev-list --objects` prints a name as it is. */ const listPaths = (paths: string[], n = 10) => list(paths.map(shownPath), n); +/** + * Why the range can't be read whole, or null (#807). A shallow repo's `shallow` file lists commits whose parents git + * then treats as missing, and any agent can write it into the shared repo: a commit listed inside the range cuts the + * walks below short (a refused file in an earlier commit read as 0 problems, measured). A cut outside the range (a + * depth-limited clone's base, the harness-eval replay's clone) leaves the range whole and is fine. Fails closed: a + * shallow file or range it can't read is a refusal too. WHY not `--shallow-file`: it is git's internal option. + */ +export function shallowCut(cwd: string, range: string): string | null { + const refuse = (why: string) => + `this repo's history is cut inside the range (${why}): the push limits can't read it whole, so nothing is pushed. A shallow clone pushes only changes above its cut; fetch the rest (git fetch --unshallow), or remove a .git/shallow file nobody meant to write`; + try { + const file = resolve(cwd, gitExecFileSync(['rev-parse', '--git-path', 'shallow'], { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()); + if (!existsSync(file)) return null; + const cut = new Set(readFileSync(file, 'utf8').split('\n').map((l) => l.trim()).filter(Boolean)); + if (!cut.size) return null; + const inRange = git(cwd, ['rev-list', range]).split('\n').filter((c) => cut.has(c)); + return inRange.length ? refuse(`a shallow boundary at ${inRange.slice(0, 3).map((c) => c.slice(0, 12)).join(', ')}`) : null; + } catch (e) { + return refuse(`can't read its shallow file: ${String((e as Error).message).split('\n')[0]}`); + } +} + /** Every reason the range base..head may not be pushed; empty means it may. */ export function pushProblems(cwd: string, base: string, head: string, limits: PushLimits): string[] { const range = `${base}..${head}`; + // Before the walks (a cut already there) and again after them (one written while they ran): a cut range is judged + // by nothing else. + const before = shallowCut(cwd, range); + if (before) return [before]; const problems: string[] = []; // Paths touched by any commit in the range, not just the net diff. const touched = [...new Set(git(cwd, ['log', '--format=', '--name-only', '--no-renames', '--diff-merges=first-parent', range]).split('\n').filter(Boolean))]; @@ -75,7 +103,8 @@ export function pushProblems(cwd: string, base: string, head: string, limits: Pu if (lint.length) badSkills.push(`${shownPath(p)} (${lint.join('; ')})`); } if (badSkills.length) problems.push(`has skills that fail the skill lint: ${list(badSkills, 5)}. Every SKILL.md needs these sections, none empty: ${SKILL_SECTIONS.join(', ')}`); - return problems; + const after = shallowCut(cwd, range); + return after ? [after] : problems; } export type PushResult = { ok: true } | { ok: false; refused: string[] } | { ok: false; error: string }; @@ -93,6 +122,9 @@ export function checkedPush(o: { cwd: string; remote: string; base: string; head if (main) return { ok: false, refused: [main] }; const refused = pushProblems(o.cwd, o.base, o.head, o.limits); if (refused.length) return { ok: false, refused }; + // Once more right before the push, in this process: a cut written since the limits ran (#807). + const late = shallowCut(o.cwd, `${o.base}..${o.head}`); + if (late) return { ok: false, refused: [late] }; const lease = o.expect ? [`--force-with-lease=${o.ref}:${o.expect}`] : []; const push = gitSpawnSync(['push',...lease, o.remote, `${o.force ? '+' : ''}${o.head}:${o.ref}`], { cwd: o.cwd, encoding: 'utf8' }); if (push.status !== 0) return { ok: false, error: (push.stderr || '').trim() }; diff --git a/test/git-runner.test.ts b/test/git-runner.test.ts index 014afcef5..1e8ca847f 100644 --- a/test/git-runner.test.ts +++ b/test/git-runner.test.ts @@ -162,8 +162,11 @@ test('scripts/lib/git.cjs (scripts run without a build) has the engine runner\'s * Where a source text starts git itself: `git` (or a path to it, or git.exe) as a quoted string in an argument or array * position or assigned to a name (`spawnSync('git', …)`, `run('git', …)`, `['git', 'fetch']`, `const bin = 'git'`), or * a shell line that starts with git (`exec('git log')`, `['-c', 'git log']`). A comparison (`=== 'git'`) or an object - * key's value (`type: 'git'`) isn't one. A backstop next to the runner, not a proof: a name built at run time - * (`'gi' + 't'`) gets past it. + * key's value (`type: 'git'`) isn't one. A backstop next to the runner, not a proof. Measured misses (#927's final + * check): a name built at run time (`'gi' + 't'`, `${'git'}`, String.raw`git`), a ternary or a default (`x ? 'git' :`, + * `?? 'git'`, `|| 'git'`), a command from config, env or an object property, git in the middle of a shell string + * (`bash -lc 'cd x && git log'`, `execSync('cd x && git log')`, `spawnSync('git log', { shell: true })`), and the + * `.sh` files under scripts/ (16 run git; not scanned). */ function gitStarts(text: string): number { const patterns = [ diff --git a/test/push-shallow.test.ts b/test/push-shallow.test.ts new file mode 100644 index 000000000..83caf8e31 --- /dev/null +++ b/test/push-shallow.test.ts @@ -0,0 +1,94 @@ +// A shallow cut inside the pushed range is refused (#807, round 2). The `shallow` file lists commits whose parents git +// treats as missing, and an agent can write it into the shared repo: with the head's parent listed, the walks of +// base..head stopped short and a refused file in an earlier commit read as 0 problems; deleted again before the push, +// the push carried that file to the remote (measured by #927's final check). The limits now refuse a range cut by +// such a file, before and after their walks and once more right before the push. A cut outside the range (a +// depth-limited clone's base) leaves the range whole and pushes as before. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { checkedPush, DEFAULT_PUSH_LIMITS, pushProblems } from '../src/push-check.js'; + +const git = (cwd: string, ...a: string[]) => execFileSync('git', a, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const LIMITS = { ...DEFAULT_PUSH_LIMITS, refuse_paths: ['**/*.pem'] }; +const guard = { defaultBranch: 'main', blocked: () => undefined }; + +/** base, then a commit adding keys/secret.pem, then one removing it: the net diff is clean, the history isn't. */ +function repo() { + const root = mkdtempSync(join(tmpdir(), 'push-shallow-')); + const work = join(root, 'work'); + const remote = join(root, 'remote.git'); + git(root, 'init', '-q', '--bare', '-b', 'main', remote); + git(root, 'init', '-q', '-b', 'main', work); + for (const [k, v] of [['user.email', 'bot@example.com'], ['user.name', 'bot'], ['commit.gpgsign', 'false']]) git(work, 'config', k!, v!); + writeFileSync(join(work, 'README.md'), 'hello\n'); + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', 'base'); + const base = git(work, 'rev-parse', 'HEAD'); + git(work, 'push', '-q', remote, 'main'); + mkdirSync(join(work, 'keys')); + writeFileSync(join(work, 'keys', 'secret.pem'), 'k\n'); + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', 'add a key'); + git(work, 'rm', '-q', 'keys/secret.pem'); + git(work, 'commit', '-q', '-m', 'drop it'); + const head = git(work, 'rev-parse', 'HEAD'); + const shallow = join(work, '.git', 'shallow'); + return { work, remote, base, head, shallow }; +} + +test('a .git/shallow written by hand that cuts the range is refused, never 0 problems', () => { + const r = repo(); + assert.equal(pushProblems(r.work, r.base, r.head, LIMITS).length, 1, 'the key is seen in a whole history'); + writeFileSync(r.shallow, `${r.head}\n`); + const p = pushProblems(r.work, r.base, r.head, LIMITS); + assert.equal(p.length, 1, p.join('\n')); + assert.match(p[0]!, /history is cut inside the range \(a shallow boundary at [0-9a-f]{12}\)/); +}); + +test('the delete-between race: the check sees the cut and refuses, so nothing reaches the remote, and deleting it afterwards changes nothing', () => { + const r = repo(); + writeFileSync(r.shallow, `${r.head}\n`); + const res = checkedPush({ cwd: r.work, remote: r.remote, base: r.base, head: r.head, ref: 'refs/heads/task', limits: LIMITS, guard, force: true }); + assert.ok(!res.ok && 'refused' in res, JSON.stringify(res)); + assert.match((res as { refused: string[] }).refused.join('\n'), /history is cut inside the range/); + rmSync(r.shallow); + assert.equal(git(r.remote, 'for-each-ref', 'refs/heads/task'), '', 'nothing pushed'); + // Pushed again with the file gone, the whole history is judged: the key is refused. + const again = checkedPush({ cwd: r.work, remote: r.remote, base: r.base, head: r.head, ref: 'refs/heads/task', limits: LIMITS, guard, force: true }); + assert.ok(!again.ok && 'refused' in again && again.refused.some((x) => /keys\/secret\.pem/.test(x)), JSON.stringify(again)); + assert.equal(git(r.remote, 'for-each-ref', 'refs/heads/task'), ''); +}); + +test('a shallow file git can\'t read as a list (a directory in its place) fails closed', () => { + const r = repo(); + mkdirSync(r.shallow); + writeFileSync(join(r.shallow, 'x'), ''); + const p = pushProblems(r.work, r.base, r.head, LIMITS); + assert.equal(p.length, 1, p.join('\n')); + assert.match(p[0]!, /history is cut inside the range \(can't read its shallow file/); + assert.ok(existsSync(r.shallow)); +}); + +test('a depth-limited clone pushes a change above its cut as before: the cut is its base, outside the range', () => { + const r = repo(); + const clone = join(mkdtempSync(join(tmpdir(), 'push-shallow-clone-')), 'c'); + git(r.work, 'clone', '-q', '--depth', '1', pathToFileURL(r.remote).href, clone); + assert.equal(git(clone, 'rev-parse', '--is-shallow-repository'), 'true'); + for (const [k, v] of [['user.email', 'bot@example.com'], ['user.name', 'bot'], ['commit.gpgsign', 'false']]) git(clone, 'config', k!, v!); + const base = git(clone, 'rev-parse', 'HEAD'); + writeFileSync(join(clone, 'a.txt'), 'a\n'); + git(clone, 'add', '-A'); + git(clone, 'commit', '-q', '-m', 'a change'); + const head = git(clone, 'rev-parse', 'HEAD'); + assert.deepEqual(pushProblems(clone, base, head, LIMITS), []); + // A refused file above the cut is still refused. + writeFileSync(join(clone, 'b.pem'), 'k\n'); + git(clone, 'add', '-A'); + git(clone, 'commit', '-q', '-m', 'a key'); + assert.match(pushProblems(clone, base, git(clone, 'rev-parse', 'HEAD'), LIMITS).join('\n'), /b\.pem/); +}); From a45d1f2c1c91cb382406168251e2046f6709ea90 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 00:20:20 -0500 Subject: [PATCH 06/18] git runner: the commit-graph is off for every git the engine runs (#927) A forged parent link in the shared objects/info/commit-graph made a walk skip the commit that added a refused file, and the push then landed it (measured by #927's round-2 check). -c core.commitGraph=false joins the runner's fixed arguments, and gitEnv appends the same pair to GIT_CONFIG_COUNT after the caller's own, for the gits a program runs itself (gitleaks). scripts/lib/git.cjs, the four standalone scripts and the push gate's gitleaks env do the same. Measured cost: about 30 ms more for three walks over this repo's 2,903 commits. Test (fails without): test/push-snapshot.test.ts's commit-graph cases (push limits, checkedPush, gitleaks) fail at a1ff0091; test/git-runner.test.ts holds the settings and their copies. Signed-off-by: Jayden Vasquez --- scripts/ci/new-deps.mjs | 2 +- scripts/ci/testrun-real-systemd.mjs | 2 +- scripts/lib/git.cjs | 20 ++++- scripts/push-gate-hook.mjs | 2 +- scripts/push-gate.mjs | 10 ++- src/os/git.ts | 115 ++++++++++++++++++++++++++-- test/intake-coordinator.test.ts | 2 +- 7 files changed, 136 insertions(+), 17 deletions(-) diff --git a/scripts/ci/new-deps.mjs b/scripts/ci/new-deps.mjs index 0c48da331..5a137e448 100644 --- a/scripts/ci/new-deps.mjs +++ b/scripts/ci/new-deps.mjs @@ -147,7 +147,7 @@ export function check({ basePkg, headPkg, baseLock, headLock, body }) { // The git runner's settings (#807; src/os/git.ts, scripts/lib/git.cjs): replace refs off and no graft file. Written // out here because CI runs the base's copy of this file alone, from a temp dir (new-deps.yml), so it imports nothing // of the repo's; test/git-runner.test.ts checks they are the runner's. -const GIT_FIXED_ARGS = ['--no-replace-objects']; +const GIT_FIXED_ARGS = ['--no-replace-objects', '-c', 'core.commitGraph=false']; const GIT_ENV = { ...process.env, GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }; const runGit = (args, opts) => execFileSync('git', [...GIT_FIXED_ARGS, ...args], { ...opts, env: GIT_ENV }); const git = (...a) => runGit(a, { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); diff --git a/scripts/ci/testrun-real-systemd.mjs b/scripts/ci/testrun-real-systemd.mjs index 14579369c..d049db582 100644 --- a/scripts/ci/testrun-real-systemd.mjs +++ b/scripts/ci/testrun-real-systemd.mjs @@ -58,7 +58,7 @@ const socketFromHere = (path) => new Promise((resolve) => { const repo = mkdtempSync(join(homedir(), 'repo-')); // The git runner's settings (#807; src/os/git.ts, scripts/lib/git.cjs): replace refs off and no graft file. Written // out here because the job copies this file alone into the VM (ci-slow.yml); test/git-runner.test.ts checks them. -const GIT_FIXED_ARGS = ['--no-replace-objects']; +const GIT_FIXED_ARGS = ['--no-replace-objects', '-c', 'core.commitGraph=false']; const GIT_ENV = { PATH: '/usr/bin:/bin', HOME: homedir(), GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_NOSYSTEM: '1', GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }; const git = (...a) => execFileSync('git', [...GIT_FIXED_ARGS, '-C', repo, '-c', 'user.name=ci', '-c', 'user.email=ci@example.com', ...a], { encoding: 'utf8', env: GIT_ENV }).trim(); git('init', '-q', '-b', 'main'); diff --git a/scripts/lib/git.cjs b/scripts/lib/git.cjs index c48e3d958..5fa6067b8 100644 --- a/scripts/lib/git.cjs +++ b/scripts/lib/git.cjs @@ -8,11 +8,24 @@ const { execFileSync, spawn, spawnSync } = require('node:child_process'); const { devNull } = require('node:os'); const { join } = require('node:path'); -const GIT_FIXED_ARGS = Object.freeze(['--no-replace-objects']); +const GIT_FIXED_ARGS = Object.freeze(['--no-replace-objects', '-c', 'core.commitGraph=false']); const GIT_FIXED_ENV = Object.freeze({ GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); +/** git config every git also gets through its environment (GIT_CONFIG_COUNT pairs), after the caller's own pairs. */ +const GIT_FIXED_CONFIG = Object.freeze([Object.freeze(['core.commitGraph', 'false'])]); -/** `env` (or this process's environment), with the fixed settings on top. */ -const gitEnv = (env) => ({ ...(env || process.env), ...GIT_FIXED_ENV }); +/** `env` (or this process's environment), with the fixed settings on top and the fixed config pairs after its own. */ +const gitEnv = (env) => { + const out = { ...(env || process.env), ...GIT_FIXED_ENV }; + let n = Number(out.GIT_CONFIG_COUNT === undefined ? '0' : out.GIT_CONFIG_COUNT); + if (!Number.isSafeInteger(n) || n < 0) n = 0; + for (const [k, v] of GIT_FIXED_CONFIG) { + out[`GIT_CONFIG_KEY_${n}`] = k; + out[`GIT_CONFIG_VALUE_${n}`] = v; + n++; + } + out.GIT_CONFIG_COUNT = String(n); + return out; +}; /** The argument list a git process gets: the fixed arguments, then `args`. */ const gitArgs = (args) => [...GIT_FIXED_ARGS, ...args]; /** True for a command that runs git: `git`, or a path to it, in any case. */ @@ -33,6 +46,7 @@ const commandSpawn = (cmd, args, options = {}) => (isGitCommand(cmd) ? spawn(cmd exports.GIT_FIXED_ARGS = GIT_FIXED_ARGS; exports.GIT_FIXED_ENV = GIT_FIXED_ENV; +exports.GIT_FIXED_CONFIG = GIT_FIXED_CONFIG; exports.gitEnv = gitEnv; exports.gitArgs = gitArgs; exports.isGitCommand = isGitCommand; diff --git a/scripts/push-gate-hook.mjs b/scripts/push-gate-hook.mjs index 3d89d18b3..1831bf365 100644 --- a/scripts/push-gate-hook.mjs +++ b/scripts/push-gate-hook.mjs @@ -44,7 +44,7 @@ class NotOnOrigin extends Refusal {} * hooks dir), so it can't import them; test/git-runner.test.ts checks they are the runner's. The gate gets them too, * in its environment, so every git it and gitleaks run has them, whichever gate origin holds. */ -export const GIT_FIXED_ARGS = Object.freeze(['--no-replace-objects']); +export const GIT_FIXED_ARGS = Object.freeze(['--no-replace-objects', '-c', 'core.commitGraph=false']); export const GIT_FIXED_ENV = Object.freeze({ GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); /** execFileSync of git, with the settings. Every git this file runs comes through here. */ const runGit = (args, opts = {}) => execFileSync('git', [...GIT_FIXED_ARGS, ...args], { ...opts, env: { ...(opts.env || process.env), ...GIT_FIXED_ENV } }); diff --git a/scripts/push-gate.mjs b/scripts/push-gate.mjs index e696df284..e41b18c0b 100644 --- a/scripts/push-gate.mjs +++ b/scripts/push-gate.mjs @@ -28,8 +28,14 @@ const ZERO = /^0+$/; // `git replace` or a graft in the repo can't swap the commits, trees or blobs scanned here. Written out here because // the hook runs this file alone, fetched from origin (scripts/push-gate-hook.mjs GATE_CODE), so it imports nothing // new; test/git-runner.test.ts checks they are the runner's. -const GIT_FIXED_ARGS = ['--no-replace-objects']; -const GIT_ENV = { ...process.env, GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }; +// The commit-graph is off too (a forged one can skip a commit), in the arguments and, for gitleaks' own gits, as a +// config pair in the environment after any the caller set. +const GIT_FIXED_ARGS = ['--no-replace-objects', '-c', 'core.commitGraph=false']; +const graphOff = (e) => { + const n = Number.isSafeInteger(Number(e.GIT_CONFIG_COUNT ?? '0')) && Number(e.GIT_CONFIG_COUNT ?? '0') >= 0 ? Number(e.GIT_CONFIG_COUNT ?? '0') : 0; + return { ...e, [`GIT_CONFIG_KEY_${n}`]: 'core.commitGraph', [`GIT_CONFIG_VALUE_${n}`]: 'false', GIT_CONFIG_COUNT: String(n + 1) }; +}; +const GIT_ENV = graphOff({ ...process.env, GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); const runGit = (args, opts) => execFileSync('git', [...GIT_FIXED_ARGS, ...args], { ...opts, env: GIT_ENV }); const git = (...a) => runGit(a, { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 }); diff --git a/src/os/git.ts b/src/os/git.ts index f4f71bd45..b09727064 100644 --- a/src/os/git.ts +++ b/src/os/git.ts @@ -13,8 +13,15 @@ // whatever environment the caller passes, so no caller can turn them off. test/git-runner.test.ts fails on a git call // in src/ or scripts/ that doesn't come through here (or scripts/lib/git.cjs, the same settings for scripts). // -// LIMIT: a shallow clone's `.git/shallow` is still read (it is how a shallow clone works, and git writes it itself). -// A program the engine runs that runs git itself (gitleaks) gets these settings only when it is handed gitEnv(). +// The commit-graph file (objects/info/commit-graph, in the shared objects dir) is the same kind of state: git trusts +// its parent links, and a forged one skipped a commit in a range (measured, #927's round-2 check). It is off for every +// git here too (`-c core.commitGraph=false`, and the same pair in the environment for gits a program runs itself). +// +// A shallow clone's `shallow` file is read (it is how a shallow clone works). A reader that must see a range whole +// takes a gitSnapshot: one copy of that file, checked, that every walk it runs reads, however the shared file changes. +// +// LIMIT: a program the engine runs that runs git itself (gitleaks) gets these settings only when it is handed gitEnv(). +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { execFile, execFileSync, spawn, spawnSync } from './child-process.js'; import type { ChildProcess, @@ -34,12 +41,18 @@ import type { StdioNull, StdioPipe, } from 'node:child_process'; -import { devNull } from 'node:os'; -import { join } from 'node:path'; +import { devNull, tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; import type { Readable } from 'node:stream'; -/** Before every git subcommand: replace refs off for this git (git also sets GIT_NO_REPLACE_OBJECTS for the gits it runs). */ -export const GIT_FIXED_ARGS: readonly string[] = Object.freeze(['--no-replace-objects']); +/** + * Before every git subcommand: replace refs off for this git (git also sets GIT_NO_REPLACE_OBJECTS for the gits it + * runs), and the commit-graph off. + */ +export const GIT_FIXED_ARGS: readonly string[] = Object.freeze(['--no-replace-objects', '-c', 'core.commitGraph=false']); + +/** git config every git also gets through its environment (GIT_CONFIG_COUNT pairs), after the caller's own pairs. */ +export const GIT_FIXED_CONFIG: readonly (readonly [string, string])[] = Object.freeze([Object.freeze(['core.commitGraph', 'false'] as const)]); /** * On every git process, over the caller's environment. GIT_GRAFT_FILE is a path under the null device, which can't @@ -50,9 +63,21 @@ export const GIT_FIXED_ENV: Readonly> = Object.freeze({ GIT_GRAFT_FILE: join(devNull, 'no-grafts'), }); -/** `env` (or this process's environment), with the fixed settings on top. */ +/** + * `env` (or this process's environment), with the fixed settings on top and the fixed config pairs after its own + * GIT_CONFIG_COUNT pairs (a later pair wins). A count that isn't a whole number is git's refusal anyway; it is replaced. + */ export function gitEnv(env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv { - return { ...(env ?? process.env), ...GIT_FIXED_ENV }; + const out: NodeJS.ProcessEnv = { ...(env ?? process.env), ...GIT_FIXED_ENV }; + let n = Number(out.GIT_CONFIG_COUNT ?? '0'); + if (!Number.isSafeInteger(n) || n < 0) n = 0; + for (const [k, v] of GIT_FIXED_CONFIG) { + out[`GIT_CONFIG_KEY_${n}`] = k; + out[`GIT_CONFIG_VALUE_${n}`] = v; + n++; + } + out.GIT_CONFIG_COUNT = String(n); + return out; } /** The argument list a git process gets: the fixed arguments, then `args`. */ @@ -110,3 +135,77 @@ export function commandSpawnSync(cmd: string, args: readonly string[], options: ? spawnSync(cmd, gitArgs(args), { ...options, env: gitEnv(options.env) }) : spawnSync(cmd, args, options); } + +const OBJECT_ID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/; + +/** + * The commits a `shallow` file lists, or null when any line isn't exactly a full lowercase object id. git reads the + * file more loosely (an uppercase id, text after the id, a line over 1 KB: each measured to cut a range on git 2.43), + * so a line this can't read the way git might is never taken as "no cut". + */ +export function shallowIds(text: string): string[] | null { + const lines = text.split('\n'); + if (lines.at(-1) === '') lines.pop(); + return lines.every((l) => OBJECT_ID.test(l)) ? lines : null; +} + +/** Variables that would point a git at another repo, worktree, index or shallow file than the snapshot's own. */ +const REPO_VARS = ['GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_COMMON_DIR', 'GIT_OBJECT_DIRECTORY', 'GIT_SHALLOW_FILE', 'GIT_NAMESPACE']; + +/** One fixed view of a repo's history: what every walk of one check reads (#927). */ +export interface GitSnapshot { + /** The environment a git (or a program that runs git, through gitEnv) reads the snapshot with. */ + readonly env: NodeJS.ProcessEnv; + /** The commits the snapshot's shallow file cuts at (empty: not shallow). */ + readonly cut: ReadonlySet; + /** The cut commits inside base..head (object ids): each hides what is below it. */ + cutInRange(base: string, head: string): string[]; + /** Removes the snapshot's directory. */ + dispose(): void; +} + +/** + * A private git directory over `cwd`'s repo: its objects (GIT_OBJECT_DIRECTORY, with their alternates) and one copy + * of its `shallow` file, read once and checked (shallowIds). No refs, no config but its own, no replace refs and no + * grafts. A walk through `env` reads that copy, so writing or removing the repo's own shallow file mid-check changes + * nothing (an agent toggling it every ~3 ms got a refused file past the old before-and-after checks: 23 of 150 runs, + * measured). Documented environment only, not git's internal --shallow-file. Commit ids only: the snapshot has no + * refs, so resolve names in the repo first. Throws, saying why, when the repo or its shallow file can't be read. + */ +export function gitSnapshot(cwd: string): GitSnapshot { + const clean = (): NodeJS.ProcessEnv => { + const e: NodeJS.ProcessEnv = { ...process.env }; + for (const k of REPO_VARS) delete e[k]; + return e; + }; + const [format, objectsPath, shallowPath] = gitExecFileSync(['rev-parse', '--show-object-format', '--git-path', 'objects', '--git-path', 'shallow'], { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).split('\n'); + if (!format || !objectsPath || !shallowPath) throw new Error('git rev-parse gave no object format or paths'); + let text = ''; + try { + text = readFileSync(resolve(cwd, shallowPath), 'utf8'); + } catch (e) { + if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw new Error(`can't read its shallow file: ${(e as Error).message}`); + } + const ids = shallowIds(text); + if (ids === null) throw new Error('its shallow file has a line that isn\'t exactly a full lowercase object id, which git may still read as a cut'); + const dir = mkdtempSync(join(tmpdir(), 'git-snapshot-')); + try { + gitExecFileSync(['init', '-q', '--bare', `--object-format=${format}`, dir], { env: clean(), stdio: ['ignore', 'pipe', 'pipe'] }); + if (ids.length) writeFileSync(join(dir, 'shallow'), `${ids.join('\n')}\n`); + } catch (e) { + rmSync(dir, { recursive: true, force: true }); + throw e; + } + const env: NodeJS.ProcessEnv = { ...clean(), GIT_DIR: dir, GIT_OBJECT_DIRECTORY: resolve(cwd, objectsPath) }; + const cut = new Set(ids); + return { + env, + cut, + cutInRange: (base, head) => { + if (!cut.size) return []; + const listed = gitExecFileSync(['rev-list', `${base}..${head}`], { env, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'] }); + return listed.split('\n').filter((c) => cut.has(c)); + }, + dispose: () => rmSync(dir, { recursive: true, force: true }), + }; +} diff --git a/test/intake-coordinator.test.ts b/test/intake-coordinator.test.ts index e18a93c84..ab4fe0e63 100644 --- a/test/intake-coordinator.test.ts +++ b/test/intake-coordinator.test.ts @@ -178,7 +178,7 @@ test('intake (#333): a bug already fixed on main is answered naming the fixing c git(t.repo, 'update-ref', 'refs/remotes/origin/main', onMain); const spawned = await spawnsDuring(() => t.c.checkIntake()); // Through the git runner (#807): replace refs off. - const check = (id: string) => `execFileSync git --no-replace-objects merge-base --is-ancestor ${id} refs/remotes/origin/main`; + const check = (id: string) => `execFileSync git --no-replace-objects -c core.commitGraph=false merge-base --is-ancestor ${id} refs/remotes/origin/main`; assert.deepEqual(spawned, [check(onMain), check('0a1b2c3d4e5f')], 'the only process: one read-only git check per fixing commit, on the trusted checkout'); assert.equal(t.discord.replies.length, 3); assert.match(t.discord.replies[0]!.text, new RegExp(`already fixed.*\\b${onMain}\\b`), 'confirmed on main: named'); From 3955c238b39b5c1bb6b5644d176e70e0376dbd28 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 00:20:20 -0500 Subject: [PATCH 07/18] push limits: every walk reads one checked snapshot of the history (#927) git reads a shallow file more loosely than round 2's exact match (an uppercase id, text after the id, a line over 1 KB each cut a range), and a shallow file toggled every ~3 ms beat round 2's before-and-after checks (23 of 150 pushProblems runs; 5 of 80 checkedPush calls landed the refused file; both measured by the round-2 check). gitSnapshot (src/os/git.ts) is a private GIT_DIR over the repo's objects (GIT_OBJECT_DIRECTORY) with one copy of its shallow file, read once and parsed strictly: any line that isn't exactly a full lowercase object id is refused. pushProblems resolves base and head in the repo, takes one snapshot, checks the cut and runs every range walk through it; checkedPush takes a fresh one right before the push; scanRange runs gitleaks through one, and a cut inside the range is never "clean". Documented environment only, not --shallow-file. Test (fails without): test/push-snapshot.test.ts, 9 of 9 fail at a1ff0091 (four shallow-line forms, the mid-walk toggle for pushProblems and checkedPush, the forged graph for the limits and gitleaks, gitleaks under a cut). Signed-off-by: Jayden Vasquez --- changelog.d/927-git-runner.md | 6 +- src/push-check.ts | 90 ++++++++++++------ src/scan/secrets.ts | 34 ++++++- test/git-runner.test.ts | 17 ++-- test/push-snapshot.test.ts | 170 ++++++++++++++++++++++++++++++++++ 5 files changed, 277 insertions(+), 40 deletions(-) create mode 100644 test/push-snapshot.test.ts diff --git a/changelog.d/927-git-runner.md b/changelog.d/927-git-runner.md index 60a043f11..fbc00de20 100644 --- a/changelog.d/927-git-runner.md +++ b/changelog.d/927-git-runner.md @@ -14,6 +14,10 @@ owner. - **A shallow cut in a pushed range is refused (#927):** a `.git/shallow` (a shallow clone's, or one written by hand) that cuts the range being pushed made the push limits read only part of it. The push is now refused, and the - message names `git fetch --unshallow`. A shallow clone still pushes changes that lie wholly above its cut. + message names `git fetch --unshallow`. A shallow clone still pushes changes that lie wholly above its cut. The push + limits and the change's secret scan read one checked copy of the shallow file, so changing it while they run + changes nothing, and a shallow file with a line git might read differently is refused. +- **The commit-graph is off for every git the engine runs (#927):** a forged commit-graph could make a walk skip a + commit; git now reads each commit's own parents (measured: about 30 ms more for three walks over 2,900 commits). - **What the owner sees (#927):** the decision diff, the fix-forward culprit diff and the patch hash show a file's lines even when the change's `.gitattributes` marks it `-diff` or `binary`. diff --git a/src/push-check.ts b/src/push-check.ts index 9600dbd93..854c41427 100644 --- a/src/push-check.ts +++ b/src/push-check.ts @@ -5,11 +5,9 @@ // leaves a SKILL.md that fails the skill lint. // The coordinator also runs pushProblems on each change before verifying it, // so a doomed change is caught before a long test run. -import { existsSync, readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; import { sizeFiles } from './coordinator/change-files.js'; import { globToRegExp } from './guardrails/glob.js'; -import { gitExecFileSync, gitSpawnSync } from './os/git.js'; +import { gitExecFileSync, gitSnapshot, gitSpawnSync, type GitSnapshot } from './os/git.js'; import { shownPath } from './path-text.js'; import { lintSkill, SKILL_SECTIONS } from './skill-lint.js'; import { defaultBranchRefusal, type PushGuard } from './push-guard.js'; @@ -31,38 +29,73 @@ const list = (paths: string[], n = 10) => `${paths.slice(0, n).join(', ')}${path /** Changed paths in a problem's text, each as shownPath shows it (#683): `rev-list --objects` prints a name as it is. */ const listPaths = (paths: string[], n = 10) => list(paths.map(shownPath), n); +/** The refusal for a range the push limits can't read whole (#807). */ +const cutRefusal = (why: string) => + `this repo's history is cut inside the range (${why}): the push limits can't read it whole, so nothing is pushed. A shallow clone pushes only changes above its cut; fetch the rest (git fetch --unshallow), or remove a .git/shallow file nobody meant to write`; + +/** base and head as commit ids, read in the repo (the snapshot has no refs). Throws as git does on a bad name. */ +const commitIds = (cwd: string, base: string, head: string): [string, string] => + [base, head].map((r) => git(cwd, ['rev-parse', '--verify', '--end-of-options', `${r}^{commit}`]).trim()) as [string, string]; + /** - * Why the range can't be read whole, or null (#807). A shallow repo's `shallow` file lists commits whose parents git - * then treats as missing, and any agent can write it into the shared repo: a commit listed inside the range cuts the - * walks below short (a refused file in an earlier commit read as 0 problems, measured). A cut outside the range (a - * depth-limited clone's base, the harness-eval replay's clone) leaves the range whole and is fine. Fails closed: a - * shallow file or range it can't read is a refusal too. WHY not `--shallow-file`: it is git's internal option. + * Why base..head can't be read whole in `snap`, or null (#807). A shallow repo's `shallow` file lists commits whose + * parents git treats as missing, and any agent can write it into the shared repo: a commit listed inside the range cut + * the walks short (a refused file in an earlier commit read as 0 problems, measured). A cut outside the range (a + * depth-limited clone's base, the harness-eval replay's clone) leaves the range whole and is fine. */ -export function shallowCut(cwd: string, range: string): string | null { - const refuse = (why: string) => - `this repo's history is cut inside the range (${why}): the push limits can't read it whole, so nothing is pushed. A shallow clone pushes only changes above its cut; fetch the rest (git fetch --unshallow), or remove a .git/shallow file nobody meant to write`; +function cutIn(snap: GitSnapshot, base: string, head: string): string | null { + const inRange = snap.cutInRange(base, head); + return inRange.length ? cutRefusal(`a shallow boundary at ${inRange.slice(0, 3).map((c) => c.slice(0, 12)).join(', ')}`) : null; +} + +/** A gitSnapshot of `cwd`, or the refusal when one can't be taken (a shallow file it can't read the way git might). */ +function snapshotOf(cwd: string): GitSnapshot | string { try { - const file = resolve(cwd, gitExecFileSync(['rev-parse', '--git-path', 'shallow'], { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim()); - if (!existsSync(file)) return null; - const cut = new Set(readFileSync(file, 'utf8').split('\n').map((l) => l.trim()).filter(Boolean)); - if (!cut.size) return null; - const inRange = git(cwd, ['rev-list', range]).split('\n').filter((c) => cut.has(c)); - return inRange.length ? refuse(`a shallow boundary at ${inRange.slice(0, 3).map((c) => c.slice(0, 12)).join(', ')}`) : null; + return gitSnapshot(cwd); } catch (e) { - return refuse(`can't read its shallow file: ${String((e as Error).message).split('\n')[0]}`); + return cutRefusal(String((e as Error).message).split('\n')[0]!); + } +} + +/** + * Why `cwd`'s history is cut inside base..head right now, or null: the check checkedPush makes once more right before + * the push. Fails closed. + */ +export function shallowCut(cwd: string, base: string, head: string): string | null { + const snap = snapshotOf(cwd); + if (typeof snap === 'string') return snap; + try { + const [b, h] = commitIds(cwd, base, head); + return cutIn(snap, b, h); + } finally { + snap.dispose(); } } -/** Every reason the range base..head may not be pushed; empty means it may. */ +/** + * Every reason the range base..head may not be pushed; empty means it may. Every walk of the range reads one + * gitSnapshot, the one whose shallow file was checked, so a cut written or removed while they run changes nothing + * (#927: toggling it every ~3 ms beat the old before-and-after checks in 23 of 150 runs, measured). + */ export function pushProblems(cwd: string, base: string, head: string, limits: PushLimits): string[] { + const snap = snapshotOf(cwd); + if (typeof snap === 'string') return [snap]; + try { + return judge(cwd, snap, ...commitIds(cwd, base, head), limits); + } finally { + snap.dispose(); + } +} + +function judge(cwd: string, snap: GitSnapshot, base: string, head: string, limits: PushLimits): string[] { + const cut = cutIn(snap, base, head); + if (cut) return [cut]; const range = `${base}..${head}`; - // Before the walks (a cut already there) and again after them (one written while they ran): a cut range is judged - // by nothing else. - const before = shallowCut(cwd, range); - if (before) return [before]; + // The range's walks, through the snapshot. + const walk = (args: string[], input?: string) => gitExecFileSync(args, { env: snap.env, encoding: 'utf8', maxBuffer: 256 * MB, stdio: ['pipe', 'pipe', 'pipe'], ...(input !== undefined ? { input } : {}) }); const problems: string[] = []; // Paths touched by any commit in the range, not just the net diff. - const touched = [...new Set(git(cwd, ['log', '--format=', '--name-only', '--no-renames', '--diff-merges=first-parent', range]).split('\n').filter(Boolean))]; + const touched = [...new Set(walk(['log', '--format=', '--name-only', '--no-renames', '--diff-merges=first-parent', range]).split('\n').filter(Boolean))]; const workflows = touched.filter((p) => p.startsWith(WORKFLOWS)); if (workflows.length) problems.push(`changes CI workflows (${listPaths(workflows)}): the harness's GitHub credential has no Workflows permission, so GitHub refuses this push; a human makes workflow changes`); @@ -73,7 +106,7 @@ export function pushProblems(cwd: string, base: string, head: string, limits: Pu // Every blob new in the range, with its path: a file added and then deleted is still pushed. const paths = new Map(); - for (const l of git(cwd, ['rev-list', '--objects', range]).split('\n')) { + for (const l of walk(['rev-list', '--objects', range]).split('\n')) { const [sha, ...p] = l.split(' '); if (sha && p.length) paths.set(sha, p.join(' ')); } @@ -81,7 +114,7 @@ export function pushProblems(cwd: string, base: string, head: string, limits: Pu if (objects.length) { const limit = limits.max_file_mb * MB; const big: string[] = []; - for (const l of git(cwd, ['cat-file', '--batch-check=%(objecttype) %(objectname) %(objectsize)'], `${objects.join('\n')}\n`).split('\n')) { + for (const l of walk(['cat-file', '--batch-check=%(objecttype) %(objectname) %(objectsize)'], `${objects.join('\n')}\n`).split('\n')) { const [type, sha, size] = l.split(' '); if (type === 'blob' && Number(size) > limit) big.push(`${shownPath(paths.get(sha!) ?? sha!)} (${(Number(size) / MB).toFixed(1)} MB)`); } @@ -103,8 +136,7 @@ export function pushProblems(cwd: string, base: string, head: string, limits: Pu if (lint.length) badSkills.push(`${shownPath(p)} (${lint.join('; ')})`); } if (badSkills.length) problems.push(`has skills that fail the skill lint: ${list(badSkills, 5)}. Every SKILL.md needs these sections, none empty: ${SKILL_SECTIONS.join(', ')}`); - const after = shallowCut(cwd, range); - return after ? [after] : problems; + return problems; } export type PushResult = { ok: true } | { ok: false; refused: string[] } | { ok: false; error: string }; @@ -123,7 +155,7 @@ export function checkedPush(o: { cwd: string; remote: string; base: string; head const refused = pushProblems(o.cwd, o.base, o.head, o.limits); if (refused.length) return { ok: false, refused }; // Once more right before the push, in this process: a cut written since the limits ran (#807). - const late = shallowCut(o.cwd, `${o.base}..${o.head}`); + const late = shallowCut(o.cwd, o.base, o.head); if (late) return { ok: false, refused: [late] }; const lease = o.expect ? [`--force-with-lease=${o.ref}:${o.expect}`] : []; const push = gitSpawnSync(['push',...lease, o.remote, `${o.force ? '+' : ''}${o.head}:${o.ref}`], { cwd: o.cwd, encoding: 'utf8' }); diff --git a/src/scan/secrets.ts b/src/scan/secrets.ts index fb9f205e9..ef11a18fc 100644 --- a/src/scan/secrets.ts +++ b/src/scan/secrets.ts @@ -4,7 +4,7 @@ import { spawn, spawnSync } from '../os/child-process.js'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { gitEnv } from '../os/git.js'; +import { gitEnv, gitExecFileSync, gitSnapshot, type GitSnapshot } from '../os/git.js'; import { findTool } from '../tool-path.js'; export interface Finding { @@ -66,7 +66,33 @@ export function scanCommit(cwd: string, includeUnstaged: boolean, binary = findT */ export function scanRange(cwd: string, range: string, binary = findTool('gitleaks')): Promise { if (!binary) return Promise.resolve({ status: 'unavailable', error: 'gitleaks not found on PATH' }); - return scanAsync(binary, (report) => ['git', '--redact', '--no-banner', '--log-opts', range, '--report-format', 'json', '--report-path', report, '--exit-code', String(LEAKS_EXIT), '.'], cwd); + // gitleaks walks the range with git: through one gitSnapshot whose shallow file was checked, so a cut written or + // removed while it runs changes nothing, and a cut inside the range is never "clean" (#927). Commit ids only there. + let snap: GitSnapshot; + let ids: string; + try { + const m = /^([^.].*?)\.\.([^.].*)$/.exec(range); + if (!m) throw new Error(`${range} is not a base..head range`); + const id = (r: string) => gitExecFileSync(['rev-parse', '--verify', '--end-of-options', `${r}^{commit}`], { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); + const [base, head] = [id(m[1]!), id(m[2]!)]; + snap = gitSnapshot(cwd); + const cut = (() => { + try { + return snap.cutInRange(base, head); + } catch (e) { + snap.dispose(); + throw e; + } + })(); + if (cut.length) { + snap.dispose(); + return Promise.resolve({ status: 'unavailable', error: `the repo's history is cut inside ${range} (a shallow boundary at ${cut[0]!.slice(0, 12)}): it can't be scanned whole` }); + } + ids = `${base}..${head}`; + } catch (e) { + return Promise.resolve({ status: 'unavailable', error: `can't read ${range} whole: ${String((e as Error).message).split('\n')[0]}` }); + } + return scanAsync(binary, (report) => ['git', '--redact', '--no-banner', '--log-opts', ids, '--report-format', 'json', '--report-path', report, '--exit-code', String(LEAKS_EXIT), '.'], cwd, snap.env).finally(() => snap.dispose()); } /** Scan one file without blocking (a dashboard upload). Findings name the rule, never the secret (--redact). */ @@ -75,13 +101,13 @@ export function scanFile(path: string, binary = findTool('gitleaks')): Promise ['dir', path, '--redact', '--no-banner', '--max-decode-depth', '2', '--report-format', 'json', '--report-path', report, '--exit-code', String(LEAKS_EXIT)], undefined); } -function scanAsync(binary: string, args: (report: string) => string[], cwd: string | undefined): Promise { +function scanAsync(binary: string, args: (report: string) => string[], cwd: string | undefined, env?: NodeJS.ProcessEnv): Promise { const dir = mkdtempSync(join(tmpdir(), 'scan-')); const report = join(dir, 'report.json'); return new Promise((resolve) => { // gitleaks runs git itself (`git log -p`): with the git runner's settings, so a replace ref or a graft can't swap // the commits it reads (#807). - const child = spawn(binary, args(report), { ...(cwd ? { cwd } : {}), stdio: ['ignore', 'ignore', 'pipe'], env: gitEnv() }); + const child = spawn(binary, args(report), { ...(cwd ? { cwd } : {}), stdio: ['ignore', 'ignore', 'pipe'], env: gitEnv(env) }); let err = ''; child.stderr.on('data', (d: Buffer) => (err = (err + d.toString()).slice(-2000))); const done = (r: ScanResult) => { diff --git a/test/git-runner.test.ts b/test/git-runner.test.ts index 1e8ca847f..d3c9a624c 100644 --- a/test/git-runner.test.ts +++ b/test/git-runner.test.ts @@ -115,7 +115,10 @@ test('the runner\'s settings go on last: a caller\'s environment can\'t turn rep const want = ['keys/secret.pem', 'weights.bin']; assert.deepEqual(names(gitExecFileSync(['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env })), want); assert.deepEqual(names(gitSpawnSync(['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env }).stdout), want); - assert.deepEqual({ ...gitEnv({ GIT_GRAFT_FILE: grafts, GIT_NO_REPLACE_OBJECTS: '0' }) }, { ...GIT_FIXED_ENV }); + const graphOff = { GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'false' }; + assert.deepEqual({ ...gitEnv({ GIT_GRAFT_FILE: grafts, GIT_NO_REPLACE_OBJECTS: '0' }) }, { ...GIT_FIXED_ENV, ...graphOff }); + // A caller's own config pairs stay; the fixed pair goes after them (a later pair wins). + assert.deepEqual({ ...gitEnv({ GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'true' }) }, { ...GIT_FIXED_ENV, GIT_CONFIG_COUNT: '2', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'true', GIT_CONFIG_KEY_1: 'core.commitGraph', GIT_CONFIG_VALUE_1: 'false' }); // Replace refs: `--no-replace-objects` is in the argument list, which no environment takes back. const decoy = git(r.work, 'commit-tree', `${r.base}^{tree}`, '-p', r.base, '-m', 'docs'); git(r.work, 'replace', r.head, decoy); @@ -144,7 +147,9 @@ test('scripts/lib/git.cjs (scripts run without a build) has the engine runner\'s assert.deepEqual([...scripts.GIT_FIXED_ARGS], [...engine.GIT_FIXED_ARGS]); assert.deepEqual({ ...scripts.GIT_FIXED_ENV }, { ...engine.GIT_FIXED_ENV }); // What the files that run on their own write out (ITS_OWN, below) is these values. - assert.deepEqual([...engine.GIT_FIXED_ARGS], ['--no-replace-objects']); + assert.deepEqual(scripts.GIT_FIXED_CONFIG.map((p) => [...p]), engine.GIT_FIXED_CONFIG.map((p) => [...p])); + assert.deepEqual({ ...scripts.gitEnv({ GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'a.b', GIT_CONFIG_VALUE_0: 'c' }) }, { ...engine.gitEnv({ GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'a.b', GIT_CONFIG_VALUE_0: 'c' }) }); + assert.deepEqual([...engine.GIT_FIXED_ARGS], ['--no-replace-objects', '-c', 'core.commitGraph=false']); assert.deepEqual({ ...engine.GIT_FIXED_ENV }, { GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); const r = repo(); const decoy = git(r.work, 'commit-tree', `${r.base}^{tree}`, '-p', r.base, '-m', 'docs'); @@ -183,7 +188,7 @@ const RUNNERS = new Set(['src/os/git.ts', 'scripts/lib/git.cjs']); * A file that runs on its own (copied alone, or fetched alone from origin) can't import a runner: it holds the * settings itself, written out exactly so, and the test below ties them to the runner's values. */ -const ITS_OWN = [/const GIT_FIXED_ARGS = (?:Object\.freeze\()?\['--no-replace-objects'\]/, /GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join\(devNull, 'no-grafts'\)/]; +const ITS_OWN = [/const GIT_FIXED_ARGS = (?:Object\.freeze\()?\['--no-replace-objects', '-c', 'core\.commitGraph=false'\]/, /GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join\(devNull, 'no-grafts'\)/]; /** A runner that takes any command (`run(cmd, args)`), and so gives git the settings by its name. */ const BY_NAME = (fn: string, call: string) => [new RegExp(`${fn}\\(${call}`)]; @@ -201,7 +206,7 @@ const OUTSIDE_THE_RUNNER: Record execFileSync('git', a, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const LIMITS = { ...DEFAULT_PUSH_LIMITS, refuse_paths: ['**/*.pem'] }; +const guard = { defaultBranch: 'main', blocked: () => undefined }; +const TOKEN = 'ghp_' + 'a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8'; + +/** base, K (adds keys/secret.pem and a token), D (drops both), H (a note): the net diff is clean, the history isn't. */ +function repo() { + const root = mkdtempSync(join(tmpdir(), 'push-snapshot-')); + const work = join(root, 'work'); + const remote = join(root, 'remote.git'); + git(root, 'init', '-q', '--bare', '-b', 'main', remote); + git(root, 'init', '-q', '-b', 'main', work); + for (const [k, v] of [['user.email', 'bot@example.com'], ['user.name', 'bot'], ['commit.gpgsign', 'false']]) git(work, 'config', k!, v!); + writeFileSync(join(work, 'README.md'), 'hello\n'); + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', 'base'); + const base = git(work, 'rev-parse', 'HEAD'); + git(work, 'push', '-q', remote, 'main'); + mkdirSync(join(work, 'keys')); + writeFileSync(join(work, 'keys', 'secret.pem'), 'k\n'); + writeFileSync(join(work, 'c.js'), `export const token = '${TOKEN}';\n`); + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', 'add a key'); + git(work, 'rm', '-q', 'keys/secret.pem', 'c.js'); + git(work, 'commit', '-q', '-m', 'drop it'); + const drop = git(work, 'rev-parse', 'HEAD'); + writeFileSync(join(work, 'notes.txt'), 'n\n'); + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', 'notes'); + const head = git(work, 'rev-parse', 'HEAD'); + return { work, remote, base, drop, head, shallow: join(work, '.git', 'shallow') }; +} + +const keyFound = (p: string[]) => p.some((x) => /keys\/secret\.pem/.test(x)); +const pushedKey = (remote: string, head: string) => { + try { + git(remote, 'cat-file', '-e', `${head}~2:keys/secret.pem`); + return true; + } catch { + return false; + } +}; + +// ---- The shallow file, read at least as loosely as git reads it. + +for (const [what, line] of [ + ['an uppercase id', (id: string) => id.toUpperCase()], + ['text right after the id', (id: string) => `${id}zz`], + ['a word after the id', (id: string) => `${id} comment`], + ['a line over 1 KB', (id: string) => `${id}${' '.repeat(1100)}x`], +] as const) { + test(`a shallow file line with ${what} is refused, never read as "no cut"`, () => { + const r = repo(); + writeFileSync(r.shallow, `${line(r.drop)}\n`); + const p = pushProblems(r.work, r.base, r.head, LIMITS); + assert.equal(p.length, 1, p.join('\n')); + assert.match(p[0]!, /history is cut inside the range \(its shallow file has a line that isn't exactly a full lowercase object id/); + }); +} + +// ---- One snapshot for every walk: a shallow file written mid-check changes nothing. + +/** Runs `fn` with node:child_process's execFileSync wrapped: `during(args)` runs before each call it is handed. */ +async function withExecHook(during: (args: string[]) => (() => void) | void, fn: () => T): Promise { + const cp = createRequire(import.meta.url)('node:child_process') as Record unknown>; + const orig = cp.execFileSync!; + cp.execFileSync = (...a: unknown[]) => { + const after = during(Array.isArray(a[1]) ? a[1].map(String) : []); + try { + return orig(...a); + } finally { + after?.(); + } + }; + syncBuiltinESMExports(); + try { + return fn(); + } finally { + cp.execFileSync = orig; + syncBuiltinESMExports(); + } +} + +/** A cut at D written into the repo while the range's `log` walk runs, and removed again as soon as it ends. */ +const toggleDuringLog = (r: ReturnType) => (args: string[]) => { + if (!(args.includes('log') && args.includes('--name-only'))) return; + writeFileSync(r.shallow, `${r.drop}\n`); + return () => rmSync(r.shallow, { force: true }); +}; + +test('a shallow file written while the walks run, and removed before they end, can\'t hide the key from pushProblems', async () => { + const r = repo(); + const p = await withExecHook(toggleDuringLog(r), () => pushProblems(r.work, r.base, r.head, LIMITS)); + assert.ok(keyFound(p), p.join('\n')); + assert.ok(!existsSync(r.shallow)); +}); + +test('the same toggle can\'t get checkedPush to land the key on the remote', async () => { + const r = repo(); + const res = await withExecHook(toggleDuringLog(r), () => checkedPush({ cwd: r.work, remote: r.remote, base: r.base, head: r.head, ref: 'refs/heads/task', limits: LIMITS, guard, force: true })); + assert.ok(!res.ok && 'refused' in res && keyFound(res.refused), JSON.stringify(res)); + assert.equal(pushedKey(r.remote, r.head), false); +}); + +// ---- A forged commit-graph. + +/** Writes the repo's commit-graph, then makes it say D's first parent is the base: K, which added the key, is skipped. */ +function forgeGraph(r: ReturnType) { + git(r.work, 'commit-graph', 'write', '--reachable', '--no-progress'); + const file = join(r.work, '.git', 'objects', 'info', 'commit-graph'); + const buf = Buffer.from(readFileSync(file)); + const chunks: Record = {}; + for (let i = 0; i < buf[6]!; i++) { + const o = 8 + i * 12; + chunks[buf.toString('latin1', o, o + 4)] = Number(buf.readBigUInt64BE(o + 4)); + } + const n = buf.readUInt32BE(chunks.OIDF! + 255 * 4); + const idx = (hex: string) => [...Array(n).keys()].find((i) => buf.toString('hex', chunks.OIDL! + i * 20, chunks.OIDL! + i * 20 + 20) === hex)!; + buf.writeUInt32BE(idx(r.base), chunks.CDAT! + idx(r.drop) * 36 + 20); + createHash('sha1').update(buf.subarray(0, buf.length - 20)).digest().copy(buf, buf.length - 20); + chmodSync(file, 0o644); + writeFileSync(file, buf); + // Plain git now believes it. + assert.equal(git(r.work, 'rev-list', `${r.base}..${r.head}`).split('\n').length, 2, 'the forged graph hides K from plain git'); +} + +test('a forged commit-graph parent link can\'t hide the key from pushProblems or let checkedPush land it', () => { + const r = repo(); + forgeGraph(r); + assert.ok(keyFound(pushProblems(r.work, r.base, r.head, LIMITS))); + const res = checkedPush({ cwd: r.work, remote: r.remote, base: r.base, head: r.head, ref: 'refs/heads/task', limits: LIMITS, guard, force: true }); + assert.ok(!res.ok && 'refused' in res && keyFound(res.refused), JSON.stringify(res)); + assert.equal(pushedKey(r.remote, r.head), false); +}); + +test('gitleaks, which runs git itself, reads past a forged commit-graph too', { skip: !findTool('gitleaks') && 'gitleaks not installed' }, async () => { + const { scanRange } = await import('../src/scan/secrets.js'); + const r = repo(); + assert.equal((await scanRange(r.work, `${r.base}..${r.head}`)).status, 'leaks'); + forgeGraph(r); + assert.equal((await scanRange(r.work, `${r.base}..${r.head}`)).status, 'leaks'); +}); + +test('gitleaks under a shallow cut inside the range is never "clean"; a ref name in the range still works', { skip: !findTool('gitleaks') && 'gitleaks not installed' }, async () => { + const { scanRange } = await import('../src/scan/secrets.js'); + const r = repo(); + assert.equal((await scanRange(r.work, `${r.base}..HEAD`)).status, 'leaks'); + writeFileSync(r.shallow, `${r.drop}\n`); + const cut = await scanRange(r.work, `${r.base}..${r.head}`); + assert.equal(cut.status, 'unavailable', JSON.stringify(cut)); +}); From 6aa0853b193ec0cda37c115b7066d124fae6b5f9 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 00:40:35 -0500 Subject: [PATCH 08/18] git runner: gitEnv reads the caller's config count from its own keys only (#927) CI at 3955c238 failed test/child-env-sites.test.ts (#846): with GIT_CONFIG_COUNT on Object.prototype, gitEnv read it through the prototype, numbered its pair 1 and left pair 0 unset, so git exited 128. The count is now read only when it is the environment's own key; the same in scripts/lib/git.cjs and the push gate. Test (fails without): test/child-env-sites.test.ts (2 cases, red in run 39549011913) and a polluted-prototype case in test/git-runner.test.ts. Signed-off-by: Jayden Vasquez --- scripts/lib/git.cjs | 3 ++- scripts/push-gate.mjs | 3 ++- src/os/git.ts | 3 ++- test/git-runner.test.ts | 8 ++++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/lib/git.cjs b/scripts/lib/git.cjs index 5fa6067b8..57f46d121 100644 --- a/scripts/lib/git.cjs +++ b/scripts/lib/git.cjs @@ -16,7 +16,8 @@ const GIT_FIXED_CONFIG = Object.freeze([Object.freeze(['core.commitGraph', 'fals /** `env` (or this process's environment), with the fixed settings on top and the fixed config pairs after its own. */ const gitEnv = (env) => { const out = { ...(env || process.env), ...GIT_FIXED_ENV }; - let n = Number(out.GIT_CONFIG_COUNT === undefined ? '0' : out.GIT_CONFIG_COUNT); + // Own keys only: an unset count must never be read through Object.prototype (#846). + let n = Number(Object.hasOwn(out, 'GIT_CONFIG_COUNT') ? out.GIT_CONFIG_COUNT : '0'); if (!Number.isSafeInteger(n) || n < 0) n = 0; for (const [k, v] of GIT_FIXED_CONFIG) { out[`GIT_CONFIG_KEY_${n}`] = k; diff --git a/scripts/push-gate.mjs b/scripts/push-gate.mjs index e41b18c0b..02808f782 100644 --- a/scripts/push-gate.mjs +++ b/scripts/push-gate.mjs @@ -32,7 +32,8 @@ const ZERO = /^0+$/; // config pair in the environment after any the caller set. const GIT_FIXED_ARGS = ['--no-replace-objects', '-c', 'core.commitGraph=false']; const graphOff = (e) => { - const n = Number.isSafeInteger(Number(e.GIT_CONFIG_COUNT ?? '0')) && Number(e.GIT_CONFIG_COUNT ?? '0') >= 0 ? Number(e.GIT_CONFIG_COUNT ?? '0') : 0; + const c = Number(Object.hasOwn(e, 'GIT_CONFIG_COUNT') ? e.GIT_CONFIG_COUNT : '0'); + const n = Number.isSafeInteger(c) && c >= 0 ? c : 0; return { ...e, [`GIT_CONFIG_KEY_${n}`]: 'core.commitGraph', [`GIT_CONFIG_VALUE_${n}`]: 'false', GIT_CONFIG_COUNT: String(n + 1) }; }; const GIT_ENV = graphOff({ ...process.env, GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); diff --git a/src/os/git.ts b/src/os/git.ts index b09727064..42b0d561f 100644 --- a/src/os/git.ts +++ b/src/os/git.ts @@ -69,7 +69,8 @@ export const GIT_FIXED_ENV: Readonly> = Object.freeze({ */ export function gitEnv(env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const out: NodeJS.ProcessEnv = { ...(env ?? process.env), ...GIT_FIXED_ENV }; - let n = Number(out.GIT_CONFIG_COUNT ?? '0'); + // Own keys only: an unset count must never be read through Object.prototype (#846). + let n = Number(Object.hasOwn(out, 'GIT_CONFIG_COUNT') ? out.GIT_CONFIG_COUNT : '0'); if (!Number.isSafeInteger(n) || n < 0) n = 0; for (const [k, v] of GIT_FIXED_CONFIG) { out[`GIT_CONFIG_KEY_${n}`] = k; diff --git a/test/git-runner.test.ts b/test/git-runner.test.ts index d3c9a624c..22b428f91 100644 --- a/test/git-runner.test.ts +++ b/test/git-runner.test.ts @@ -119,6 +119,14 @@ test('the runner\'s settings go on last: a caller\'s environment can\'t turn rep assert.deepEqual({ ...gitEnv({ GIT_GRAFT_FILE: grafts, GIT_NO_REPLACE_OBJECTS: '0' }) }, { ...GIT_FIXED_ENV, ...graphOff }); // A caller's own config pairs stay; the fixed pair goes after them (a later pair wins). assert.deepEqual({ ...gitEnv({ GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'true' }) }, { ...GIT_FIXED_ENV, GIT_CONFIG_COUNT: '2', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'true', GIT_CONFIG_KEY_1: 'core.commitGraph', GIT_CONFIG_VALUE_1: 'false' }); + // A count on Object.prototype is never read (#846): the fixed pair is pair 0, and nothing else is counted. + const proto = Object.prototype as Record; + proto.GIT_CONFIG_COUNT = '1'; + try { + assert.deepEqual({ ...gitEnv({}) }, { ...GIT_FIXED_ENV, ...graphOff }); + } finally { + delete proto.GIT_CONFIG_COUNT; + } // Replace refs: `--no-replace-objects` is in the argument list, which no environment takes back. const decoy = git(r.work, 'commit-tree', `${r.base}^{tree}`, '-p', r.base, '-m', 'docs'); git(r.work, 'replace', r.head, decoy); From 0455bebbdf98c3ad9f19c1949b579b0662aecca4 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 01:48:40 -0500 Subject: [PATCH 09/18] push limits: a shallow line is one object id of the repo's own format, nothing else (#927) shallowIds took any 40- or 64-hex line as an id, whatever the repo's format. In a sha1 repo git reads a 64-hex line as a cut at its first 40 hex digits, so the line hid the commit that added a refused file from pushProblems, and the snapshot copied the same cut. A process toggling the line got the file onto the remote in 34 of 80 checkedPush calls (measured by the round-3 check). In a sha256 repo git refuses a 40-hex line, and pushProblems threw instead of refusing. shallowIds now takes the format gitSnapshot already reads and accepts exactly 40 lowercase hex digits for sha1 and 64 for sha256; any other line, or a format it doesn't know, is a refusal. The new test file also holds round 2's 1 KB form (an id outside the range, 983 characters, then an id inside it), which git reads as a cut of its own. Test (fails without): test/push-shallow-format.test.ts. At 6aa0853b, 4 of 6 fail: the two 64-hex lines in a sha1 repo (expected 1, actual 0), the line removed right before the push's spawn ({"ok":true}, the push landed), and the sha256 repo's 40-hex line (a throw). At a1ff0091, the 1 KB test fails for its own reason (expected 1, actual 0). Signed-off-by: Jayden Vasquez --- src/os/git.ts | 21 +++-- test/push-shallow-format.test.ts | 139 +++++++++++++++++++++++++++++++ test/push-snapshot.test.ts | 2 + 3 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 test/push-shallow-format.test.ts diff --git a/src/os/git.ts b/src/os/git.ts index 42b0d561f..30a0230c8 100644 --- a/src/os/git.ts +++ b/src/os/git.ts @@ -137,17 +137,22 @@ export function commandSpawnSync(cmd: string, args: readonly string[], options: : spawnSync(cmd, args, options); } -const OBJECT_ID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/; +/** A full lowercase object id of each object format git names (`rev-parse --show-object-format`): nothing else. */ +const OBJECT_ID = new Map([['sha1', /^[0-9a-f]{40}$/], ['sha256', /^[0-9a-f]{64}$/]]); /** - * The commits a `shallow` file lists, or null when any line isn't exactly a full lowercase object id. git reads the - * file more loosely (an uppercase id, text after the id, a line over 1 KB: each measured to cut a range on git 2.43), - * so a line this can't read the way git might is never taken as "no cut". + * The commits a `shallow` file lists, or null when any line isn't exactly one full lowercase object id of the repo's + * own `format`, or the format is one this doesn't know. git reads the file more loosely: an uppercase id, text after + * the id, a line over 1 KB, and in a sha1 repo a 64-hex line, read as its first 40 (each measured to cut a range on + * git 2.43; the last got a refused file onto the remote in 34 of 80 pushes, #927 round 3). So a line this can't read + * the way git might is never taken as "no cut". */ -export function shallowIds(text: string): string[] | null { +export function shallowIds(text: string, format: string): string[] | null { + const id = OBJECT_ID.get(format); + if (!id) return null; const lines = text.split('\n'); if (lines.at(-1) === '') lines.pop(); - return lines.every((l) => OBJECT_ID.test(l)) ? lines : null; + return lines.every((l) => id.test(l)) ? lines : null; } /** Variables that would point a git at another repo, worktree, index or shallow file than the snapshot's own. */ @@ -187,8 +192,8 @@ export function gitSnapshot(cwd: string): GitSnapshot { } catch (e) { if ((e as NodeJS.ErrnoException).code !== 'ENOENT') throw new Error(`can't read its shallow file: ${(e as Error).message}`); } - const ids = shallowIds(text); - if (ids === null) throw new Error('its shallow file has a line that isn\'t exactly a full lowercase object id, which git may still read as a cut'); + const ids = shallowIds(text, format); + if (ids === null) throw new Error(`its shallow file has a line that isn't exactly a full lowercase object id of this repo's format (${format}), which git may still read as a cut`); const dir = mkdtempSync(join(tmpdir(), 'git-snapshot-')); try { gitExecFileSync(['init', '-q', '--bare', `--object-format=${format}`, dir], { env: clean(), stdio: ['ignore', 'pipe', 'pipe'] }); diff --git a/test/push-shallow-format.test.ts b/test/push-shallow-format.test.ts new file mode 100644 index 000000000..767786f5c --- /dev/null +++ b/test/push-shallow-format.test.ts @@ -0,0 +1,139 @@ +// Round 4 of #927: a shallow file line is read as one object id of the repo's own format, and nothing else. +// - In a sha1 repo git reads a 64-hex line as a cut at its first 40 hex digits. Round 3 took any 40- or 64-hex line as +// an id, so the line hid the commit that added a refused file, and the snapshot copied the same cut. A separate +// process toggling the line got the file onto the remote in 34 of 80 checkedPush calls (measured by the round-3 check). +// - In a sha256 repo git refuses a 40-hex line (`fatal: bad shallow line`), and round 3's pushProblems threw instead of +// refusing. +// - Round 2's 1 KB form: git reads the file 1 KB at a time, so an id 1,023 characters into one line is a line of its own +// to git. An exact-match check took the whole line as no id at all. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire, syncBuiltinESMExports } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { checkedPush, DEFAULT_PUSH_LIMITS, pushProblems } from '../src/push-check.js'; + +const git = (cwd: string, ...a: string[]) => execFileSync('git', a, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const LIMITS = { ...DEFAULT_PUSH_LIMITS, refuse_paths: ['**/*.pem'] }; +const guard = { defaultBranch: 'main', blocked: () => undefined }; +const CUT = /history is cut inside the range \(its shallow file has a line that isn't exactly a full lowercase object id/; + +/** base, K (adds keys/secret.pem), D (drops it), H (a note), in a repo of `format`: the net diff is clean, the history isn't. */ +function repo(format: 'sha1' | 'sha256' = 'sha1') { + const root = mkdtempSync(join(tmpdir(), 'push-shallow-format-')); + const work = join(root, 'work'); + const remote = join(root, 'remote.git'); + git(root, 'init', '-q', '--bare', `--object-format=${format}`, '-b', 'main', remote); + git(root, 'init', '-q', `--object-format=${format}`, '-b', 'main', work); + for (const [k, v] of [['user.email', 'bot@example.com'], ['user.name', 'bot'], ['commit.gpgsign', 'false']]) git(work, 'config', k!, v!); + writeFileSync(join(work, 'README.md'), 'hello\n'); + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', 'base'); + const base = git(work, 'rev-parse', 'HEAD'); + git(work, 'push', '-q', remote, 'main'); + mkdirSync(join(work, 'keys')); + writeFileSync(join(work, 'keys', 'secret.pem'), 'k\n'); + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', 'add a key'); + git(work, 'rm', '-q', 'keys/secret.pem'); + git(work, 'commit', '-q', '-m', 'drop it'); + const drop = git(work, 'rev-parse', 'HEAD'); + writeFileSync(join(work, 'notes.txt'), 'n\n'); + git(work, 'add', '-A'); + git(work, 'commit', '-q', '-m', 'notes'); + const head = git(work, 'rev-parse', 'HEAD'); + return { work, remote, base, drop, head, shallow: join(work, '.git', 'shallow') }; +} + +const plainCount = (r: ReturnType) => git(r.work, 'rev-list', '--count', `${r.base}..${r.head}`); +const pushedKey = (r: ReturnType) => { + try { + git(r.remote, 'cat-file', '-e', `${r.head}~2:keys/secret.pem`); + return true; + } catch { + return false; + } +}; +const push = (r: ReturnType) => + checkedPush({ cwd: r.work, remote: r.remote, base: r.base, head: r.head, ref: 'refs/heads/task', limits: LIMITS, guard, force: true }); + +/** Runs `fn` with node:child_process's spawnSync wrapped: `before(args)` runs before each call it is handed. */ +function withSpawnHook(before: (args: string[]) => void, fn: () => T): T { + const cp = createRequire(import.meta.url)('node:child_process') as Record unknown>; + const orig = cp.spawnSync!; + cp.spawnSync = (...a: unknown[]) => { + before(Array.isArray(a[1]) ? a[1].map(String) : []); + return orig(...a); + }; + syncBuiltinESMExports(); + try { + return fn(); + } finally { + cp.spawnSync = orig; + syncBuiltinESMExports(); + } +} + +// ---- A sha1 repo: a 64-hex line is a cut at its first 40 to git. + +for (const [what, at] of [['the drop', 'drop'], ['the head', 'head']] as const) { + test(`a sha1 repo whose shallow line is ${what}'s id plus 24 hex digits is refused, never 0 problems`, () => { + const r = repo(); + writeFileSync(r.shallow, `${r[at]}${'0'.repeat(24)}\n`); + assert.notEqual(plainCount(r), '3', 'git reads the line as a cut'); + const p = pushProblems(r.work, r.base, r.head, LIMITS); + assert.equal(p.length, 1, p.join('\n')); + assert.match(p[0]!, CUT); + assert.match(p[0]!, /\(sha1\)/); + }); +} + +test('the same 64-hex line, removed right before the push\'s spawn, lands nothing on the remote', () => { + const r = repo(); + writeFileSync(r.shallow, `${r.drop}${'0'.repeat(24)}\n`); + const res = withSpawnHook((args) => { + if (args.includes('push')) rmSync(r.shallow, { force: true }); + }, () => push(r)); + assert.ok(!res.ok && 'refused' in res && CUT.test(res.refused.join('\n')), JSON.stringify(res)); + assert.equal(pushedKey(r), false); +}); + +// ---- A sha256 repo: a 40-hex line is a refusal, not an exception. + +test('a sha256 repo whose shallow line is 40 hex digits gets a refusal from pushProblems and checkedPush, not a throw', () => { + const r = repo('sha256'); + assert.equal(r.drop.length, 64); + writeFileSync(r.shallow, `${r.drop.slice(0, 40)}\n`); + let p: string[] = []; + assert.doesNotThrow(() => { p = pushProblems(r.work, r.base, r.head, LIMITS); }); + assert.equal(p.length, 1, p.join('\n')); + assert.match(p[0]!, CUT); + assert.match(p[0]!, /\(sha256\)/); + let res: ReturnType | undefined; + assert.doesNotThrow(() => { res = push(r); }); + assert.ok(res && !res.ok && 'refused' in res && CUT.test(res.refused.join('\n')), JSON.stringify(res)); + assert.equal(pushedKey(r), false); +}); + +test('a sha256 repo still reads its own 64-hex line: a cut outside the range pushes, one inside is refused', () => { + const r = repo('sha256'); + writeFileSync(r.shallow, `${r.drop}\n`); + const inside = pushProblems(r.work, r.base, r.head, LIMITS); + assert.equal(inside.length, 1, inside.join('\n')); + assert.match(inside[0]!, /a shallow boundary at /); + writeFileSync(r.shallow, `${r.base}\n`); + assert.ok(pushProblems(r.work, r.base, r.head, LIMITS).some((x) => /keys\/secret\.pem/.test(x))); +}); + +// ---- Round 2's 1 KB form: an id outside the range, 983 characters, then an id inside it, on one line. + +test('a shallow line that hides an in-range id 1,023 characters in is refused, never 0 problems', () => { + const r = repo(); + writeFileSync(r.shallow, `${r.base}${'x'.repeat(983)}${r.drop}\n`); + assert.notEqual(plainCount(r), '3', 'git reads the drop as a cut of its own'); + const p = pushProblems(r.work, r.base, r.head, LIMITS); + assert.equal(p.length, 1, p.join('\n')); + assert.match(p[0]!, /history is cut inside the range/); +}); diff --git a/test/push-snapshot.test.ts b/test/push-snapshot.test.ts index a6ff9238f..6834e5980 100644 --- a/test/push-snapshot.test.ts +++ b/test/push-snapshot.test.ts @@ -66,6 +66,8 @@ for (const [what, line] of [ ['an uppercase id', (id: string) => id.toUpperCase()], ['text right after the id', (id: string) => `${id}zz`], ['a word after the id', (id: string) => `${id} comment`], + // A line git itself rejects, so round 2 refused it too (only the message differed). Round 2's 1 KB form, which git + // reads as a cut and round 2 took as no cut, is in test/push-shallow-format.test.ts. ['a line over 1 KB', (id: string) => `${id}${' '.repeat(1100)}x`], ] as const) { test(`a shallow file line with ${what} is refused, never read as "no cut"`, () => { From 541c64241cfd966689871e9735a516c11a3cb156 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 02:16:51 -0500 Subject: [PATCH 10/18] test: the mid-walk shallow toggle reaches #928's plumbing walks, and says it did (#927) push-snapshot's two toggle tests wrote the cut only while `git log --name-only` ran, through execFileSync. After the main merge the push limits read names with #928's rev-list and diff-tree, through spawnSync, and no `git log` is left in push-check.ts or change-files.ts (measured by grep), so the hook never fired and the tests passed without toggling anything (inferred). The hook now wraps execFileSync and spawnSync, toggles during every rev-list and diff-tree (and the old log walk), and each test asserts it fired. Test (fails without): no src change. The hook's own `fired > 0` assertion fails if it reaches no walk. A scratch probe (names-probe.mjs) measured that the same toggle hides keys/secret.pem from rangeNames without the snapshot's env, and that with the env it is listed. Signed-off-by: Jayden Vasquez --- test/push-snapshot.test.ts | 54 +++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/test/push-snapshot.test.ts b/test/push-snapshot.test.ts index 6834e5980..8b09124c3 100644 --- a/test/push-snapshot.test.ts +++ b/test/push-snapshot.test.ts @@ -81,44 +81,62 @@ for (const [what, line] of [ // ---- One snapshot for every walk: a shallow file written mid-check changes nothing. -/** Runs `fn` with node:child_process's execFileSync wrapped: `during(args)` runs before each call it is handed. */ +/** + * Runs `fn` with node:child_process's execFileSync and spawnSync wrapped: `during(args)` runs before each call either + * is handed. + */ async function withExecHook(during: (args: string[]) => (() => void) | void, fn: () => T): Promise { const cp = createRequire(import.meta.url)('node:child_process') as Record unknown>; - const orig = cp.execFileSync!; - cp.execFileSync = (...a: unknown[]) => { - const after = during(Array.isArray(a[1]) ? a[1].map(String) : []); - try { - return orig(...a); - } finally { - after?.(); - } - }; + const orig = { execFileSync: cp.execFileSync!, spawnSync: cp.spawnSync! }; + for (const name of ['execFileSync', 'spawnSync'] as const) { + cp[name] = (...a: unknown[]) => { + const after = during(Array.isArray(a[1]) ? a[1].map(String) : []); + try { + return orig[name](...a); + } finally { + after?.(); + } + }; + } syncBuiltinESMExports(); try { return fn(); } finally { - cp.execFileSync = orig; + Object.assign(cp, orig); syncBuiltinESMExports(); } } -/** A cut at D written into the repo while the range's `log` walk runs, and removed again as soon as it ends. */ -const toggleDuringLog = (r: ReturnType) => (args: string[]) => { - if (!(args.includes('log') && args.includes('--name-only'))) return; - writeFileSync(r.shallow, `${r.drop}\n`); - return () => rmSync(r.shallow, { force: true }); +/** + * A cut at D written into the repo while each walk of the range runs (`git log --name-only` before #928; its + * `rev-list` and `diff-tree --stdin` after it; `rev-list --objects`), and removed again as soon as it ends. `fired` + * counts the walks it hit, so the test can't pass with the hook never reaching a walk. + */ +const toggleDuringNameWalk = (r: ReturnType) => { + const hook = (args: string[]) => { + if (!(args.includes('rev-list') || args.includes('diff-tree') || (args.includes('log') && args.includes('--name-only')))) return; + hook.fired++; + writeFileSync(r.shallow, `${r.drop}\n`); + return () => rmSync(r.shallow, { force: true }); + }; + hook.fired = 0; + return hook; }; test('a shallow file written while the walks run, and removed before they end, can\'t hide the key from pushProblems', async () => { const r = repo(); - const p = await withExecHook(toggleDuringLog(r), () => pushProblems(r.work, r.base, r.head, LIMITS)); + const hook = toggleDuringNameWalk(r); + const p = await withExecHook(hook, () => pushProblems(r.work, r.base, r.head, LIMITS)); + assert.ok(hook.fired > 0, 'the hook reached the name walk'); assert.ok(keyFound(p), p.join('\n')); assert.ok(!existsSync(r.shallow)); }); test('the same toggle can\'t get checkedPush to land the key on the remote', async () => { const r = repo(); - const res = await withExecHook(toggleDuringLog(r), () => checkedPush({ cwd: r.work, remote: r.remote, base: r.base, head: r.head, ref: 'refs/heads/task', limits: LIMITS, guard, force: true })); + const hook = toggleDuringNameWalk(r); + const res = await withExecHook(hook, () => checkedPush({ cwd: r.work, remote: r.remote, base: r.base, head: r.head, ref: 'refs/heads/task', limits: LIMITS, guard, force: true })); + assert.ok(hook.fired > 0, 'the hook reached the name walk'); assert.ok(!res.ok && 'refused' in res && keyFound(res.refused), JSON.stringify(res)); assert.equal(pushedKey(r.remote, r.head), false); }); From 7696ed8b20cfff569d821323126743c5cfdb4d4e Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 17:23:02 -0500 Subject: [PATCH 11/18] git runner: a GIT_CONFIG_COUNT git refuses stays refused (#927) gitEnv read the caller's GIT_CONFIG_COUNT with Number(), which accepts what git's strtoul doesn't: `0x1`, `1e0` and `1 ` became 1 and `abc` became 0, so the runner turned git's "bogus count" refusal into a run, and for the first three into a run that applied the caller's first pair (measured by #927's second review, A8; #1001 item 1). A count is now read as git reads it (optional leading whitespace and `+`, then digits; empty is 0). Any other count is left as it is, with no pair added, so git still refuses it; the fixed arguments still turn the commit-graph off for the runner's own gits. The same in scripts/lib/git.cjs and in the push gate's graphOff (scripts/push-gate.mjs). Test (fails without): test/git-config-count.test.ts. At 5d8c98ab, 7 of 12 fail: the seven refused counts (`0x1`, `1e0`, `1 `, `abc`, `-1`, `1.5`, a lone space) got a pair and a new count; the five accepted forms pass either way. Signed-off-by: Jayden Vasquez --- scripts/lib/git.cjs | 13 +++++++++--- scripts/push-gate.mjs | 6 ++++-- src/os/git.ts | 16 +++++++++++--- test/git-config-count.test.ts | 39 +++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 test/git-config-count.test.ts diff --git a/scripts/lib/git.cjs b/scripts/lib/git.cjs index 57f46d121..b2ba36bb2 100644 --- a/scripts/lib/git.cjs +++ b/scripts/lib/git.cjs @@ -13,12 +13,19 @@ const GIT_FIXED_ENV = Object.freeze({ GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FIL /** git config every git also gets through its environment (GIT_CONFIG_COUNT pairs), after the caller's own pairs. */ const GIT_FIXED_CONFIG = Object.freeze([Object.freeze(['core.commitGraph', 'false'])]); -/** `env` (or this process's environment), with the fixed settings on top and the fixed config pairs after its own. */ +/** A GIT_CONFIG_COUNT as git reads it (strtoul, base 10, the whole value); empty or unset is 0. */ +const CONFIG_COUNT = /^[ \t\n\v\f\r]*\+?[0-9]+$/; + +/** + * `env` (or this process's environment), with the fixed settings on top and the fixed config pairs after its own. A + * count git refuses is left as it is, with no pair added, so git still refuses it (as src/os/git.ts gitEnv). + */ const gitEnv = (env) => { const out = { ...(env || process.env), ...GIT_FIXED_ENV }; // Own keys only: an unset count must never be read through Object.prototype (#846). - let n = Number(Object.hasOwn(out, 'GIT_CONFIG_COUNT') ? out.GIT_CONFIG_COUNT : '0'); - if (!Number.isSafeInteger(n) || n < 0) n = 0; + const raw = Object.hasOwn(out, 'GIT_CONFIG_COUNT') ? out.GIT_CONFIG_COUNT : undefined; + let n = raw === undefined || raw === '' ? 0 : CONFIG_COUNT.test(raw) ? Number(raw) : NaN; + if (!Number.isSafeInteger(n)) return out; for (const [k, v] of GIT_FIXED_CONFIG) { out[`GIT_CONFIG_KEY_${n}`] = k; out[`GIT_CONFIG_VALUE_${n}`] = v; diff --git a/scripts/push-gate.mjs b/scripts/push-gate.mjs index e6ecd34d6..a031a590e 100644 --- a/scripts/push-gate.mjs +++ b/scripts/push-gate.mjs @@ -31,9 +31,11 @@ const ZERO = /^0+$/; // The commit-graph is off too (a forged one can skip a commit), in the arguments and, for gitleaks' own gits, as a // config pair in the environment after any the caller set. const GIT_FIXED_ARGS = ['--no-replace-objects', '-c', 'core.commitGraph=false']; +// A count git refuses is left as it is, with no pair added, so git still refuses it (as src/os/git.ts gitEnv). const graphOff = (e) => { - const c = Number(Object.hasOwn(e, 'GIT_CONFIG_COUNT') ? e.GIT_CONFIG_COUNT : '0'); - const n = Number.isSafeInteger(c) && c >= 0 ? c : 0; + const raw = Object.hasOwn(e, 'GIT_CONFIG_COUNT') ? e.GIT_CONFIG_COUNT : undefined; + const n = raw === undefined || raw === '' ? 0 : /^[ \t\n\v\f\r]*\+?[0-9]+$/.test(raw) ? Number(raw) : NaN; + if (!Number.isSafeInteger(n)) return e; return { ...e, [`GIT_CONFIG_KEY_${n}`]: 'core.commitGraph', [`GIT_CONFIG_VALUE_${n}`]: 'false', GIT_CONFIG_COUNT: String(n + 1) }; }; const GIT_ENV = graphOff({ ...process.env, GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); diff --git a/src/os/git.ts b/src/os/git.ts index 30a0230c8..3befbb8c5 100644 --- a/src/os/git.ts +++ b/src/os/git.ts @@ -63,15 +63,25 @@ export const GIT_FIXED_ENV: Readonly> = Object.freeze({ GIT_GRAFT_FILE: join(devNull, 'no-grafts'), }); +/** + * A GIT_CONFIG_COUNT as git reads it (strtoul, base 10, over the whole value): optional leading whitespace and `+`, + * then digits; empty or unset is 0. Anything else git refuses ("bogus count", or "too many entries" for a minus). + */ +const CONFIG_COUNT = /^[ \t\n\v\f\r]*\+?[0-9]+$/; + /** * `env` (or this process's environment), with the fixed settings on top and the fixed config pairs after its own - * GIT_CONFIG_COUNT pairs (a later pair wins). A count that isn't a whole number is git's refusal anyway; it is replaced. + * GIT_CONFIG_COUNT pairs (a later pair wins). A count git refuses (`0x1`, `1e0`, `1 `, `abc`, `-1`) is left as it is, + * with no pair added, so git still refuses it: read with Number(), it became a count git never accepted, and the + * caller's first pair was applied where plain git ran nothing (#927's second review, #1001). The fixed arguments + * still turn the commit-graph off for the runner's own gits. */ export function gitEnv(env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const out: NodeJS.ProcessEnv = { ...(env ?? process.env), ...GIT_FIXED_ENV }; // Own keys only: an unset count must never be read through Object.prototype (#846). - let n = Number(Object.hasOwn(out, 'GIT_CONFIG_COUNT') ? out.GIT_CONFIG_COUNT : '0'); - if (!Number.isSafeInteger(n) || n < 0) n = 0; + const raw = Object.hasOwn(out, 'GIT_CONFIG_COUNT') ? out.GIT_CONFIG_COUNT : undefined; + let n = raw === undefined || raw === '' ? 0 : CONFIG_COUNT.test(raw) ? Number(raw) : NaN; + if (!Number.isSafeInteger(n)) return out; for (const [k, v] of GIT_FIXED_CONFIG) { out[`GIT_CONFIG_KEY_${n}`] = k; out[`GIT_CONFIG_VALUE_${n}`] = v; diff --git a/test/git-config-count.test.ts b/test/git-config-count.test.ts new file mode 100644 index 000000000..72cb1701e --- /dev/null +++ b/test/git-config-count.test.ts @@ -0,0 +1,39 @@ +// A GIT_CONFIG_COUNT git refuses stays refused through the git runner (#927 round 8, #1001 item 1). git reads the count +// with strtoul over the whole value: `0x1`, `1e0`, `1 ` and `abc` are "bogus count", and git runs nothing. gitEnv read +// it with Number(), so the runner turned that refusal into a run, and for `0x1`, `1e0` and `1 ` into one that applied +// the caller's first pair (measured by the second review of 5d8c98ab). A count git accepts is read as git reads it. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { gitEnv, gitSpawnSync, GIT_FIXED_ENV } from '../src/os/git.js'; +import { repoRoot } from './helpers.js'; + +const scripts = createRequire(import.meta.url)(join(repoRoot, 'scripts', 'lib', 'git.cjs')) as { gitEnv: (e?: NodeJS.ProcessEnv) => NodeJS.ProcessEnv }; +const pair = { GIT_CONFIG_KEY_0: 'x.y', GIT_CONFIG_VALUE_0: 'z' }; + +for (const count of ['0x1', '1e0', '1 ', 'abc', '-1', '1.5', ' ']) { + test(`a GIT_CONFIG_COUNT of ${JSON.stringify(count)}, which git refuses, is left as it is: git still refuses it`, () => { + for (const [which, env] of [['src/os/git.ts', gitEnv], ['scripts/lib/git.cjs', scripts.gitEnv]] as const) { + assert.deepEqual({ ...env({ GIT_CONFIG_COUNT: count, ...pair }) }, { GIT_CONFIG_COUNT: count, ...pair, ...GIT_FIXED_ENV }, which); + } + const r = gitSpawnSync(['config', '--get', 'x.y'], { cwd: mkdtempSync(join(tmpdir(), 'config-count-')), encoding: 'utf8', env: { PATH: process.env.PATH, HOME: process.env.HOME, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_COUNT: count, ...pair } }); + assert.equal(r.status, 128, `${r.stdout}${r.stderr}`); + assert.equal(r.stdout, '', 'the pair was not applied'); + }); +} + +for (const [count, n] of [['1', 1], ['01', 1], [' 1', 1], ['+1', 1], ['', 0]] as const) { + test(`a GIT_CONFIG_COUNT of ${JSON.stringify(count)}, which git reads as ${n}, keeps the caller's pairs and adds the runner's after them`, () => { + const out = gitEnv({ GIT_CONFIG_COUNT: count, ...pair }); + assert.equal(out.GIT_CONFIG_COUNT, String(n + 1)); + assert.equal(out[`GIT_CONFIG_KEY_${n}`], 'core.commitGraph'); + assert.equal(out[`GIT_CONFIG_VALUE_${n}`], 'false'); + assert.deepEqual({ ...scripts.gitEnv({ GIT_CONFIG_COUNT: count, ...pair }) }, { ...out }); + const r = gitSpawnSync(['config', '--get', 'x.y'], { cwd: mkdtempSync(join(tmpdir(), 'config-count-')), encoding: 'utf8', env: { PATH: process.env.PATH, HOME: process.env.HOME, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_COUNT: count, ...pair } }); + assert.equal(r.status, n ? 0 : 1, r.stderr); + assert.equal(r.stdout.trim(), n ? 'z' : ''); + }); +} From 550323cfcd2bced2917c98111ac3d35f400ac644 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 19:21:46 -0500 Subject: [PATCH 12/18] git runner: a strict config count, and the commit-graph off last in GIT_CONFIG_PARAMETERS (#927) Round 8 read GIT_CONFIG_COUNT the way it took git's strtoul to: optional whitespace and `+`, then digits. It missed the minus sign. git reads `-0`, ` -0` and `-00` as 0 and runs, while round 8 left them alone with no pair, so a git that reads only its environment (gitleaks') ran with the commit-graph on (measured by both round-8 reviews). And git reads GIT_CONFIG_PARAMETERS (what `git -c` hands the gits it runs) after the COUNT pairs, so a caller's `git -c core.commitGraph=true`, which the agent guardrail allows, turned it back on for the push gate's gitleaks (measured). In all three copies (src/os/git.ts gitEnv, scripts/lib/git.cjs gitEnv, the push gate's graphOff): - only a canonical count, a plain decimal 0 to 9999, gets the runner's pair after the caller's. Any other value becomes `refused`, a count git refuses on every platform, so the git fails closed instead of guessing git's parse; - `'core.commitGraph=false'` is appended to GIT_CONFIG_PARAMETERS, which git reads last; - no early return, so a later tail (#944's guardEnv) still runs. test/git-runner.test.ts's two expected gitEnv objects gain the GIT_CONFIG_PARAMETERS entry; nothing else in it changed. Test (fails without): test/git-config-count.test.ts, rewritten to the reviewers' tables and run through real git, with the push gate's graphOff taken from its source. As a new file in a fresh worktree at 7696ed8b, 44 of 46 fail: every non-canonical count (36), every canonical one (5, no GIT_CONFIG_PARAMETERS), and the three GIT_CONFIG_PARAMETERS cases. graphOff alone, from 7696ed8b's source, is wrong on 37 of 40 cases and on 0 here. Signed-off-by: Jayden Vasquez --- scripts/lib/git.cjs | 37 ++++++---- scripts/push-gate.mjs | 15 ++-- src/os/git.ts | 48 +++++++------ test/git-config-count.test.ts | 126 +++++++++++++++++++++++++++------- test/git-runner.test.ts | 4 +- 5 files changed, 164 insertions(+), 66 deletions(-) diff --git a/scripts/lib/git.cjs b/scripts/lib/git.cjs index b2ba36bb2..82d802c51 100644 --- a/scripts/lib/git.cjs +++ b/scripts/lib/git.cjs @@ -13,25 +13,32 @@ const GIT_FIXED_ENV = Object.freeze({ GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FIL /** git config every git also gets through its environment (GIT_CONFIG_COUNT pairs), after the caller's own pairs. */ const GIT_FIXED_CONFIG = Object.freeze([Object.freeze(['core.commitGraph', 'false'])]); -/** A GIT_CONFIG_COUNT as git reads it (strtoul, base 10, the whole value); empty or unset is 0. */ -const CONFIG_COUNT = /^[ \t\n\v\f\r]*\+?[0-9]+$/; +/** The only GIT_CONFIG_COUNT read: a plain decimal 0 to 9999. Any other becomes a value git refuses. */ +const CANONICAL_CONFIG_COUNT = /^(?:0|[1-9][0-9]{0,3})$/; +const REFUSED_CONFIG_COUNT = 'refused'; /** - * `env` (or this process's environment), with the fixed settings on top and the fixed config pairs after its own. A - * count git refuses is left as it is, with no pair added, so git still refuses it (as src/os/git.ts gitEnv). + * `env` (or this process's environment), with the fixed settings on top, and the fixed config after the caller's own + * GIT_CONFIG_COUNT pairs and at the end of GIT_CONFIG_PARAMETERS (which git reads last). A count that isn't canonical + * is replaced with one git refuses, so the git fails closed (as src/os/git.ts gitEnv). */ const gitEnv = (env) => { const out = { ...(env || process.env), ...GIT_FIXED_ENV }; - // Own keys only: an unset count must never be read through Object.prototype (#846). - const raw = Object.hasOwn(out, 'GIT_CONFIG_COUNT') ? out.GIT_CONFIG_COUNT : undefined; - let n = raw === undefined || raw === '' ? 0 : CONFIG_COUNT.test(raw) ? Number(raw) : NaN; - if (!Number.isSafeInteger(n)) return out; - for (const [k, v] of GIT_FIXED_CONFIG) { - out[`GIT_CONFIG_KEY_${n}`] = k; - out[`GIT_CONFIG_VALUE_${n}`] = v; - n++; - } - out.GIT_CONFIG_COUNT = String(n); + // Own keys only: an unset value must never be read through Object.prototype (#846). + const own = (k) => (Object.hasOwn(out, k) ? out[k] : undefined); + const raw = own('GIT_CONFIG_COUNT'); + if (raw === undefined || CANONICAL_CONFIG_COUNT.test(raw)) { + let n = raw === undefined ? 0 : Number(raw); + for (const [k, v] of GIT_FIXED_CONFIG) { + out[`GIT_CONFIG_KEY_${n}`] = k; + out[`GIT_CONFIG_VALUE_${n}`] = v; + n++; + } + out.GIT_CONFIG_COUNT = String(n); + } else out.GIT_CONFIG_COUNT = REFUSED_CONFIG_COUNT; + const params = own('GIT_CONFIG_PARAMETERS'); + const fixed = GIT_FIXED_CONFIG.map(([k, v]) => `'${k}=${v}'`).join(' '); + out.GIT_CONFIG_PARAMETERS = params ? `${params} ${fixed}` : fixed; return out; }; /** The argument list a git process gets: the fixed arguments, then `args`. */ @@ -55,6 +62,8 @@ const commandSpawn = (cmd, args, options = {}) => (isGitCommand(cmd) ? spawn(cmd exports.GIT_FIXED_ARGS = GIT_FIXED_ARGS; exports.GIT_FIXED_ENV = GIT_FIXED_ENV; exports.GIT_FIXED_CONFIG = GIT_FIXED_CONFIG; +exports.CANONICAL_CONFIG_COUNT = CANONICAL_CONFIG_COUNT; +exports.REFUSED_CONFIG_COUNT = REFUSED_CONFIG_COUNT; exports.gitEnv = gitEnv; exports.gitArgs = gitArgs; exports.isGitCommand = isGitCommand; diff --git a/scripts/push-gate.mjs b/scripts/push-gate.mjs index a031a590e..0d6ad3a57 100644 --- a/scripts/push-gate.mjs +++ b/scripts/push-gate.mjs @@ -28,15 +28,18 @@ const ZERO = /^0+$/; // `git replace` or a graft in the repo can't swap the commits, trees or blobs scanned here. Written out here because // the hook runs this file alone, fetched from origin (scripts/push-gate-hook.mjs GATE_CODE), so it imports nothing // new; test/git-runner.test.ts checks they are the runner's. -// The commit-graph is off too (a forged one can skip a commit), in the arguments and, for gitleaks' own gits, as a -// config pair in the environment after any the caller set. +// The commit-graph is off too (a forged one can skip a commit), in the arguments and, for gitleaks' own gits, in the +// environment: a config pair after the caller's GIT_CONFIG_COUNT pairs, and at the end of GIT_CONFIG_PARAMETERS +// (what `git -c` hands the gits it runs), which git reads last. A count that isn't a plain decimal 0 to 9999 is +// replaced with one git refuses, so the git fails closed (as src/os/git.ts gitEnv). const GIT_FIXED_ARGS = ['--no-replace-objects', '-c', 'core.commitGraph=false']; -// A count git refuses is left as it is, with no pair added, so git still refuses it (as src/os/git.ts gitEnv). const graphOff = (e) => { const raw = Object.hasOwn(e, 'GIT_CONFIG_COUNT') ? e.GIT_CONFIG_COUNT : undefined; - const n = raw === undefined || raw === '' ? 0 : /^[ \t\n\v\f\r]*\+?[0-9]+$/.test(raw) ? Number(raw) : NaN; - if (!Number.isSafeInteger(n)) return e; - return { ...e, [`GIT_CONFIG_KEY_${n}`]: 'core.commitGraph', [`GIT_CONFIG_VALUE_${n}`]: 'false', GIT_CONFIG_COUNT: String(n + 1) }; + const params = Object.hasOwn(e, 'GIT_CONFIG_PARAMETERS') ? e.GIT_CONFIG_PARAMETERS : undefined; + const last = { GIT_CONFIG_PARAMETERS: params ? `${params} 'core.commitGraph=false'` : "'core.commitGraph=false'" }; + if (raw !== undefined && !/^(?:0|[1-9][0-9]{0,3})$/.test(raw)) return { ...e, GIT_CONFIG_COUNT: 'refused', ...last }; + const n = raw === undefined ? 0 : Number(raw); + return { ...e, [`GIT_CONFIG_KEY_${n}`]: 'core.commitGraph', [`GIT_CONFIG_VALUE_${n}`]: 'false', GIT_CONFIG_COUNT: String(n + 1), ...last }; }; const GIT_ENV = graphOff({ ...process.env, GIT_NO_REPLACE_OBJECTS: '1', GIT_GRAFT_FILE: join(devNull, 'no-grafts') }); const runGit = (args, opts) => execFileSync('git', [...GIT_FIXED_ARGS, ...args], { ...opts, env: GIT_ENV }); diff --git a/src/os/git.ts b/src/os/git.ts index 3befbb8c5..2f2298976 100644 --- a/src/os/git.ts +++ b/src/os/git.ts @@ -63,31 +63,39 @@ export const GIT_FIXED_ENV: Readonly> = Object.freeze({ GIT_GRAFT_FILE: join(devNull, 'no-grafts'), }); -/** - * A GIT_CONFIG_COUNT as git reads it (strtoul, base 10, over the whole value): optional leading whitespace and `+`, - * then digits; empty or unset is 0. Anything else git refuses ("bogus count", or "too many entries" for a minus). - */ -const CONFIG_COUNT = /^[ \t\n\v\f\r]*\+?[0-9]+$/; +/** The only GIT_CONFIG_COUNT the runner reads: a plain decimal 0 to 9999, as the engine itself writes one. */ +export const CANONICAL_CONFIG_COUNT = /^(?:0|[1-9][0-9]{0,3})$/; +/** What any other count becomes: a value git refuses ("bogus count") on every platform, so the git fails. */ +export const REFUSED_CONFIG_COUNT = 'refused'; /** - * `env` (or this process's environment), with the fixed settings on top and the fixed config pairs after its own - * GIT_CONFIG_COUNT pairs (a later pair wins). A count git refuses (`0x1`, `1e0`, `1 `, `abc`, `-1`) is left as it is, - * with no pair added, so git still refuses it: read with Number(), it became a count git never accepted, and the - * caller's first pair was applied where plain git ran nothing (#927's second review, #1001). The fixed arguments - * still turn the commit-graph off for the runner's own gits. + * `env` (or this process's environment), with the fixed settings on top, and the fixed config in it twice: + * - as GIT_CONFIG_COUNT pairs after the caller's own (a later pair wins). Only a canonical count is read. Any other + * (`-0`, `+1`, ` 1`, `01`, `0x1`, `abc`, an overlong one) is replaced with REFUSED_CONFIG_COUNT, so git fails + * instead of running: git's strtoul reads some of them (`-0`, `+1`) and refuses others, and a parse that tried to + * match it got the sign wrong (#927 rounds 8 and 9). Fails closed; + * - appended to GIT_CONFIG_PARAMETERS (what `git -c` hands the gits it runs), which git reads after the COUNT pairs: + * a caller's `git -c core.commitGraph=true` there would otherwise win (#927 round 9, measured). + * The fixed arguments turn the commit-graph off for the runner's own gits too; the environment covers the gits a + * program like gitleaks runs. */ export function gitEnv(env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const out: NodeJS.ProcessEnv = { ...(env ?? process.env), ...GIT_FIXED_ENV }; - // Own keys only: an unset count must never be read through Object.prototype (#846). - const raw = Object.hasOwn(out, 'GIT_CONFIG_COUNT') ? out.GIT_CONFIG_COUNT : undefined; - let n = raw === undefined || raw === '' ? 0 : CONFIG_COUNT.test(raw) ? Number(raw) : NaN; - if (!Number.isSafeInteger(n)) return out; - for (const [k, v] of GIT_FIXED_CONFIG) { - out[`GIT_CONFIG_KEY_${n}`] = k; - out[`GIT_CONFIG_VALUE_${n}`] = v; - n++; - } - out.GIT_CONFIG_COUNT = String(n); + // Own keys only: an unset value must never be read through Object.prototype (#846). + const own = (k: string) => (Object.hasOwn(out, k) ? out[k] : undefined); + const raw = own('GIT_CONFIG_COUNT'); + if (raw === undefined || CANONICAL_CONFIG_COUNT.test(raw)) { + let n = raw === undefined ? 0 : Number(raw); + for (const [k, v] of GIT_FIXED_CONFIG) { + out[`GIT_CONFIG_KEY_${n}`] = k; + out[`GIT_CONFIG_VALUE_${n}`] = v; + n++; + } + out.GIT_CONFIG_COUNT = String(n); + } else out.GIT_CONFIG_COUNT = REFUSED_CONFIG_COUNT; + const params = own('GIT_CONFIG_PARAMETERS'); + const fixed = GIT_FIXED_CONFIG.map(([k, v]) => `'${k}=${v}'`).join(' '); + out.GIT_CONFIG_PARAMETERS = params ? `${params} ${fixed}` : fixed; return out; } diff --git a/test/git-config-count.test.ts b/test/git-config-count.test.ts index 72cb1701e..c1a7ef3fa 100644 --- a/test/git-config-count.test.ts +++ b/test/git-config-count.test.ts @@ -1,39 +1,117 @@ -// A GIT_CONFIG_COUNT git refuses stays refused through the git runner (#927 round 8, #1001 item 1). git reads the count -// with strtoul over the whole value: `0x1`, `1e0`, `1 ` and `abc` are "bogus count", and git runs nothing. gitEnv read -// it with Number(), so the runner turned that refusal into a run, and for `0x1`, `1e0` and `1 ` into one that applied -// the caller's first pair (measured by the second review of 5d8c98ab). A count git accepts is read as git reads it. +// The git runner's commit-graph setting in the environment, which the gits a program like gitleaks runs get (#927). +// Three copies set it: src/os/git.ts gitEnv, scripts/lib/git.cjs gitEnv, and the push gate's graphOff (it runs alone, +// so it writes them out). Each is tested here, the gate's taken from its source. +// - GIT_CONFIG_COUNT: only a canonical count (a plain decimal 0 to 9999) is read, and the runner's pair goes after the +// caller's. Any other count becomes one git refuses, so the git fails closed. Round 8 tried to read a count as git's +// strtoul does and got the sign wrong: `-0` is 0 to git, and lost the pair (#927 round 8's reviews, measured). +// Before that, Number() turned counts git refuses (`0x1`, `1e0`, `1 `) into a run that applied the caller's pair. +// - GIT_CONFIG_PARAMETERS (what `git -c` hands the gits it runs) is read after the COUNT pairs: a caller's +// `git -c core.commitGraph=true` there won (measured). The setting is appended to it too. import { test } from 'node:test'; import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; import { createRequire } from 'node:module'; -import { mkdtempSync } from 'node:fs'; +import { mkdtempSync, readFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { gitEnv, gitSpawnSync, GIT_FIXED_ENV } from '../src/os/git.js'; +import { gitEnv, gitSpawnSync } from '../src/os/git.js'; import { repoRoot } from './helpers.js'; -const scripts = createRequire(import.meta.url)(join(repoRoot, 'scripts', 'lib', 'git.cjs')) as { gitEnv: (e?: NodeJS.ProcessEnv) => NodeJS.ProcessEnv }; +type EnvFn = (e: NodeJS.ProcessEnv) => NodeJS.ProcessEnv; +const scripts = createRequire(import.meta.url)(join(repoRoot, 'scripts', 'lib', 'git.cjs')) as { gitEnv: EnvFn }; +const gateSource = readFileSync(join(repoRoot, 'scripts', 'push-gate.mjs'), 'utf8'); +const gateFn = /const graphOff = (\(e\) => \{[\s\S]*?\n\});/.exec(gateSource); +assert.ok(gateFn, 'the push gate declares graphOff'); +const graphOff = new Function(`return ${gateFn[1]}`)() as EnvFn; +const COPIES: [string, EnvFn][] = [['src/os/git.ts gitEnv', gitEnv], ['scripts/lib/git.cjs gitEnv', scripts.gitEnv], ['scripts/push-gate.mjs graphOff', graphOff]]; + +const dir = mkdtempSync(join(tmpdir(), 'config-count-')); +const base = (): NodeJS.ProcessEnv => ({ PATH: process.env.PATH, HOME: dir, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null' }); +/** `git config --get key` as a git that reads only its environment (gitleaks' gits): status and value. */ +const envOnly = (env: NodeJS.ProcessEnv, key: string) => { + const r = spawnSync('git', ['config', '--get', key], { cwd: dir, env, encoding: 'utf8' }); + return { status: r.status, value: r.stdout.trim() }; +}; const pair = { GIT_CONFIG_KEY_0: 'x.y', GIT_CONFIG_VALUE_0: 'z' }; +const NBSP = String.fromCharCode(0xa0); + +// Counts git itself refuses, counts git reads that aren't canonical (`-0`, `+1`, ` 1`, `01`), and overlong ones. +const NOT_CANONICAL = [ + '', '+0', '+1', ' 1', '\v1', '\n1', '01', '0000001', '-0', ' -0', '-00', '\t-0', '-', '+', '+-1', '-+0', '1\n', '0x1', + '1e0', '1 ', 'abc', '-1', '-01', '1.5', ' ', `${NBSP}1`, '10000', '2147483646', '2147483647', '2147483648', '4294967296', + '9007199254740991', '9007199254740993', '18446744073709551615', '18446744073709551616', '-18446744073709551615', +]; -for (const count of ['0x1', '1e0', '1 ', 'abc', '-1', '1.5', ' ']) { - test(`a GIT_CONFIG_COUNT of ${JSON.stringify(count)}, which git refuses, is left as it is: git still refuses it`, () => { - for (const [which, env] of [['src/os/git.ts', gitEnv], ['scripts/lib/git.cjs', scripts.gitEnv]] as const) { - assert.deepEqual({ ...env({ GIT_CONFIG_COUNT: count, ...pair }) }, { GIT_CONFIG_COUNT: count, ...pair, ...GIT_FIXED_ENV }, which); +for (const count of NOT_CANONICAL) { + test(`GIT_CONFIG_COUNT ${JSON.stringify(count)} isn't canonical: every copy makes it one git refuses, and git runs nothing`, () => { + for (const [which, fn] of COPIES) { + const env = fn({ ...base(), GIT_CONFIG_COUNT: count, ...pair }); + assert.equal(env.GIT_CONFIG_COUNT, 'refused', which); + assert.equal(env.GIT_CONFIG_PARAMETERS, "'core.commitGraph=false'", which); + const r = envOnly(env, 'x.y'); + assert.equal(r.status, 128, `${which}: git refuses the count`); + assert.equal(r.value, '', `${which}: the caller's pair is not applied`); } - const r = gitSpawnSync(['config', '--get', 'x.y'], { cwd: mkdtempSync(join(tmpdir(), 'config-count-')), encoding: 'utf8', env: { PATH: process.env.PATH, HOME: process.env.HOME, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_COUNT: count, ...pair } }); - assert.equal(r.status, 128, `${r.stdout}${r.stderr}`); - assert.equal(r.stdout, '', 'the pair was not applied'); + // And through the runner itself. + const r = gitSpawnSync(['config', '--get', 'x.y'], { cwd: dir, encoding: 'utf8', env: { ...base(), GIT_CONFIG_COUNT: count, ...pair } }); + assert.equal(r.status, 128, r.stderr); + assert.equal(r.stdout, ''); }); } -for (const [count, n] of [['1', 1], ['01', 1], [' 1', 1], ['+1', 1], ['', 0]] as const) { - test(`a GIT_CONFIG_COUNT of ${JSON.stringify(count)}, which git reads as ${n}, keeps the caller's pairs and adds the runner's after them`, () => { - const out = gitEnv({ GIT_CONFIG_COUNT: count, ...pair }); - assert.equal(out.GIT_CONFIG_COUNT, String(n + 1)); - assert.equal(out[`GIT_CONFIG_KEY_${n}`], 'core.commitGraph'); - assert.equal(out[`GIT_CONFIG_VALUE_${n}`], 'false'); - assert.deepEqual({ ...scripts.gitEnv({ GIT_CONFIG_COUNT: count, ...pair }) }, { ...out }); - const r = gitSpawnSync(['config', '--get', 'x.y'], { cwd: mkdtempSync(join(tmpdir(), 'config-count-')), encoding: 'utf8', env: { PATH: process.env.PATH, HOME: process.env.HOME, GIT_CONFIG_NOSYSTEM: '1', GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_COUNT: count, ...pair } }); - assert.equal(r.status, n ? 0 : 1, r.stderr); - assert.equal(r.stdout.trim(), n ? 'z' : ''); +for (const count of [undefined, '0', '1', '12', '9999']) { + test(`GIT_CONFIG_COUNT ${count === undefined ? 'unset' : JSON.stringify(count)} is canonical: every copy keeps the caller's pairs and puts the runner's after them`, () => { + const n = count === undefined ? 0 : Number(count); + for (const [which, fn] of COPIES) { + const env = fn({ ...base(), ...(count === undefined ? {} : { GIT_CONFIG_COUNT: count }), ...pair }); + assert.equal(env.GIT_CONFIG_COUNT, String(n + 1), which); + assert.equal(env[`GIT_CONFIG_KEY_${n}`], 'core.commitGraph', which); + assert.equal(env[`GIT_CONFIG_VALUE_${n}`], 'false', which); + assert.equal(env.GIT_CONFIG_PARAMETERS, "'core.commitGraph=false'", which); + if (n > 1) continue; // git needs every pair below the count: the shape is what is checked here + assert.deepEqual(envOnly(env, 'core.commitGraph'), { status: 0, value: 'false' }, which); + assert.deepEqual(envOnly(env, 'x.y'), n === 1 ? { status: 0, value: 'z' } : { status: 1, value: '' }, which); + } }); } + +for (const [what, params] of [ + ['old form', "'core.commitGraph=true'"], + ['new form', "'core.commitGraph'='true'"], + ['with another key first', "'a.b'='c' 'core.commitGraph'='true'"], +] as const) { + test(`a caller's GIT_CONFIG_PARAMETERS (${what}) setting core.commitGraph=true: a git that reads only the environment still reads false`, () => { + // Plain git believes the caller's. + assert.deepEqual(envOnly({ ...base(), GIT_CONFIG_PARAMETERS: params }, 'core.commitGraph'), { status: 0, value: 'true' }); + for (const [which, fn] of COPIES) { + const env = fn({ ...base(), GIT_CONFIG_PARAMETERS: params }); + assert.equal(env.GIT_CONFIG_PARAMETERS, `${params} 'core.commitGraph=false'`, which); + assert.deepEqual(envOnly(env, 'core.commitGraph'), { status: 0, value: 'false' }, which); + // Also with the caller's count pair saying true. + const both = fn({ ...base(), GIT_CONFIG_PARAMETERS: params, GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'true' }); + assert.deepEqual(envOnly(both, 'core.commitGraph'), { status: 0, value: 'false' }, which); + } + const r = gitSpawnSync(['config', '--get', 'core.commitGraph'], { cwd: dir, encoding: 'utf8', env: { ...base(), GIT_CONFIG_PARAMETERS: params } }); + assert.equal(r.stdout.trim(), 'false'); + }); +} + +test('a malformed caller GIT_CONFIG_PARAMETERS stays one git refuses, or reads false: never true', () => { + for (const params of ["'a.b'='c", "'core.commitGraph=true", "core.commitGraph=true", "'core.commitGraph'='true"]) { + for (const [which, fn] of COPIES) { + const r = envOnly(fn({ ...base(), GIT_CONFIG_PARAMETERS: params }), 'core.commitGraph'); + assert.ok(r.status === 128 || r.value === 'false', `${which} ${JSON.stringify(params)}: ${JSON.stringify(r)}`); + } + } +}); + +test('the copies agree on every count and parameters value tried here', () => { + for (const count of [...NOT_CANONICAL, '0', '1', '12', '9999']) { + const input = { ...base(), GIT_CONFIG_COUNT: count, ...pair, GIT_CONFIG_PARAMETERS: "'a.b=c'" }; + const want = gitEnv({ ...input }); + for (const [which, fn] of COPIES.slice(1)) { + const got = fn({ ...input }); + for (const k of Object.keys(want).filter((key) => key.startsWith('GIT_CONFIG'))) assert.equal(got[k], want[k], `${which} ${JSON.stringify(count)} ${k}`); + } + } +}); diff --git a/test/git-runner.test.ts b/test/git-runner.test.ts index 57017fdd0..e09eca992 100644 --- a/test/git-runner.test.ts +++ b/test/git-runner.test.ts @@ -116,10 +116,10 @@ test('the runner\'s settings go on last: a caller\'s environment can\'t turn rep const want = ['keys/secret.pem', 'weights.bin']; assert.deepEqual(names(gitExecFileSync(['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env })), want); assert.deepEqual(names(gitSpawnSync(['log', '--format=', '--name-only', `${r.base}..${r.head}`], { cwd: r.work, encoding: 'utf8', env }).stdout), want); - const graphOff = { GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'false' }; + const graphOff = { GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'false', GIT_CONFIG_PARAMETERS: "'core.commitGraph=false'" }; assert.deepEqual({ ...gitEnv({ GIT_GRAFT_FILE: grafts, GIT_NO_REPLACE_OBJECTS: '0' }) }, { ...GIT_FIXED_ENV, ...graphOff }); // A caller's own config pairs stay; the fixed pair goes after them (a later pair wins). - assert.deepEqual({ ...gitEnv({ GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'true' }) }, { ...GIT_FIXED_ENV, GIT_CONFIG_COUNT: '2', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'true', GIT_CONFIG_KEY_1: 'core.commitGraph', GIT_CONFIG_VALUE_1: 'false' }); + assert.deepEqual({ ...gitEnv({ GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'true' }) }, { ...GIT_FIXED_ENV, GIT_CONFIG_COUNT: '2', GIT_CONFIG_KEY_0: 'core.commitGraph', GIT_CONFIG_VALUE_0: 'true', GIT_CONFIG_KEY_1: 'core.commitGraph', GIT_CONFIG_VALUE_1: 'false', GIT_CONFIG_PARAMETERS: "'core.commitGraph=false'" }); // A count on Object.prototype is never read (#846): the fixed pair is pair 0, and nothing else is counted. const proto = Object.prototype as Record; proto.GIT_CONFIG_COUNT = '1'; From 4e4287e6fb1bf5835b59102a5d48f17dbd31fa1c Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 19:21:46 -0500 Subject: [PATCH 13/18] decision detail: the owner's diff is redacted and shown as a NUL-free view (#927) The decision diff is read with `--text` (#807 F2), so a binary file's bytes are shown. UTF-16 text puts a NUL between ASCII characters, and no redaction rule matched it: a token in a UTF-16 file read whole once the NULs were dropped, where main shows "Binary files differ" (measured by round 8's second review). That reopened something main closed. Both branches of decisionDiff now drop every NUL before redacting and cutting. `--text` stays: a NUL, or a `-diff` attribute, is also how a change hides its lines from review as "binary", which F2 closes. Test (fails without): test/decision-diff-utf16.test.ts, a token in a UTF-16LE and in a UTF-16BE file. As a new file in a fresh worktree at 7696ed8b, both fail: the shown diff, NULs dropped, holds `ghp_`. Signed-off-by: Jayden Vasquez --- src/decision-detail.ts | 10 +++++-- test/decision-diff-utf16.test.ts | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 test/decision-diff-utf16.test.ts diff --git a/src/decision-detail.ts b/src/decision-detail.ts index 7886f144c..c21866cf1 100644 --- a/src/decision-detail.ts +++ b/src/decision-detail.ts @@ -357,11 +357,15 @@ export async function decisionDiff(root: string, base: string, head: string): Pr return new Promise((done) => { gitExecFile(args, { cwd: root, env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: 'utf8' }, (err, stdout, stderr) => { if (err && !(err.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' && stdout)) return done({ diff: null, why: `not in this checkout's git: ${(stderr || err.message).trim().split('\n')[0]}` }); + // A NUL-free view (#927 round 9): `--text` shows a binary file's bytes, and UTF-16 text (a NUL between ASCII + // characters) went past every redaction rule, where main showed "Binary files differ". `--text` stays: a NUL, or + // a `-diff` attribute, is also how a change hides its lines from review as "binary". + const shown = stdout.replace(/\0/g, ''); // Redacted before it is cut, over the kept part and a margin only (#889): up to 8 MB of diff, and the redaction is // synchronous on the dashboard. Output past the buffer the same way: a token across the cut is redacted whole. - if (err) return done({ diff: redactCut(stdout, MAX), truncated: true }); - const truncated = stdout.length > MAX; - done({ diff: redactCut(stdout, MAX), truncated }); + if (err) return done({ diff: redactCut(shown, MAX), truncated: true }); + const truncated = shown.length > MAX; + done({ diff: redactCut(shown, MAX), truncated }); }); }); } diff --git a/test/decision-diff-utf16.test.ts b/test/decision-diff-utf16.test.ts new file mode 100644 index 000000000..48d06e971 --- /dev/null +++ b/test/decision-diff-utf16.test.ts @@ -0,0 +1,51 @@ +// The owner's decision diff and a token in UTF-16 (#927 round 9). The diff is read with `--text` (#807 F2: a NUL, or a +// `-diff` attribute, is also how a change hides its lines from review as "binary"), so a binary file's bytes are shown. +// UTF-16 text puts a NUL between ASCII characters, and no redaction rule matched it: the token read whole once the NULs +// were dropped (measured by #927 round 8's second review), where main showed "Binary files differ". The diff is now +// redacted, and shown, as a NUL-free view. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { decisionDiff } from '../src/decision-detail.js'; + +/** GitHub-token-shaped (ghp_ and 36 letters and digits), not a real one. */ +const TOKEN = `ghp_${'Utf16Tk9Lm3Qr7Ws2Xv5Zb8Nc4Hd6Jf1Gp0'.padEnd(36, 'a').slice(0, 36)}`; + +function utf16(text: string, order: 'le' | 'be'): Buffer { + const le = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, 'utf16le')]); + if (order === 'le') return le; + const be = Buffer.from(le); + be.swap16(); + return be; +} + +for (const order of ['le', 'be'] as const) { + test(`a token in a UTF-16${order.toUpperCase()} file is not shown in the owner's decision diff`, async () => { + assert.equal(TOKEN.length, 40); + const repo = mkdtempSync(join(tmpdir(), `decision-utf16-${order}-`)); + const git = (...a: string[]) => execFileSync('git', a, { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + git('init', '-q', '-b', 'main'); + for (const [k, v] of [['user.name', 'Ada'], ['user.email', 'ada@example.com'], ['commit.gpgsign', 'false']]) git('config', k!, v!); + writeFileSync(join(repo, 'a.txt'), 'a\n'); + git('add', '-A'); + git('commit', '-q', '-m', 'base'); + const base = git('rev-parse', 'HEAD').trim(); + writeFileSync(join(repo, 'notes.txt'), utf16(`notes\ntoken: ${TOKEN}\nend\n`, order)); + git('add', '-A'); + git('commit', '-q', '-m', 'notes'); + const head = git('rev-parse', 'HEAD').trim(); + // The file holds the token's characters with a NUL between them. + assert.ok(git('show', `${head}:notes.txt`).replace(/\0/g, '').includes(TOKEN)); + + const d = await decisionDiff(repo, base, head); + assert.ok(d.diff !== null, JSON.stringify(d)); + const seen = d.diff.replace(/\0/g, ''); + assert.equal(seen.includes('ghp_'), false, `no part of the token: ${JSON.stringify(seen.slice(0, 400))}`); + assert.equal(seen.includes(TOKEN.slice(4, 20)), false); + // The rest of the file still shows: the change isn't hidden as "binary". + assert.match(seen, /notes/); + }); +} From 633b03d8031e130022ec0ce06d5cf6637a54bf97 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 22:28:35 -0500 Subject: [PATCH 14/18] decision detail: a binary file's bytes never show on the owner's diff, its text lines do (#927) Round 9 showed the `--text` diff with its NULs dropped. That showed a binary file's printable bytes (a DER private key's), and it glued byte runs together, so a fixed-length key shape a NUL had ended lost the boundary its redaction rule needs (AWS, Google, 0x hex; and `\b` before `sk-`, Stripe). main shows "Binary files differ" (both round-9 reviews, measured and inferred). diffTextView reads git's bytes (latin1 from the runner, one character per byte). A file whose diff lines hold a NUL, invalid UTF-8 or a control byte text doesn't hold is binary, and each of its lines shows only as text it decodes to: UTF-32 or UTF-16 when its NULs fall as mostly-Latin text's do, else UTF-8 of at least 16 characters, each NUL run as U+2400, a boundary for every rule. Any other line becomes `[binary: N bytes not shown]`. Every other file shows as it is. `--text` stays (#807 F2: a NUL or a `-diff` attribute must not hide a text change from review). Test (fails without): test/decision-diff-binary.test.ts. As a new file in a fresh worktree at 4e4287e6, 5 of 7 fail: the DER keys (ed25519 PKCS#8, RSA-2048 PKCS#1; no 6-byte run of their printable bytes may show), an AWS, a Google and a 0x hex key shape between NULs with a letter after (shown in the clear there), and a text file with one NUL (its NUL must show as a mark). The two UTF-32 cases pass there too; they pin that this view still reads UTF-32. Signed-off-by: Jayden Vasquez --- src/decision-detail.ts | 139 ++++++++++++++++++++++++++++-- test/decision-diff-binary.test.ts | 108 +++++++++++++++++++++++ 2 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 test/decision-diff-binary.test.ts diff --git a/src/decision-detail.ts b/src/decision-detail.ts index c21866cf1..6dfcc4f81 100644 --- a/src/decision-detail.ts +++ b/src/decision-detail.ts @@ -346,6 +346,132 @@ export function decisionDetail(id: string, d: DetailDeps): DecisionDetail | null }; } +const UTF8_STRICT = new TextDecoder('utf-8', { fatal: true }); +/** The control bytes a text line may hold besides NUL: tab, vertical tab, form feed, carriage return and escape. */ +const TEXT_CONTROLS = new Set([0x09, 0x0b, 0x0c, 0x0d, 0x1b]); +/** What a NUL (a run of them) in a shown line becomes: not a letter, digit or `_`, so every redaction rule still sees a boundary there. */ +export const NUL_MARK = '␀'; +/** A binary file's line that is shown as UTF-8 text needs this many characters: random bytes almost never make one. */ +const MIN_BINARY_TEXT = 16; + +/** `b` as UTF-8 text, NULs kept as U+0000; or null for invalid UTF-8, DEL, or a control byte text doesn't hold. */ +function utf8Text(b: Buffer): string | null { + for (const c of b) if ((c < 0x20 && c !== 0 && !TEXT_CONTROLS.has(c)) || c === 0x7f) return null; + try { + return UTF8_STRICT.decode(b); + } catch { + return null; + } +} + +/** `s` (whole units of `width` bytes) as UTF-16 or UTF-32 text, when its NULs fall as mostly-Latin text's do; or null. */ +function decodeWide(s: Buffer, width: 2 | 4): string | null { + const units = s.length / width; + if (units < 1) return null; + let t = ''; + if (width === 2) { + let low = 0; + let high = 0; + for (let i = 0; i < s.length; i += 2) { + if (s[i] === 0) low++; + if (s[i + 1] === 0) high++; + } + const order = high >= units * 0.75 && low <= units * 0.25 ? 'utf-16le' : low >= units * 0.75 && high <= units * 0.25 ? 'utf-16be' : null; + if (!order) return null; + try { + t = new TextDecoder(order, { fatal: true, ignoreBOM: false }).decode(s); + } catch { + return null; + } + } else { + const unitLE = (i: number) => s[i]! | (s[i + 1]! << 8) | (s[i + 2]! << 16) | (s[i + 3]! << 24); + const unitBE = (i: number) => s[i + 3]! | (s[i + 2]! << 8) | (s[i + 1]! << 16) | (s[i]! << 24); + for (const read of [unitLE, unitBE]) { + const points: number[] = []; + let narrow = 0; + for (let i = 0; i < s.length; i += 4) points.push(read(i) >>> 0); + for (const p of points) if (p < 0x100) narrow++; + if (narrow < units * 0.75 || points.some((p) => p > 0x10ffff || (p >= 0xd800 && p <= 0xdfff))) continue; + t = String.fromCodePoint(...(points[0] === 0xfeff ? points.slice(1) : points)); + break; + } + if (!t) return null; + } + for (const ch of t) { + const c = ch.codePointAt(0)!; + if ((c < 0x20 && c !== 0 && !TEXT_CONTROLS.has(c)) || c === 0x7f) return null; + } + return t; +} + +/** + * `b` as UTF-32 or UTF-16 text (little- or big-endian, as its NULs fall), or null. git splits such lines at the 0x0A + * byte, so a line can start or end with the rest of a newline's unit; those NUL bytes are dropped. Only a line whose + * NULs fall as mostly-Latin text's do is read so (random bytes don't), and only when it decodes cleanly to text. + */ +function wideText(b: Buffer): string | null { + for (const width of [4, 2] as const) { + const r = b.length % width; + const tries = r === 0 ? [b] : [b.subarray(0, r).every((x) => x === 0) ? b.subarray(r) : null, b.subarray(b.length - r).every((x) => x === 0) ? b.subarray(0, b.length - r) : null]; + for (const s of tries) { + const t = s && decodeWide(s, width); + if (t) return t; + } + } + return null; +} + +/** One line of a binary file's diff (`prefix`, then its bytes) as shown: its text, or a marker that holds none of it. */ +function binaryLine(prefix: string, body: Buffer): string { + const wide = wideText(body); + const t = wide ?? utf8Text(body); + if (t !== null) { + const chars = t.replace(/\0/g, '').length; + if (wide !== null || chars === 0 || chars >= MIN_BINARY_TEXT) return `${prefix}${t.replace(/\0+/g, NUL_MARK)}`; + } + return `${prefix}[binary: ${body.length} bytes not shown]`; +} + +/** + * What the owner's decision diff shows of `git diff --text` output (#927 round 10). A file whose diff lines hold a + * NUL, invalid UTF-8 or a control byte text doesn't hold is binary: each of its lines shows only as text it decodes + * to (UTF-16 or UTF-8, a NUL as NUL_MARK, a UTF-8 line at least MIN_BINARY_TEXT characters), and otherwise as a marker. + * So no run of a binary file's bytes is shown, while its text lines are, and a NUL still separates what it separated + * for the redaction rules that need a boundary. Every other file shows as it is. + */ +export function diffTextView(raw: Buffer): string { + const lines: Buffer[] = []; + for (let at = 0; at <= raw.length; ) { + const nl = raw.indexOf(0x0a, at); + const end = nl === -1 ? raw.length : nl; + lines.push(raw.subarray(at, end)); + if (nl === -1) break; + at = nl + 1; + } + const out: string[] = []; + let file: Buffer[] = []; + const flush = () => { + let inHunk = false; + const content = (l: Buffer) => inHunk && l.length > 0 && (l[0] === 0x2b || l[0] === 0x2d || l[0] === 0x20); + const binary = file.some((l) => { + if (l.subarray(0, 3).toString('latin1') === '@@ ') inHunk = true; + return content(l) && utf8Text(l.subarray(1))?.includes('\0') !== false; + }); + inHunk = false; + for (const l of file) { + if (l.subarray(0, 3).toString('latin1') === '@@ ') inHunk = true; + out.push(binary && content(l) ? binaryLine(String.fromCharCode(l[0]!), l.subarray(1)) : l.toString('utf8')); + } + file = []; + }; + for (const l of lines) { + if (l.subarray(0, 11).toString('latin1') === 'diff --git ') flush(); + file.push(l); + } + flush(); + return out.join('\n'); +} + /** The change's diff, base..head, from the checkout's git (owners only: the route checks); redacted, capped. */ export async function decisionDiff(root: string, base: string, head: string): Promise<{ diff: string; truncated: boolean } | { diff: null; why: string }> { if (!SHA.test(base) || !SHA.test(head)) return { diff: null, why: 'no commits recorded for this change' }; @@ -355,12 +481,13 @@ export async function decisionDiff(root: string, base: string, head: string): Pr // attributes file is not read. Output past the buffer is shown cut, like output past MAX. const args = ['-c', 'core.attributesFile=/dev/null', 'diff', '--no-color', '--text', '--no-textconv', '--no-ext-diff', '--ignore-submodules=none', `${base}..${head}`]; return new Promise((done) => { - gitExecFile(args, { cwd: root, env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: 'utf8' }, (err, stdout, stderr) => { - if (err && !(err.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' && stdout)) return done({ diff: null, why: `not in this checkout's git: ${(stderr || err.message).trim().split('\n')[0]}` }); - // A NUL-free view (#927 round 9): `--text` shows a binary file's bytes, and UTF-16 text (a NUL between ASCII - // characters) went past every redaction rule, where main showed "Binary files differ". `--text` stays: a NUL, or - // a `-diff` attribute, is also how a change hides its lines from review as "binary". - const shown = stdout.replace(/\0/g, ''); + // latin1: one character per byte, so the bytes come back as git wrote them (diffTextView reads them). + gitExecFile(args, { cwd: root, env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: 'latin1' }, (err, stdout, stderr) => { + if (err && !(err.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' && stdout)) return done({ diff: null, why: `not in this checkout's git: ${(Buffer.from(stderr, 'latin1').toString('utf8') || err.message).trim().split('\n')[0]}` }); + // `--text` stays (#807 F2): a NUL, or a `-diff` attribute, is also how a change hides its text from review as + // "binary". But a binary file's bytes are never shown (#927 round 10): where main showed "Binary files differ", + // its lines show only as the text they decode to (UTF-16 included), each NUL a boundary mark (diffTextView). + const shown = diffTextView(Buffer.from(stdout, 'latin1')); // Redacted before it is cut, over the kept part and a margin only (#889): up to 8 MB of diff, and the redaction is // synchronous on the dashboard. Output past the buffer the same way: a token across the cut is redacted whole. if (err) return done({ diff: redactCut(shown, MAX), truncated: true }); diff --git a/test/decision-diff-binary.test.ts b/test/decision-diff-binary.test.ts new file mode 100644 index 000000000..dd4a9086d --- /dev/null +++ b/test/decision-diff-binary.test.ts @@ -0,0 +1,108 @@ +// The owner's decision diff never shows a binary file's bytes (#927 round 10). The diff is read with `--text` (#807 F2: +// a NUL, or a `-diff` attribute, is also how a change hides its text from review as "binary"), and round 9 showed it +// with its NULs dropped. That showed a binary file's printable bytes (a DER private key's), and it glued byte runs +// together, so a key shape a NUL had ended lost the boundary its redaction rule needs (both measured or inferred by +// round 9's reviews). main shows "Binary files differ". Now a binary file's lines show only as the text they decode to, +// and a NUL shows as a mark that is still a boundary. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { generateKeyPairSync } from 'node:crypto'; +import { mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { decisionDiff } from '../src/decision-detail.js'; + +/** A scratch repo with a base commit, and a commit adding `files`: its diff as the owner's decision view shows it. */ +async function shown(files: Record): Promise { + const repo = mkdtempSync(join(tmpdir(), 'decision-binary-')); + const git = (...a: string[]) => execFileSync('git', a, { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + git('init', '-q', '-b', 'main'); + for (const [k, v] of [['user.name', 'Ada'], ['user.email', 'ada@example.com'], ['commit.gpgsign', 'false']]) git('config', k!, v!); + writeFileSync(join(repo, 'a.txt'), 'a\n'); + git('add', '-A'); + git('commit', '-q', '-m', 'base'); + const base = git('rev-parse', 'HEAD').trim(); + for (const [name, data] of Object.entries(files)) writeFileSync(join(repo, name), data); + git('add', '-A'); + git('commit', '-q', '-m', 'change'); + const head = git('rev-parse', 'HEAD').trim(); + const d = await decisionDiff(repo, base, head); + assert.ok(d.diff !== null, JSON.stringify(d)); + return d.diff; +} + +/** Every run of printable ASCII bytes in `b` at least `n` long. */ +function printableRuns(b: Buffer, n: number): string[] { + return b.toString('latin1').match(new RegExp(`[\\x20-\\x7e]{${n},}`, 'g')) ?? []; +} + +/** A fresh DER private key with at least one run of 6 printable bytes, so the check below has something to find. */ +function derKey(make: () => Buffer): Buffer { + for (let i = 0; i < 400; i++) { + const k = make(); + if (printableRuns(k, 6).length) return k; + } + throw new Error('no key with a printable run of 6 in 400 tries'); +} + +test('a DER private key (ed25519 PKCS#8, RSA-2048 PKCS#1) shows no 6-byte run of its printable bytes', async () => { + const ed = derKey(() => generateKeyPairSync('ed25519').privateKey.export({ type: 'pkcs8', format: 'der' })); + const rsa = derKey(() => generateKeyPairSync('rsa', { modulusLength: 2048 }).privateKey.export({ type: 'pkcs1', format: 'der' })); + const diff = await shown({ 'id_ed25519.der': ed, 'id_rsa.der': rsa }); + assert.match(diff, /id_ed25519\.der/); + assert.match(diff, /id_rsa\.der/); + for (const [name, key] of [['ed25519', ed], ['rsa', rsa]] as const) { + const leaked: string[] = []; + for (const run of printableRuns(key, 6)) for (let i = 0; i + 6 <= run.length; i++) if (diff.includes(run.slice(i, i + 6))) leaked.push(run.slice(i, i + 6)); + assert.deepEqual(leaked, [], `${name}: printable bytes of the key are shown`); + } +}); + +// Fixed-length key shapes, each between NULs, with a letter or digit right after the trailing NUL: with the NULs +// dropped, the rule's boundary (a lookahead, or `\b` before) is gone. Fake shapes, none of them real. +const SHAPES: [string, string][] = [ + ['AWS access key', `AKIA${'Q7W2R9T4Y6H3L8V5'}`], + ['Google API key', `AIza${'SyD7k2Lm9Qr3Ws8Xv5Zb1Nc4Hd6Jf0Gp2T'.padEnd(35, 'a').slice(0, 35)}`], + ['32-byte hex key', `0x${'9f3a'.repeat(16)}`], +]; +for (const [what, key] of SHAPES) { + test(`a ${what} between NULs in a binary file, a letter after the trailing NUL, is redacted`, async () => { + const body = Buffer.concat([Buffer.from('a binary blob with a key inside it\n'), Buffer.from('k'), Buffer.from([0]), Buffer.from(key), Buffer.from([0]), Buffer.from('x tail bytes\n')]); + const diff = await shown({ 'blob.bin': body }); + assert.equal(diff.includes(key.slice(4, 20)), false, `the key is shown: ${JSON.stringify(diff.slice(-300))}`); + assert.match(diff, /a binary blob with a key inside it/, 'the text line still shows'); + }); +} + +/** `text` as UTF-32 with a byte-order mark. */ +function utf32(text: string, order: 'le' | 'be'): Buffer { + const points = [0xfeff, ...Array.from(text, (c) => c.codePointAt(0)!)]; + const b = Buffer.alloc(points.length * 4); + points.forEach((p, i) => (order === 'le' ? b.writeUInt32LE(p, i * 4) : b.writeUInt32BE(p, i * 4))); + return b; +} + +for (const order of ['le', 'be'] as const) { + test(`a UTF-32${order.toUpperCase()} file shows its lines as text, and a token in it is redacted`, async () => { + const token = `ghp_${'Utf32Tk9Lm3Qr7Ws2Xv5Zb8Nc4Hd6Jf1Gp0'.padEnd(36, 'a').slice(0, 36)}`; + const diff = await shown({ 'notes32.txt': utf32(`first line of the notes\ntoken: ${token}\nthe last line\n`, order) }); + const seen = diff.replace(/␀/g, ''); + assert.equal(seen.includes(token.slice(4, 20)), false, `the token is shown: ${JSON.stringify(diff.slice(-300))}`); + assert.match(diff, /first line of the notes/); + assert.match(diff, /the last line/); + }); +} + +test('a text file with one NUL still shows its lines, the NUL as a mark', async () => { + const body = Buffer.concat([ + Buffer.from('the first line of this text file\nthe second line holds one '), + Buffer.from([0]), + Buffer.from(' NUL in the middle of it\nand the third line is plain text again\n'), + ]); + const diff = await shown({ 'notes.txt': body }); + assert.match(diff, /\+the first line of this text file\n/); + assert.match(diff, /\+the second line holds one ␀ NUL in the middle of it\n/); + assert.match(diff, /\+and the third line is plain text again\n/); + assert.equal(diff.includes('\0'), false); +}); From 0b1c7da044cc9a4a44a227513448e35c4a02ebaf Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 22:44:56 -0500 Subject: [PATCH 15/18] decision detail: the binary view decides per hunk, so it reads no file header (#927) CI at 633b03d8 failed test/change-files-renames.test.ts's "one helper is the only way src/ lists changed names or counts" in shard 2/4 on Node 22 and 24: diffTextView split the diff into files at their `diff --git` header lines, which that lint's "patch headers" words find, and src/decision-detail.ts isn't on its never-grow list. The view now decides per hunk, from its `@@ ` line to the first line that doesn't start with `+`, `-`, a space or `\`, and reads no file header. A hunk is one file's, so what it shows is the same. The lint is unchanged. Test (fails without): test/change-files-renames.test.ts's lint, as CI ran it at 633b03d8. test/decision-diff-binary and test/decision-diff-utf16 still pass. Signed-off-by: Jayden Vasquez --- src/decision-detail.ts | 50 +++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/src/decision-detail.ts b/src/decision-detail.ts index 6dfcc4f81..9412d4c82 100644 --- a/src/decision-detail.ts +++ b/src/decision-detail.ts @@ -433,41 +433,35 @@ function binaryLine(prefix: string, body: Buffer): string { } /** - * What the owner's decision diff shows of `git diff --text` output (#927 round 10). A file whose diff lines hold a - * NUL, invalid UTF-8 or a control byte text doesn't hold is binary: each of its lines shows only as text it decodes - * to (UTF-16 or UTF-8, a NUL as NUL_MARK, a UTF-8 line at least MIN_BINARY_TEXT characters), and otherwise as a marker. - * So no run of a binary file's bytes is shown, while its text lines are, and a NUL still separates what it separated - * for the redaction rules that need a boundary. Every other file shows as it is. + * What the owner's decision diff shows of `git diff --text` output (#927 round 10). A hunk whose lines hold a NUL, + * invalid UTF-8 or a control byte text doesn't hold is binary: each of its lines shows only as text it decodes to + * (UTF-32, UTF-16 or UTF-8, a NUL as NUL_MARK, a UTF-8 line at least MIN_BINARY_TEXT characters), and otherwise as a + * marker. So no run of a binary file's bytes is shown, while its text lines are, and a NUL still separates what it + * separated for the redaction rules that need a boundary. Every other line shows as it is. A hunk runs from its `@@ ` + * line to the first line that doesn't start with `+`, `-`, a space or `\`. */ export function diffTextView(raw: Buffer): string { - const lines: Buffer[] = []; + const out: string[] = []; + let hunk: Buffer[] | null = null; + const isContent = (l: Buffer) => l[0] === 0x2b || l[0] === 0x2d || l[0] === 0x20; + const flush = () => { + if (!hunk) return; + const binary = hunk.some((l) => isContent(l) && utf8Text(l.subarray(1))?.includes('\0') !== false); + for (const l of hunk) out.push(binary && isContent(l) ? binaryLine(String.fromCharCode(l[0]!), l.subarray(1)) : l.toString('utf8')); + hunk = null; + }; for (let at = 0; at <= raw.length; ) { const nl = raw.indexOf(0x0a, at); - const end = nl === -1 ? raw.length : nl; - lines.push(raw.subarray(at, end)); + const l = raw.subarray(at, nl === -1 ? raw.length : nl); + if (hunk && l.length > 0 && (isContent(l) || l[0] === 0x5c)) hunk.push(l); + else { + flush(); + if (l.subarray(0, 3).toString('latin1') === '@@ ') hunk = []; + out.push(l.toString('utf8')); + } if (nl === -1) break; at = nl + 1; } - const out: string[] = []; - let file: Buffer[] = []; - const flush = () => { - let inHunk = false; - const content = (l: Buffer) => inHunk && l.length > 0 && (l[0] === 0x2b || l[0] === 0x2d || l[0] === 0x20); - const binary = file.some((l) => { - if (l.subarray(0, 3).toString('latin1') === '@@ ') inHunk = true; - return content(l) && utf8Text(l.subarray(1))?.includes('\0') !== false; - }); - inHunk = false; - for (const l of file) { - if (l.subarray(0, 3).toString('latin1') === '@@ ') inHunk = true; - out.push(binary && content(l) ? binaryLine(String.fromCharCode(l[0]!), l.subarray(1)) : l.toString('utf8')); - } - file = []; - }; - for (const l of lines) { - if (l.subarray(0, 11).toString('latin1') === 'diff --git ') flush(); - file.push(l); - } flush(); return out.join('\n'); } From 99b27ecebf3c178fff8845b3859766641302d474 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Wed, 21 Oct 2026 23:39:43 -0500 Subject: [PATCH 16/18] decision and culprit diffs: main's arguments via the runner; --text and the binary view go (#927) The owner, 2026-10-21: "Cut --text, land the runner." Every required change in both round-10 reviews came from `--text` and the view that tried to show a binary file's text lines safely: a UTF-32 line that crashed it, a hunk header's function-context bytes, diff.suppressBlankEmpty ending a hunk early, and the fix-forward culprit diff carrying a binary file's bytes into a tracker issue. - src/decision-detail.ts: decisionDiff runs main's `git diff --no-color --no-ext-diff base..head` through gitExecFile; diffTextView and its decoders are removed. A binary file shows as "Binary files ... differ", as on main, and output past the buffer is no diff, as on main. - src/coordinator/merge-flow.ts: the culprit diff is main's `git diff ^1 ` again, through the runner (gitOrNull), not patchText. patchText (`--text`) stays for the patch hash only, which is hashed, never shown. - The PR's own tests for the removed behaviour go: test/decision-diff-binary, decision-diff-utf16 and decision-diff-overflow (none is on main). - changelog.d/927-git-runner.md: scoped to the JS and TS code the scan reads, the .sh helpers and the old hook named, the strict GIT_CONFIG_COUNT named, and the two diffs said to be as before. Test (fails without): test/binary-diffs-as-main.test.ts. Each diff shows "Binary files /dev/null and b/blob.bin differ" for a binary file and a text file's lines. As a new file in a fresh worktree at 0b1c7da0, 2 of 2 fail. Signed-off-by: Jayden Vasquez --- changelog.d/927-git-runner.md | 26 +++-- src/coordinator/merge-flow.ts | 11 +-- src/decision-detail.ts | 142 ++-------------------------- test/binary-diffs-as-main.test.ts | 46 +++++++++ test/decision-diff-binary.test.ts | 108 --------------------- test/decision-diff-overflow.test.ts | 63 ------------ test/decision-diff-utf16.test.ts | 51 ---------- 7 files changed, 72 insertions(+), 375 deletions(-) create mode 100644 test/binary-diffs-as-main.test.ts delete mode 100644 test/decision-diff-binary.test.ts delete mode 100644 test/decision-diff-overflow.test.ts delete mode 100644 test/decision-diff-utf16.test.ts diff --git a/changelog.d/927-git-runner.md b/changelog.d/927-git-runner.md index fbc00de20..5aee0b10b 100644 --- a/changelog.d/927-git-runner.md +++ b/changelog.d/927-git-runner.md @@ -1,23 +1,29 @@ ### Upgrading (from v0.5.2) -- **Every git the engine runs ignores replace refs and grafts (#927).** An agent could write `refs/replace/*` or - `.git/info/grafts` in the shared repo and make the push limits, the secret scan and the review read other commits. - Nothing to do; a change's recorded `patch_hash` now covers every file's lines, so it differs from an older engine's - for the same change. Run `npm run setup:hooks` once to get the push-gate hook that also hands the gate these - settings (the old hook keeps working). +- **Every git the engine's JS and TS code runs ignores replace refs and grafts (#927).** An agent could write + `refs/replace/*` or `.git/info/grafts` in the shared repo and make the push limits, the secret scan and the review + read other commits. The `.sh` helpers under `scripts/` don't go through the runner. A change's recorded + `patch_hash` now covers every file's lines, so it differs from an older engine's for the same change. Run + `npm run setup:hooks` once after upgrading: until then, an install's old copy of the push-gate hook runs its own + git calls (the one that loads the gate among them) without these settings. It keeps working. +- **A `GIT_CONFIG_COUNT` that isn't a plain decimal from 0 to 9999 now makes git refuse (#927)**, for every git the + runner starts and in the push gate: the runner can't add its setting after a count it can't read. Nothing in the + engine writes such a count; an environment that sets one must write it plainly. ### Guardrail changes: what each one does - **The git runner (#927):** `src/os/git.ts` (and `scripts/lib/git.cjs` for scripts) starts every git with replace refs off and no graft file, over whatever environment the caller passes; gitleaks gets the same settings. A test - fails on a git call in `src/` or `scripts/` that doesn't come through it, and the runner and that test wait for the - owner. + fails on a git call in the JS and TS files under `src/` and `scripts/` that doesn't come through it (it doesn't + read `.sh` files, git named in a shell string, or a name built at run time), and the runner and that test wait for + the owner. - **A shallow cut in a pushed range is refused (#927):** a `.git/shallow` (a shallow clone's, or one written by hand) that cuts the range being pushed made the push limits read only part of it. The push is now refused, and the message names `git fetch --unshallow`. A shallow clone still pushes changes that lie wholly above its cut. The push limits and the change's secret scan read one checked copy of the shallow file, so changing it while they run changes nothing, and a shallow file with a line git might read differently is refused. -- **The commit-graph is off for every git the engine runs (#927):** a forged commit-graph could make a walk skip a +- **The commit-graph is off for every git the runner starts (#927):** a forged commit-graph could make a walk skip a commit; git now reads each commit's own parents (measured: about 30 ms more for three walks over 2,900 commits). -- **What the owner sees (#927):** the decision diff, the fix-forward culprit diff and the patch hash show a file's - lines even when the change's `.gitattributes` marks it `-diff` or `binary`. +- **The patch hash (#927)** covers a file's lines even when the change's `.gitattributes` marks it `-diff` or + `binary`. The owner's decision diff and the fix-forward culprit diff are as before: a binary file shows as + "Binary files ... differ". diff --git a/src/coordinator/merge-flow.ts b/src/coordinator/merge-flow.ts index b5f3df766..fcc565bad 100644 --- a/src/coordinator/merge-flow.ts +++ b/src/coordinator/merge-flow.ts @@ -19,7 +19,7 @@ import { instructionTargets } from '../skilleval.js'; import { checkedPush } from '../push-check.js'; import { conflictTrigger } from '../conflicts.js'; import { abortMerge, mergeBaseInto } from '../merge-base.js'; -import { changedNamesOrNull, patchText } from './change-files.js'; +import { changedNamesOrNull } from './change-files.js'; import { lightOutcome } from '../light-check.js'; import { readiness, requiredOutcomes, type Readiness } from '../pr-watch.js'; import { checksFor, effectiveClass, riskClassOf, rulesetHold } from '../pr-rules.js'; @@ -128,14 +128,7 @@ export abstract class MergeFlow extends FixRuns { subject: (sha) => git('log', '-1', '--format=%s', sha) ?? '', // A rename as both its paths and a valid UTF-8 path as itself, so a locked category on either one holds (#683). files: (sha) => changedNamesOrNull(this.d.repo, [`${sha}^1`, sha]) ?? [], - // The culprit's patch as text whatever its attributes say (#807): a `-diff` file shows its lines, not "Binary files differ". - diff: (sha) => { - try { - return patchText(this.d.repo, [`${sha}^1`, sha]); - } catch { - return ''; - } - }, + diff: (sha) => git('diff', `${sha}^1`, sha) ?? '', }, config: () => this.d.cfg.project.fix_forward, required: () => this.requiredChecks(), diff --git a/src/decision-detail.ts b/src/decision-detail.ts index 9412d4c82..402b732e5 100644 --- a/src/decision-detail.ts +++ b/src/decision-detail.ts @@ -346,147 +346,21 @@ export function decisionDetail(id: string, d: DetailDeps): DecisionDetail | null }; } -const UTF8_STRICT = new TextDecoder('utf-8', { fatal: true }); -/** The control bytes a text line may hold besides NUL: tab, vertical tab, form feed, carriage return and escape. */ -const TEXT_CONTROLS = new Set([0x09, 0x0b, 0x0c, 0x0d, 0x1b]); -/** What a NUL (a run of them) in a shown line becomes: not a letter, digit or `_`, so every redaction rule still sees a boundary there. */ -export const NUL_MARK = '␀'; -/** A binary file's line that is shown as UTF-8 text needs this many characters: random bytes almost never make one. */ -const MIN_BINARY_TEXT = 16; - -/** `b` as UTF-8 text, NULs kept as U+0000; or null for invalid UTF-8, DEL, or a control byte text doesn't hold. */ -function utf8Text(b: Buffer): string | null { - for (const c of b) if ((c < 0x20 && c !== 0 && !TEXT_CONTROLS.has(c)) || c === 0x7f) return null; - try { - return UTF8_STRICT.decode(b); - } catch { - return null; - } -} - -/** `s` (whole units of `width` bytes) as UTF-16 or UTF-32 text, when its NULs fall as mostly-Latin text's do; or null. */ -function decodeWide(s: Buffer, width: 2 | 4): string | null { - const units = s.length / width; - if (units < 1) return null; - let t = ''; - if (width === 2) { - let low = 0; - let high = 0; - for (let i = 0; i < s.length; i += 2) { - if (s[i] === 0) low++; - if (s[i + 1] === 0) high++; - } - const order = high >= units * 0.75 && low <= units * 0.25 ? 'utf-16le' : low >= units * 0.75 && high <= units * 0.25 ? 'utf-16be' : null; - if (!order) return null; - try { - t = new TextDecoder(order, { fatal: true, ignoreBOM: false }).decode(s); - } catch { - return null; - } - } else { - const unitLE = (i: number) => s[i]! | (s[i + 1]! << 8) | (s[i + 2]! << 16) | (s[i + 3]! << 24); - const unitBE = (i: number) => s[i + 3]! | (s[i + 2]! << 8) | (s[i + 1]! << 16) | (s[i]! << 24); - for (const read of [unitLE, unitBE]) { - const points: number[] = []; - let narrow = 0; - for (let i = 0; i < s.length; i += 4) points.push(read(i) >>> 0); - for (const p of points) if (p < 0x100) narrow++; - if (narrow < units * 0.75 || points.some((p) => p > 0x10ffff || (p >= 0xd800 && p <= 0xdfff))) continue; - t = String.fromCodePoint(...(points[0] === 0xfeff ? points.slice(1) : points)); - break; - } - if (!t) return null; - } - for (const ch of t) { - const c = ch.codePointAt(0)!; - if ((c < 0x20 && c !== 0 && !TEXT_CONTROLS.has(c)) || c === 0x7f) return null; - } - return t; -} - -/** - * `b` as UTF-32 or UTF-16 text (little- or big-endian, as its NULs fall), or null. git splits such lines at the 0x0A - * byte, so a line can start or end with the rest of a newline's unit; those NUL bytes are dropped. Only a line whose - * NULs fall as mostly-Latin text's do is read so (random bytes don't), and only when it decodes cleanly to text. - */ -function wideText(b: Buffer): string | null { - for (const width of [4, 2] as const) { - const r = b.length % width; - const tries = r === 0 ? [b] : [b.subarray(0, r).every((x) => x === 0) ? b.subarray(r) : null, b.subarray(b.length - r).every((x) => x === 0) ? b.subarray(0, b.length - r) : null]; - for (const s of tries) { - const t = s && decodeWide(s, width); - if (t) return t; - } - } - return null; -} - -/** One line of a binary file's diff (`prefix`, then its bytes) as shown: its text, or a marker that holds none of it. */ -function binaryLine(prefix: string, body: Buffer): string { - const wide = wideText(body); - const t = wide ?? utf8Text(body); - if (t !== null) { - const chars = t.replace(/\0/g, '').length; - if (wide !== null || chars === 0 || chars >= MIN_BINARY_TEXT) return `${prefix}${t.replace(/\0+/g, NUL_MARK)}`; - } - return `${prefix}[binary: ${body.length} bytes not shown]`; -} - /** - * What the owner's decision diff shows of `git diff --text` output (#927 round 10). A hunk whose lines hold a NUL, - * invalid UTF-8 or a control byte text doesn't hold is binary: each of its lines shows only as text it decodes to - * (UTF-32, UTF-16 or UTF-8, a NUL as NUL_MARK, a UTF-8 line at least MIN_BINARY_TEXT characters), and otherwise as a - * marker. So no run of a binary file's bytes is shown, while its text lines are, and a NUL still separates what it - * separated for the redaction rules that need a boundary. Every other line shows as it is. A hunk runs from its `@@ ` - * line to the first line that doesn't start with `+`, `-`, a space or `\`. + * The change's diff, base..head, from the checkout's git (owners only: the route checks); redacted, capped. main's + * arguments, through the git runner (#807): a binary file shows as "Binary files ... differ" (the owner, 2026-10-21: + * "Cut --text, land the runner."). */ -export function diffTextView(raw: Buffer): string { - const out: string[] = []; - let hunk: Buffer[] | null = null; - const isContent = (l: Buffer) => l[0] === 0x2b || l[0] === 0x2d || l[0] === 0x20; - const flush = () => { - if (!hunk) return; - const binary = hunk.some((l) => isContent(l) && utf8Text(l.subarray(1))?.includes('\0') !== false); - for (const l of hunk) out.push(binary && isContent(l) ? binaryLine(String.fromCharCode(l[0]!), l.subarray(1)) : l.toString('utf8')); - hunk = null; - }; - for (let at = 0; at <= raw.length; ) { - const nl = raw.indexOf(0x0a, at); - const l = raw.subarray(at, nl === -1 ? raw.length : nl); - if (hunk && l.length > 0 && (isContent(l) || l[0] === 0x5c)) hunk.push(l); - else { - flush(); - if (l.subarray(0, 3).toString('latin1') === '@@ ') hunk = []; - out.push(l.toString('utf8')); - } - if (nl === -1) break; - at = nl + 1; - } - flush(); - return out.join('\n'); -} - -/** The change's diff, base..head, from the checkout's git (owners only: the route checks); redacted, capped. */ export async function decisionDiff(root: string, base: string, head: string): Promise<{ diff: string; truncated: boolean } | { diff: null; why: string }> { if (!SHA.test(base) || !SHA.test(head)) return { diff: null, why: 'no commits recorded for this change' }; const MAX = 512 * 1024; - // Every file's lines whatever the change's attributes say (#807): a file its .gitattributes marks `-diff` or - // `binary` (or gives a textconv or an external diff) shows as it is, not as "Binary files differ"; the user's - // attributes file is not read. Output past the buffer is shown cut, like output past MAX. - const args = ['-c', 'core.attributesFile=/dev/null', 'diff', '--no-color', '--text', '--no-textconv', '--no-ext-diff', '--ignore-submodules=none', `${base}..${head}`]; return new Promise((done) => { - // latin1: one character per byte, so the bytes come back as git wrote them (diffTextView reads them). - gitExecFile(args, { cwd: root, env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: 'latin1' }, (err, stdout, stderr) => { - if (err && !(err.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER' && stdout)) return done({ diff: null, why: `not in this checkout's git: ${(Buffer.from(stderr, 'latin1').toString('utf8') || err.message).trim().split('\n')[0]}` }); - // `--text` stays (#807 F2): a NUL, or a `-diff` attribute, is also how a change hides its text from review as - // "binary". But a binary file's bytes are never shown (#927 round 10): where main showed "Binary files differ", - // its lines show only as the text they decode to (UTF-16 included), each NUL a boundary mark (diffTextView). - const shown = diffTextView(Buffer.from(stdout, 'latin1')); + gitExecFile(['diff', '--no-color', '--no-ext-diff', `${base}..${head}`], { cwd: root, env: { ...withToolPath(), GIT_TERMINAL_PROMPT: '0' }, timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: 'utf8' }, (err, stdout, stderr) => { + if (err) return done({ diff: null, why: `not in this checkout's git: ${(stderr || err.message).trim().split('\n')[0]}` }); + const truncated = stdout.length > MAX; // Redacted before it is cut, over the kept part and a margin only (#889): up to 8 MB of diff, and the redaction is - // synchronous on the dashboard. Output past the buffer the same way: a token across the cut is redacted whole. - if (err) return done({ diff: redactCut(shown, MAX), truncated: true }); - const truncated = shown.length > MAX; - done({ diff: redactCut(shown, MAX), truncated }); + // synchronous on the dashboard. + done({ diff: redactCut(stdout, MAX), truncated }); }); }); } diff --git a/test/binary-diffs-as-main.test.ts b/test/binary-diffs-as-main.test.ts new file mode 100644 index 000000000..b11dd4615 --- /dev/null +++ b/test/binary-diffs-as-main.test.ts @@ -0,0 +1,46 @@ +// The owner's decision diff and the fix-forward culprit diff show a binary file as main does (#927; the owner's +// decision, 2026-10-21: "Cut --text, land the runner."). Both went through git with `--text` in this PR, which showed +// a binary file's bytes (on the owner's dashboard, and in a fix task's tracker issue). They run through the git runner +// as before, with main's arguments: a binary file is "Binary files ... differ", a text file shows its lines. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { decisionDiff } from '../src/decision-detail.js'; +import { agents, commitAll, coordinator, fixture, git } from './repro-state-graph/harness.js'; + +/** A binary file (a NUL in it) holding a printable run that must not show, and a control byte. */ +const SECRET_RUN = 'NotShownRun7Qx2'; +const BLOB = Buffer.concat([Buffer.from(`header line\n${SECRET_RUN}`), Buffer.from([0, 1, 2, 0x0a]), Buffer.from('tail\n')]); + +/** In the fixture's clone: a commit adding a binary file and changing a text file. Its id and its parent's. */ +function culprit(repo: string): { base: string; head: string } { + const base = git(repo, 'rev-parse', 'HEAD'); + writeFileSync(join(repo, 'blob.bin'), BLOB); + writeFileSync(join(repo, 'notes.md'), 'a text line the diff shows\n'); + commitAll(repo, 'add a blob and notes'); + return { base, head: git(repo, 'rev-parse', 'HEAD') }; +} + +function asMain(diff: string) { + assert.match(diff, /Binary files \/dev\/null and b\/blob\.bin differ/, 'the binary file shows as main shows it'); + assert.match(diff, /^\+a text line the diff shows$/m, "the text file's lines show"); + assert.equal(diff.includes(SECRET_RUN), false, "none of the binary file's bytes show"); + assert.equal(diff.includes('\0'), false); +} + +test("the owner's decision diff shows a binary file as \"Binary files differ\", and a text file's lines", async () => { + const f = fixture(); + const { base, head } = culprit(f.repo); + const d = await decisionDiff(f.repo, base, head); + assert.ok(d.diff !== null, JSON.stringify(d)); + asMain(d.diff); +}); + +test('the fix-forward culprit diff (what a fix task\'s issue body holds) shows a binary file as "Binary files differ"', () => { + const f = fixture(); + const { head } = culprit(f.repo); + const c = coordinator(f, agents()); + const ff = (c as unknown as { fixForward: { d: { git: { diff: (sha: string) => string } } } }).fixForward; + asMain(ff.d.git.diff(head)); +}); diff --git a/test/decision-diff-binary.test.ts b/test/decision-diff-binary.test.ts deleted file mode 100644 index dd4a9086d..000000000 --- a/test/decision-diff-binary.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -// The owner's decision diff never shows a binary file's bytes (#927 round 10). The diff is read with `--text` (#807 F2: -// a NUL, or a `-diff` attribute, is also how a change hides its text from review as "binary"), and round 9 showed it -// with its NULs dropped. That showed a binary file's printable bytes (a DER private key's), and it glued byte runs -// together, so a key shape a NUL had ended lost the boundary its redaction rule needs (both measured or inferred by -// round 9's reviews). main shows "Binary files differ". Now a binary file's lines show only as the text they decode to, -// and a NUL shows as a mark that is still a boundary. -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { execFileSync } from 'node:child_process'; -import { generateKeyPairSync } from 'node:crypto'; -import { mkdtempSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { decisionDiff } from '../src/decision-detail.js'; - -/** A scratch repo with a base commit, and a commit adding `files`: its diff as the owner's decision view shows it. */ -async function shown(files: Record): Promise { - const repo = mkdtempSync(join(tmpdir(), 'decision-binary-')); - const git = (...a: string[]) => execFileSync('git', a, { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); - git('init', '-q', '-b', 'main'); - for (const [k, v] of [['user.name', 'Ada'], ['user.email', 'ada@example.com'], ['commit.gpgsign', 'false']]) git('config', k!, v!); - writeFileSync(join(repo, 'a.txt'), 'a\n'); - git('add', '-A'); - git('commit', '-q', '-m', 'base'); - const base = git('rev-parse', 'HEAD').trim(); - for (const [name, data] of Object.entries(files)) writeFileSync(join(repo, name), data); - git('add', '-A'); - git('commit', '-q', '-m', 'change'); - const head = git('rev-parse', 'HEAD').trim(); - const d = await decisionDiff(repo, base, head); - assert.ok(d.diff !== null, JSON.stringify(d)); - return d.diff; -} - -/** Every run of printable ASCII bytes in `b` at least `n` long. */ -function printableRuns(b: Buffer, n: number): string[] { - return b.toString('latin1').match(new RegExp(`[\\x20-\\x7e]{${n},}`, 'g')) ?? []; -} - -/** A fresh DER private key with at least one run of 6 printable bytes, so the check below has something to find. */ -function derKey(make: () => Buffer): Buffer { - for (let i = 0; i < 400; i++) { - const k = make(); - if (printableRuns(k, 6).length) return k; - } - throw new Error('no key with a printable run of 6 in 400 tries'); -} - -test('a DER private key (ed25519 PKCS#8, RSA-2048 PKCS#1) shows no 6-byte run of its printable bytes', async () => { - const ed = derKey(() => generateKeyPairSync('ed25519').privateKey.export({ type: 'pkcs8', format: 'der' })); - const rsa = derKey(() => generateKeyPairSync('rsa', { modulusLength: 2048 }).privateKey.export({ type: 'pkcs1', format: 'der' })); - const diff = await shown({ 'id_ed25519.der': ed, 'id_rsa.der': rsa }); - assert.match(diff, /id_ed25519\.der/); - assert.match(diff, /id_rsa\.der/); - for (const [name, key] of [['ed25519', ed], ['rsa', rsa]] as const) { - const leaked: string[] = []; - for (const run of printableRuns(key, 6)) for (let i = 0; i + 6 <= run.length; i++) if (diff.includes(run.slice(i, i + 6))) leaked.push(run.slice(i, i + 6)); - assert.deepEqual(leaked, [], `${name}: printable bytes of the key are shown`); - } -}); - -// Fixed-length key shapes, each between NULs, with a letter or digit right after the trailing NUL: with the NULs -// dropped, the rule's boundary (a lookahead, or `\b` before) is gone. Fake shapes, none of them real. -const SHAPES: [string, string][] = [ - ['AWS access key', `AKIA${'Q7W2R9T4Y6H3L8V5'}`], - ['Google API key', `AIza${'SyD7k2Lm9Qr3Ws8Xv5Zb1Nc4Hd6Jf0Gp2T'.padEnd(35, 'a').slice(0, 35)}`], - ['32-byte hex key', `0x${'9f3a'.repeat(16)}`], -]; -for (const [what, key] of SHAPES) { - test(`a ${what} between NULs in a binary file, a letter after the trailing NUL, is redacted`, async () => { - const body = Buffer.concat([Buffer.from('a binary blob with a key inside it\n'), Buffer.from('k'), Buffer.from([0]), Buffer.from(key), Buffer.from([0]), Buffer.from('x tail bytes\n')]); - const diff = await shown({ 'blob.bin': body }); - assert.equal(diff.includes(key.slice(4, 20)), false, `the key is shown: ${JSON.stringify(diff.slice(-300))}`); - assert.match(diff, /a binary blob with a key inside it/, 'the text line still shows'); - }); -} - -/** `text` as UTF-32 with a byte-order mark. */ -function utf32(text: string, order: 'le' | 'be'): Buffer { - const points = [0xfeff, ...Array.from(text, (c) => c.codePointAt(0)!)]; - const b = Buffer.alloc(points.length * 4); - points.forEach((p, i) => (order === 'le' ? b.writeUInt32LE(p, i * 4) : b.writeUInt32BE(p, i * 4))); - return b; -} - -for (const order of ['le', 'be'] as const) { - test(`a UTF-32${order.toUpperCase()} file shows its lines as text, and a token in it is redacted`, async () => { - const token = `ghp_${'Utf32Tk9Lm3Qr7Ws2Xv5Zb8Nc4Hd6Jf1Gp0'.padEnd(36, 'a').slice(0, 36)}`; - const diff = await shown({ 'notes32.txt': utf32(`first line of the notes\ntoken: ${token}\nthe last line\n`, order) }); - const seen = diff.replace(/␀/g, ''); - assert.equal(seen.includes(token.slice(4, 20)), false, `the token is shown: ${JSON.stringify(diff.slice(-300))}`); - assert.match(diff, /first line of the notes/); - assert.match(diff, /the last line/); - }); -} - -test('a text file with one NUL still shows its lines, the NUL as a mark', async () => { - const body = Buffer.concat([ - Buffer.from('the first line of this text file\nthe second line holds one '), - Buffer.from([0]), - Buffer.from(' NUL in the middle of it\nand the third line is plain text again\n'), - ]); - const diff = await shown({ 'notes.txt': body }); - assert.match(diff, /\+the first line of this text file\n/); - assert.match(diff, /\+the second line holds one ␀ NUL in the middle of it\n/); - assert.match(diff, /\+and the third line is plain text again\n/); - assert.equal(diff.includes('\0'), false); -}); diff --git a/test/decision-diff-overflow.test.ts b/test/decision-diff-overflow.test.ts deleted file mode 100644 index 3aa9c1fc1..000000000 --- a/test/decision-diff-overflow.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -// The owner's decision diff past git's 8 MiB output buffer (#927 round 8). #921 (#889) redacts before it cuts, so a -// token across the 512 KiB cut is redacted whole. This PR's buffer branch, which shows output past the buffer instead -// of no diff, cut first and redacted after (`redactString(stdout.slice(0, MAX))`): the head of a token across the cut -// was too short for its rule and was shown. The fixture is a real diff over 8 MiB, so the branch runs as it does. -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { execFileSync } from 'node:child_process'; -import { mkdtempSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { decisionDiff } from '../src/decision-detail.js'; - -const MAX = 512 * 1024; -const BUFFER = 8 * 1024 * 1024; -const FILLER = 'x'.repeat(60); -const TOKENS = 100; -/** A distinct GitHub-token-shaped string per line (ghp_ and 36 letters and digits), none of them real. */ -const token = (i: number) => `ghp_${String(i).padStart(4, '0')}${'Qz7Wk2Rm9Tb4Yp6Hn3Lc8Vd5Fs1Gj0Xa'.padEnd(32, 'a').slice(0, 32)}`; -/** The visible head of the token the cut lands in: `ghp_` and four more characters, under every token rule's length. */ -const VISIBLE = 8; - -test('a decision diff past the 8 MiB buffer is redacted before it is cut: no part of a token across the cut is shown', async () => { - assert.equal(token(0).length, 40, 'a token line is 40 characters (42 as a diff line)'); - const repo = mkdtempSync(join(tmpdir(), 'decision-overflow-')); - const git = (...a: string[]) => execFileSync('git', a, { cwd: repo, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'] }); - git('init', '-q', '-b', 'main'); - for (const [k, v] of [['user.name', 'Ada'], ['user.email', 'ada@example.com'], ['commit.gpgsign', 'false']]) git('config', k!, v!); - writeFileSync(join(repo, 'a.txt'), 'a\n'); - git('add', '-A'); - git('commit', '-q', '-m', 'base'); - const base = git('rev-parse', 'HEAD').trim(); - // big.txt: a pad line, filler up to just before the cut, the token lines, then filler past the buffer. Diff lines - // are `+` and the line: filler 62 characters, a token 42. - const before = Math.floor((MAX - 50 * 42) / 62) - 5; - const after = Math.ceil((BUFFER + 512 * 1024) / 62); - const file = (pad: number) => [`${'y'.repeat(pad)}`, ...Array(before).fill(FILLER), ...Array.from({ length: TOKENS }, (_, i) => token(i)), ...Array(after).fill(FILLER)].join('\n') + '\n'; - const diffArgs = (head: string) => ['-c', 'core.attributesFile=/dev/null', 'diff', '--no-color', '--text', '--no-textconv', '--no-ext-diff', '--ignore-submodules=none', `${base}..${head}`]; - // First with a one-character pad, to measure the diff's header; then with the pad that puts the cut inside a token, - // VISIBLE characters in. The line count and the header's length stay the same. - writeFileSync(join(repo, 'big.txt'), file(1)); - git('add', '-A'); - git('commit', '-q', '-m', 'measure'); - const header = git(...diffArgs('HEAD')).indexOf('\n+y\n') + 1; - const firstToken = (pad: number) => header + (1 + pad + 1) + before * 62; - const pad = ((((MAX - VISIBLE - 1 - firstToken(0)) % 42) + 42) % 42) || 42; - writeFileSync(join(repo, 'big.txt'), file(pad)); - git('add', '-A'); - git('commit', '-q', '-m', 'big'); - const head = git('rev-parse', 'HEAD').trim(); - - // The fixture does what it says: git's own output is past the buffer, and the cut lands VISIBLE characters into a token. - const plain = git(...diffArgs(head)); - assert.ok(plain.length > BUFFER, `the diff (${plain.length}) is past the buffer`); - const start = plain.lastIndexOf('ghp_', MAX); - assert.equal(MAX - start, VISIBLE, 'the cut is inside a token'); - assert.match(plain.slice(start, start + 40), /^ghp_[A-Za-z0-9]{36}$/); - - const d = await decisionDiff(repo, base, head); - assert.ok(d.diff !== null, JSON.stringify(d).slice(0, 300)); - assert.equal((d as { truncated: boolean }).truncated, true); - assert.ok(d.diff.length <= MAX); - assert.equal(d.diff.includes('ghp_'), false, `no part of any token is shown; the end: ${JSON.stringify(d.diff.slice(-80))}`); -}); diff --git a/test/decision-diff-utf16.test.ts b/test/decision-diff-utf16.test.ts deleted file mode 100644 index 48d06e971..000000000 --- a/test/decision-diff-utf16.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -// The owner's decision diff and a token in UTF-16 (#927 round 9). The diff is read with `--text` (#807 F2: a NUL, or a -// `-diff` attribute, is also how a change hides its lines from review as "binary"), so a binary file's bytes are shown. -// UTF-16 text puts a NUL between ASCII characters, and no redaction rule matched it: the token read whole once the NULs -// were dropped (measured by #927 round 8's second review), where main showed "Binary files differ". The diff is now -// redacted, and shown, as a NUL-free view. -import { test } from 'node:test'; -import assert from 'node:assert/strict'; -import { execFileSync } from 'node:child_process'; -import { mkdtempSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { decisionDiff } from '../src/decision-detail.js'; - -/** GitHub-token-shaped (ghp_ and 36 letters and digits), not a real one. */ -const TOKEN = `ghp_${'Utf16Tk9Lm3Qr7Ws2Xv5Zb8Nc4Hd6Jf1Gp0'.padEnd(36, 'a').slice(0, 36)}`; - -function utf16(text: string, order: 'le' | 'be'): Buffer { - const le = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, 'utf16le')]); - if (order === 'le') return le; - const be = Buffer.from(le); - be.swap16(); - return be; -} - -for (const order of ['le', 'be'] as const) { - test(`a token in a UTF-16${order.toUpperCase()} file is not shown in the owner's decision diff`, async () => { - assert.equal(TOKEN.length, 40); - const repo = mkdtempSync(join(tmpdir(), `decision-utf16-${order}-`)); - const git = (...a: string[]) => execFileSync('git', a, { cwd: repo, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); - git('init', '-q', '-b', 'main'); - for (const [k, v] of [['user.name', 'Ada'], ['user.email', 'ada@example.com'], ['commit.gpgsign', 'false']]) git('config', k!, v!); - writeFileSync(join(repo, 'a.txt'), 'a\n'); - git('add', '-A'); - git('commit', '-q', '-m', 'base'); - const base = git('rev-parse', 'HEAD').trim(); - writeFileSync(join(repo, 'notes.txt'), utf16(`notes\ntoken: ${TOKEN}\nend\n`, order)); - git('add', '-A'); - git('commit', '-q', '-m', 'notes'); - const head = git('rev-parse', 'HEAD').trim(); - // The file holds the token's characters with a NUL between them. - assert.ok(git('show', `${head}:notes.txt`).replace(/\0/g, '').includes(TOKEN)); - - const d = await decisionDiff(repo, base, head); - assert.ok(d.diff !== null, JSON.stringify(d)); - const seen = d.diff.replace(/\0/g, ''); - assert.equal(seen.includes('ghp_'), false, `no part of the token: ${JSON.stringify(seen.slice(0, 400))}`); - assert.equal(seen.includes(TOKEN.slice(4, 20)), false); - // The rest of the file still shows: the change isn't hidden as "binary". - assert.match(seen, /notes/); - }); -} From 35f873ec7e201d3e9ffd1a2a2cf4f82ca02ae969 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Thu, 22 Oct 2026 00:17:58 -0500 Subject: [PATCH 17/18] git runner: scope the replace-refs claim to git 2.42+, and patchText's comment to the hash (#927) Wording only, no logic change. The round-11 final check: the replace-refs claims hold on git 2.42 and later (measured on 2.43); before 2.42 a config `core.useReplaceRefs=true` may turn replace refs back on (inferred). The changelog fragment now says so. patchText's doc comment said its patch was for a person, a prompt or a hash; since the cut it is for the patch hash only, never shown (both reviewers). Signed-off-by: Jayden Vasquez --- changelog.d/927-git-runner.md | 10 ++++++---- src/coordinator/change-files.ts | 7 ++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/changelog.d/927-git-runner.md b/changelog.d/927-git-runner.md index 5aee0b10b..a7723986c 100644 --- a/changelog.d/927-git-runner.md +++ b/changelog.d/927-git-runner.md @@ -1,8 +1,9 @@ ### Upgrading (from v0.5.2) -- **Every git the engine's JS and TS code runs ignores replace refs and grafts (#927).** An agent could write - `refs/replace/*` or `.git/info/grafts` in the shared repo and make the push limits, the secret scan and the review - read other commits. The `.sh` helpers under `scripts/` don't go through the runner. A change's recorded +- **Every git the engine's JS and TS code runs ignores replace refs and grafts, on git 2.42 and later (#927).** An + agent could write `refs/replace/*` or `.git/info/grafts` in the shared repo and make the push limits, the secret + scan and the review read other commits. Measured on git 2.43. Before 2.42, a config `core.useReplaceRefs=true` may + turn replace refs back on (inferred, not run). The `.sh` helpers under `scripts/` don't go through the runner. A change's recorded `patch_hash` now covers every file's lines, so it differs from an older engine's for the same change. Run `npm run setup:hooks` once after upgrading: until then, an install's old copy of the push-gate hook runs its own git calls (the one that loads the gate among them) without these settings. It keeps working. @@ -13,7 +14,8 @@ ### Guardrail changes: what each one does - **The git runner (#927):** `src/os/git.ts` (and `scripts/lib/git.cjs` for scripts) starts every git with replace - refs off and no graft file, over whatever environment the caller passes; gitleaks gets the same settings. A test + refs off and no graft file, over whatever environment the caller passes; gitleaks gets the same settings. Replace + refs stay off whatever the config says on git 2.42 and later (measured on 2.43; before 2.42, see above). A test fails on a git call in the JS and TS files under `src/` and `scripts/` that doesn't come through it (it doesn't read `.sh` files, git named in a shell string, or a name built at run time), and the runner and that test wait for the owner. diff --git a/src/coordinator/change-files.ts b/src/coordinator/change-files.ts index 1d9315224..350dfeec3 100644 --- a/src/coordinator/change-files.ts +++ b/src/coordinator/change-files.ts @@ -233,9 +233,10 @@ export function diffStatText(cwd: string, rev: string): string { } /** - * `git diff ` as a patch, for a person, a prompt or a hash (#807): every file's lines whatever the attributes say - * (a `-diff` or `binary` mark, a textconv or an external diff), and every submodule link. `args` as changedNames takes - * them. Never parsed. + * `git diff ` as a patch, for a hash only, never shown (#807): every file's lines whatever the attributes say + * (a `-diff` or `binary` mark, a textconv or an external diff), and every submodule link, a binary file's bytes + * included. `args` as changedNames takes them. Never parsed, and never put on a page, in a prompt or in an issue: the + * owner cut `--text` from every diff a person reads (#927). */ export function patchText(cwd: string, args: string[]): string { return must(run(cwd, ['diff', '--no-color', ...AS_TEXT, ...nameArgs(args)])); From 2172591bd89887413b1d6801b55053d892b49241 Mon Sep 17 00:00:00 2001 From: Jayden Vasquez Date: Thu, 22 Oct 2026 01:07:27 -0500 Subject: [PATCH 18/18] merge policy: the git runner is a guard file, since it runs in a backstop's process (#927) CI at 802150f8 failed #746's desktop backstop in every shard: "src/os/git.ts runs in this backstop's process, so it must be a guard file" (test/desktop-removed.test.ts). After the merge with main, src/merge-policy-weakening.ts reads changed files through src/coordinator/change-files.ts, which now runs git through the runner, so a changed line of src/os/git.ts could change what a backstop reads. GUARD_FILES gains 'src/os/git.ts' (an addition only), next to src/os/child-process.ts, so such a line waits for the owner. Test (fails without): test/desktop-removed.test.ts's "every module this backstop runs ... is held", as CI ran it at 802150f8; it passes here. Signed-off-by: Jayden Vasquez --- src/merge-policy-weakening.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/merge-policy-weakening.ts b/src/merge-policy-weakening.ts index 3306038c4..49bea9065 100644 --- a/src/merge-policy-weakening.ts +++ b/src/merge-policy-weakening.ts @@ -213,6 +213,8 @@ export const GUARD_FILES: readonly string[] = [ 'src/os/system-paths.ts', 'src/os/own-env.ts', 'src/os/child-process.ts', + // The git runner (#927): src/coordinator/change-files.ts reads through it, so it runs in a backstop's process too. + 'src/os/git.ts', ]; /** A package manager's settings: every one changes what install and each script run under (`script-shell=`, `ignore-scripts=`). */