From cc6b173bfad6ab89078ebab360a99e5e61416935 Mon Sep 17 00:00:00 2001 From: Liam Steiner Date: Tue, 14 Jul 2026 17:47:19 +0300 Subject: [PATCH] feat(a2): prove blocking warden enforcement via wrapToolCall (LIA-395) Proves LangChain's real `createMiddleware({wrapToolCall})` contract can enforce the unmodified `codex_warden_hooks.py run plan-review-gate` script: a denied tool call never reaches its handler, an allowed call reaches it exactly once, and the model sees the gate's actual deny text as a ToolMessage on its next turn. All three proven deterministically via FakeToolCallingModel against the real gate script (13/13 tests) -- no live model credential required, unlike A1's split. Verdict: PASS (framework-mechanism); see scripts/spikes/lia395_warden_veto_walking_skeleton.md for full scope. Forks fresh off origin/main per the established per-slice convention (A1/PR #1031 is still open, so its langchain devDeps are re-added here independently -- trivial to reconcile on rebase if A1 lands first). Co-Authored-By: Claude Sonnet 5 --- package-lock.json | 462 +++++++++++++++--- package.json | 3 + .../lia395_warden_veto_walking_skeleton.md | 155 ++++++ ...ia395_warden_veto_walking_skeleton.test.ts | 357 ++++++++++++++ .../lia395_warden_veto_walking_skeleton.ts | 404 +++++++++++++++ vitest.config.ts | 6 +- 6 files changed, 1308 insertions(+), 79 deletions(-) create mode 100644 scripts/spikes/lia395_warden_veto_walking_skeleton.md create mode 100644 scripts/spikes/lia395_warden_veto_walking_skeleton.test.ts create mode 100644 scripts/spikes/lia395_warden_veto_walking_skeleton.ts diff --git a/package-lock.json b/package-lock.json index 7e6eccba..ac1820a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,8 @@ "@commitlint/cli": "^21.1.0", "@commitlint/config-conventional": "^21.2.0", "@eslint/js": "^10.0.1", + "@langchain/anthropic": "^1.5.1", + "@langchain/core": "^1.2.2", "@types/better-sqlite3": "^7.6.12", "@types/node": "^25.9.3", "@vitest/coverage-v8": "^4.1.10", @@ -34,6 +36,7 @@ "eslint-config-prettier": "^10.1.8", "fast-check": "^4.8.0", "husky": "^9.1.7", + "langchain": "^1.5.3", "lint-staged": "^17.0.8", "playwright": "^1.61.1", "prettier": "^3.9.4", @@ -48,6 +51,28 @@ "node": ">=20" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.103.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.103.0.tgz", + "integrity": "sha512-1uG7RNgoHTUxzOXqSCODKt0UTVlxWiHk/2Tt2/uQJiPW7XzBeKVuJyd3Aw6T3LPyvZV/jDTnPLX7SaM70WLLjA==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -106,6 +131,16 @@ "node": ">=6.0.0" } }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/types": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", @@ -180,6 +215,13 @@ "keyv": "^5.6.0" } }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "devOptional": true, + "license": "MIT" + }, "node_modules/@clack/core": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@clack/core/-/core-1.2.0.tgz", @@ -1467,9 +1509,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1486,9 +1525,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1505,9 +1541,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1524,9 +1557,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1543,9 +1573,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1562,9 +1589,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1581,9 +1605,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1600,9 +1621,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1619,9 +1637,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1644,9 +1659,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1669,9 +1681,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1694,9 +1703,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1719,9 +1725,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1744,9 +1747,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1769,9 +1769,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1794,9 +1791,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -1956,6 +1950,153 @@ "dev": true, "license": "MIT" }, + "node_modules/@langchain/anthropic": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.5.1.tgz", + "integrity": "sha512-j92zCCd5BFH3rHMRzc2wBmSKDoVpinof1oh8aFiAz9TWbSOc4tGU4n6bqwy/wP0GH1uO96zZHLGCHBMPgrxTNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@anthropic-ai/sdk": "^0.103.0", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.2.1" + } + }, + "node_modules/@langchain/core": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.2.tgz", + "integrity": "sha512-KfjEOT6sCg0vvItagfEtGpmrGoLMGfma4Affb5BGEqPmS2YR3AxW54pABSkhQlzCehTB+0BnLquAe1lGF4J9zQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "@standard-schema/spec": "^1.1.0", + "js-tiktoken": "^1.0.12", + "langsmith": ">=0.5.0 <1.0.0", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@langchain/core/node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@langchain/core/node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/core/node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@langchain/langgraph": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.7.tgz", + "integrity": "sha512-2tcyf3QGC7v89kqSxMCtRvzg/3L/4yHtOaWC49A8KieCciWJs7LGaxHoPB6QRxXyUgyR+Zg9Q1ss/XJIE+JuSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@langchain/langgraph-checkpoint": "^1.1.3", + "@langchain/langgraph-sdk": "~1.9.25", + "@langchain/protocol": "^0.0.18", + "@standard-schema/spec": "1.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "zod": "^3.25.32 || ^4.2.0" + } + }, + "node_modules/@langchain/langgraph-checkpoint": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.3.tgz", + "integrity": "sha512-wgzdQNeEsdw1e+4lvlj0tdq/RYR/k1vPin10g0ymGoehZDDgd9nvIllGXSXN4TFgF9sf5qQP/KTkOcLfeseIhA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48" + } + }, + "node_modules/@langchain/langgraph-sdk": { + "version": "1.9.25", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.9.25.tgz", + "integrity": "sha512-mRKW8zyQUaHox+HirRFMRrPqOvNbQI3xeXDt6kkk4PbBg77V92bsO1WzUVNrmJ81zCkvxyOrWSK8D6ioCj0a8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@langchain/protocol": "^0.0.18", + "@types/json-schema": "^7.0.15", + "p-queue": "^9.0.1", + "p-retry": "^7.1.1" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "react": "^18 || ^19", + "react-dom": "^18 || ^19", + "svelte": "^4.0.0 || ^5.0.0", + "vue": "^3.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "svelte": { + "optional": true + }, + "vue": { + "optional": true + } + } + }, + "node_modules/@langchain/protocol": { + "version": "0.0.18", + "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.18.tgz", + "integrity": "sha512-XW1egQtPfsGI41w2AMZNFZrUIwFSQHTjVMZs0OaTpCAvht/QLoaPN8FQcsysMVypOhupG28J29yOorrc70otBQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@linear/sdk": { "version": "88.0.0", "resolved": "https://registry.npmjs.org/@linear/sdk/-/sdk-88.0.0.tgz", @@ -2210,9 +2351,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2230,9 +2368,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2250,9 +2385,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2270,9 +2402,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2290,9 +2419,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2310,9 +2436,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2428,6 +2551,13 @@ "url": "https://ko-fi.com/dangreen" } }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@standard-schema/spec": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", @@ -4510,6 +4640,13 @@ "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", "license": "MIT" }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "dev": true, + "license": "Unlicense" + }, "node_modules/fast-string-truncated-width": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-1.2.1.tgz", @@ -5251,6 +5388,19 @@ "node": ">=0.10.0" } }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-obj": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", @@ -5353,6 +5503,16 @@ "node": ">=10" } }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "dev": true, + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", @@ -5406,6 +5566,20 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -5456,6 +5630,96 @@ "@keyv/serialize": "^1.1.1" } }, + "node_modules/langchain": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.3.tgz", + "integrity": "sha512-YRYPy1xPq4CgKe0NdPoscYHnKBW5cViEyWiZsUKQfN3xL1X5DpGlIpaBQg1A/t6o9ePKE7J5b7nqrfLz8t7/Rw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@langchain/langgraph": "^1.4.7", + "@langchain/langgraph-checkpoint": "^1.1.3", + "langsmith": ">=0.5.0 <1.0.0", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.2.1" + } + }, + "node_modules/langsmith": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.8.2.tgz", + "integrity": "sha512-WX85yuL9vHAFIXJ4Nde/2VsjZT5o6FzX617uKDGtcgMojPe+IM6lgiZeVagv/egC79bQS9HPKVD/UPe/6zGFPg==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, + "node_modules/langsmith/node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, + "node_modules/langsmith/node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/langsmith/node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -5624,9 +5888,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -5648,9 +5909,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -5672,9 +5930,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -5696,9 +5951,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6243,6 +6495,16 @@ "node": ">=18" } }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "dev": true, + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/nanoid": { "version": "3.3.15", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", @@ -6442,6 +6704,16 @@ "node": ">= 0.8.0" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/p-queue": { "version": "9.1.1", "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.1.1.tgz", @@ -6459,6 +6731,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-retry": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", + "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-network-error": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-timeout": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", @@ -7504,6 +7792,17 @@ "dev": true, "license": "MIT" }, + "node_modules/standardwebhooks": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", + "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, "node_modules/statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -7784,6 +8083,13 @@ "node": ">=14.17" } }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "dev": true, + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", diff --git a/package.json b/package.json index 14d55177..30389910 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,8 @@ "@commitlint/cli": "^21.1.0", "@commitlint/config-conventional": "^21.2.0", "@eslint/js": "^10.0.1", + "@langchain/anthropic": "^1.5.1", + "@langchain/core": "^1.2.2", "@types/better-sqlite3": "^7.6.12", "@types/node": "^25.9.3", "@vitest/coverage-v8": "^4.1.10", @@ -67,6 +69,7 @@ "eslint-config-prettier": "^10.1.8", "fast-check": "^4.8.0", "husky": "^9.1.7", + "langchain": "^1.5.3", "lint-staged": "^17.0.8", "playwright": "^1.61.1", "prettier": "^3.9.4", diff --git a/scripts/spikes/lia395_warden_veto_walking_skeleton.md b/scripts/spikes/lia395_warden_veto_walking_skeleton.md new file mode 100644 index 00000000..6cb421af --- /dev/null +++ b/scripts/spikes/lia395_warden_veto_walking_skeleton.md @@ -0,0 +1,155 @@ +# Spike: warden veto through LangChain's `wrapToolCall` middleware (LIA-395 / A2) + +**Date:** 2026-07-14 · **Verdict: PASS (framework-mechanism)** — blocking +enforcement, allow-once semantics, and model-visible feedback all proven +deterministically against the real, unmodified `codex_warden_hooks.py` gate +script. No piece of this proof needed a live model credential. Real-provider +serialization stays unverified — see Scope below. + +## Question + +Can LangChain JS's real `wrapToolCall` middleware contract — not a Decorator +around `.invoke()`, the actual `createMiddleware({ wrapToolCall })` hook +`createAgent` calls into on every tool invocation — enforce an **unmodified** +Deus warden gate (`scripts/codex_warden_hooks.py run plan-review-gate`)? This +is A2 of the Linear-tracked MA milestone (LIA-394..400), continuing from A1 +(LIA-394, PR #1031, still open/unmerged). Three things must hold: + +1. A denied tool call never reaches the wrapped tool's real execute function. +2. An allowed tool call reaches it exactly once. +3. The model sees the gate's actual denial text as a `ToolMessage`, and the + agent loop continues using that feedback. + +## Method + +### Reuse target — unmodified + +`scripts/codex_warden_hooks.py run plan-review-gate --repo-root ` is +the exact mechanism that gated every `Edit`/`Write` in the session that +produced this spike. Nothing in this file modifies it; `invokeWardenGate` +below only spawns it as a subprocess and interprets its stdout/exit code. + +### `invokeWardenGate` — subprocess wrapper, fail-closed for subprocess/protocol failures + +Spawns ` run --repo-root ` +(both `PYTHON_BIN` and the absolute `WARDEN_SCRIPT` path are resolved via +`src/platform.ts` and `import.meta.url` respectively — no hardcoded +`python3`, no relative script path that would break once a scratch repo's +`cwd` differs from this file's own location). Writes the constructed hook +event as JSON to stdin, waits for the child to `close`, and interprets: + +- Nonzero exit, spawn failure, a signal-terminated child, or a timeout → a + **hard error** (throw), never treated as allow. +- Exit 0 with empty/whitespace stdout → `{ decision: 'allow' }`. +- Exit 0 with non-empty stdout that parses as the gate's real deny JSON + shape (`hookSpecificOutput.hookEventName === 'PreToolUse'`, + `permissionDecision === 'deny'`, a non-empty string + `permissionDecisionReason`) → `{ decision: 'deny', reason }`. +- Any other shape (malformed JSON, missing fields, empty reason) → a **hard + error**, never a silent allow. + +**Honest scope limitation, not a bug in this wrapper:** the *unmodified* gate +script itself has pre-existing fail-open paths — malformed stdin silently +becomes `{}`, and a failed `git` invocation returns `None` and is +subsequently treated permissively. Both produce a normal exit-0-empty-stdout +"allow" indistinguishable from a legitimate one at the subprocess boundary. +This wrapper cannot and does not paper over that; fixing it would be new +scope against a script this spike's whole premise is to reuse unmodified. + +### `createWardenGateMiddleware` — real `wrapToolCall`, not a Decorator + +```ts +createMiddleware({ + name: 'WardenGate', + wrapToolCall: async (request, handler) => { + const toolCallId = request.toolCall.id; + if (!toolCallId?.trim()) throw new Error('Warden-denied tool call has no tool-call ID'); + const decision = await invokeGate(gateName, toEvent(request), repoRoot); + if (decision.decision === 'deny') { + return new ToolMessage({ content: decision.reason, tool_call_id: toolCallId, + name: request.toolCall.name, status: 'error' }); + } + return handler(request); + }, +}) +``` + +This is passed to `createAgent({ middleware: [...] })` — the same contract +`createAgent` uses for every tool call, not a wrapper applied around one +specific tool instance. `invokeGate` is an injectable seam (default: the +real `invokeWardenGate`) so the "gate is never called" assertion on the +missing-ID path uses a real mock, not a `vi.spyOn` against a module-local +binding that Vitest's ESM transform doesn't reliably intercept. + +### Synthetic tool — no path argument, no redirection surface + +`makeScratchEditTool(scratchFilePath)` has schema `{ content: string }` +only — closes over the write target at construction time. Empirically +confirmed (not assumed): LangChain's own schema validation +(`additionalProperties: false`) rejects a call carrying a forged extra +`path`/`file_path` property *before* it ever reaches the execute function — +a stronger guarantee than "the code happens not to read it." Scratch repo is +a throwaway `git init`'d `mkdtemp` directory, torn down in every test's +`afterEach`; never the real project worktree. + +## What was built + +- `scripts/spikes/lia395_warden_veto_walking_skeleton.ts` — `invokeWardenGate`, + `createWardenGateMiddleware`, `makeScratchEditTool`, scratch-repo helpers, + and a `main()` demo (blocked path + allowed path, printed transcript). +- `scripts/spikes/lia395_warden_veto_walking_skeleton.test.ts` — 13 tests, all + passing on first real run against the actual gate script: + - Real-gate blocked/allowed paths (subprocess, no mocking of the gate). + - 6 subprocess/protocol fail-closed cases: nonzero exit, spawn failure, + signal termination, timeout, malformed stdout, deny-with-empty-reason — + each via an injected `spawnFn` seam, since the real gate script can't be + made to time out or die by signal on demand. + - Middleware blocked/allowed paths through a real `createAgent` + + `FakeToolCallingModel` loop (execute-spy call counts, deny/allow content). + - The fixed-path negative test (forged extra argument rejected by schema). + - Missing/empty tool-call-ID ordering (gate mock + handler both zero calls). + - Model-visible-feedback: a spy on `FakeToolCallingModel.prototype._generate` + proves the model's **second invocation** actually received the denied + `ToolMessage` (matching content, `tool_call_id`, `status: 'error'`) — not + just that the agent's final state contains one. +- `package.json`/`vitest.config.ts`/`package-lock.json` — same + `langchain`/`@langchain/core`/`@langchain/anthropic` devDependencies and + spike-test-discovery glob A1 added, independently, since `origin/main` + doesn't yet have A1's unmerged additions and A2 forks fresh off + `origin/main` per the established per-slice convention (no branch stacking). + +## Verdict + +All three kill-switch criterion-1 requirements are proven deterministically, +with zero dependency on a live model credential: + +1. **Blocking**: the wrapped tool's execute function is asserted **never + called** on a denied call (via both a direct spy and, separately, the + `FakeToolCallingModel` prototype spy) — proven against the real, + unmodified gate script, not a stub. +2. **Allow-once**: the wrapped tool's execute function is asserted **called + exactly once** on an allowed call, and the scratch file's actual written + content is checked, not just a call count. +3. **Model-visible feedback**: the fake model's second invocation is proven + to have actually received the denied `ToolMessage` — the full + `createAgent → wrapToolCall → ToolMessage → next model call` transport, + not merely that the final agent state happens to contain one. + +**Scope of this PASS:** it proves LangChain's own middleware transport +mechanism deterministically. It does **not** prove real-provider +serialization or how an actual Anthropic model reacts to the feedback text — +`ANTHROPIC_API_KEY` remains genuinely absent in this environment (same as +A1), so the optional live smoke test did not run. Nothing in this spike's +core claim depends on that smoke test, unlike A1's split — but a future +session with real credentials (or the subscription-billing route documented +in `Research/2026-07-14-deus-v2-subscription-billing-credential-proxy.md`) +should still run it before treating real-provider behavior as verified. + +## Scope of this spike + +Read-only proof-of-concept; nothing wired into production. No modifications +to `scripts/codex_warden_hooks.py` or any other reused mechanism. No +`.plan-reviewed` marker or `.warden-verdicts.json` was ever written into the +real `deus-v2` worktree by any test — confirmed via marker-mtime comparison +before/after the test run. Not merged to `main` — pushed as a PR for review +only. diff --git a/scripts/spikes/lia395_warden_veto_walking_skeleton.test.ts b/scripts/spikes/lia395_warden_veto_walking_skeleton.test.ts new file mode 100644 index 00000000..21b28204 --- /dev/null +++ b/scripts/spikes/lia395_warden_veto_walking_skeleton.test.ts @@ -0,0 +1,357 @@ +import { EventEmitter } from 'node:events'; +import fs from 'node:fs'; + +import { createAgent, FakeToolCallingModel } from 'langchain'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { + createWardenGateMiddleware, + invokeWardenGate, + makeScratchEditTool, + markScratchRepoShip, + setupScratchRepo, + teardownScratchRepo, + type WardenGateDecision, +} from './lia395_warden_veto_walking_skeleton.js'; + +/** Minimal fake ChildProcess for invokeWardenGate's protocol-fault tests. */ +function fakeChild() { + const child = new EventEmitter() as EventEmitter & { + stdin: EventEmitter & { write: (v: unknown) => boolean; end: () => void }; + stdout: EventEmitter; + stderr: EventEmitter; + kill: () => void; + }; + child.stdin = Object.assign(new EventEmitter(), { + write: () => true, + end: () => {}, + }); + child.stdout = new EventEmitter(); + child.stderr = new EventEmitter(); + child.kill = vi.fn(); + return child; +} + +const scratchRepos: string[] = []; +function newScratchRepo() { + const repo = setupScratchRepo(); + scratchRepos.push(repo.scratchRepoRoot); + return repo; +} + +afterEach(() => { + while (scratchRepos.length) { + const dir = scratchRepos.pop(); + if (dir) teardownScratchRepo(dir); + } +}); + +describe('invokeWardenGate — real gate, scratch repo', () => { + it('denies with the real gate reason when no marker is present', async () => { + const { scratchRepoRoot, scratchFilePath } = newScratchRepo(); + const decision = await invokeWardenGate( + 'plan-review-gate', + { + tool_name: 'Edit', + tool_input: { file_path: scratchFilePath }, + cwd: scratchRepoRoot, + }, + scratchRepoRoot, + ); + expect(decision.decision).toBe('deny'); + expect((decision as { reason: string }).reason.length).toBeGreaterThan(0); + }); + + it('allows after the scratch repo is marked SHIP via the real mark subcommand', async () => { + const { scratchRepoRoot, scratchFilePath } = newScratchRepo(); + expect(() => markScratchRepoShip(scratchRepoRoot)).not.toThrow(); + const decision = await invokeWardenGate( + 'plan-review-gate', + { + tool_name: 'Edit', + tool_input: { file_path: scratchFilePath }, + cwd: scratchRepoRoot, + }, + scratchRepoRoot, + ); + expect(decision).toEqual({ decision: 'allow' }); + }); +}); + +describe('invokeWardenGate — subprocess/protocol fail-closed', () => { + it('hard-errors on nonzero exit with empty stdout', async () => { + const spawnFn = vi.fn(() => { + const child = fakeChild(); + queueMicrotask(() => child.emit('close', 1, null)); + return child as never; + }); + await expect( + invokeWardenGate('plan-review-gate', {}, '/tmp/whatever', { spawnFn }), + ).rejects.toThrow(/exited with code 1/); + }); + + it('hard-errors on spawn failure', async () => { + const spawnFn = vi.fn(() => { + const child = fakeChild(); + queueMicrotask(() => child.emit('error', new Error('ENOENT'))); + return child as never; + }); + await expect( + invokeWardenGate('plan-review-gate', {}, '/tmp/whatever', { spawnFn }), + ).rejects.toThrow(/failed to spawn/); + }); + + it('hard-errors on a signal-terminated child (distinct from nonzero exit)', async () => { + const spawnFn = vi.fn(() => { + const child = fakeChild(); + queueMicrotask(() => child.emit('close', null, 'SIGKILL')); + return child as never; + }); + await expect( + invokeWardenGate('plan-review-gate', {}, '/tmp/whatever', { spawnFn }), + ).rejects.toThrow(/killed by signal SIGKILL/); + }); + + it('hard-errors on timeout instead of waiting the production duration', async () => { + const spawnFn = vi.fn(() => fakeChild() as never); // never emits close/error + await expect( + invokeWardenGate('plan-review-gate', {}, '/tmp/whatever', { + spawnFn, + timeoutMs: 50, + }), + ).rejects.toThrow(/timed out after 50ms/); + }); + + it('hard-errors on malformed (non-JSON) stdout', async () => { + const spawnFn = vi.fn(() => { + const child = fakeChild(); + queueMicrotask(() => { + child.stdout.emit('data', Buffer.from('not json')); + child.emit('close', 0, null); + }); + return child as never; + }); + await expect( + invokeWardenGate('plan-review-gate', {}, '/tmp/whatever', { spawnFn }), + ).rejects.toThrow(/non-empty, non-JSON stdout/); + }); + + it('hard-errors on a deny shape with a missing/empty reason (never allow, never empty-reason deny)', async () => { + const spawnFn = vi.fn(() => { + const child = fakeChild(); + queueMicrotask(() => { + child.stdout.emit( + 'data', + Buffer.from( + JSON.stringify({ + hookSpecificOutput: { + hookEventName: 'PreToolUse', + permissionDecision: 'deny', + permissionDecisionReason: '', + }, + }), + ), + ); + child.emit('close', 0, null); + }); + return child as never; + }); + await expect( + invokeWardenGate('plan-review-gate', {}, '/tmp/whatever', { spawnFn }), + ).rejects.toThrow(/unrecognized stdout shape/); + }); +}); + +describe('createWardenGateMiddleware — blocked/allowed paths, real gate', () => { + it('blocked path: wrapped tool execute never called, denied ToolMessage carries the real reason', async () => { + const { scratchRepoRoot, scratchFilePath } = newScratchRepo(); + const wrapped = makeScratchEditTool(scratchFilePath); + const executeSpy = vi.spyOn(wrapped, 'invoke'); + + const middleware = createWardenGateMiddleware( + 'plan-review-gate', + () => ({ + tool_name: 'Edit', + tool_input: { file_path: scratchFilePath }, + cwd: scratchRepoRoot, + }), + scratchRepoRoot, + ); + + const agent = createAgent({ + model: new FakeToolCallingModel({ + toolCalls: [ + [{ name: 'edit_scratch', args: { content: 'hello' }, id: 'call_1' }], + [], + ], + }), + tools: [wrapped], + middleware: [middleware], + }); + + const result = await agent.invoke({ + messages: [{ role: 'user', content: 'write hello to the scratch file' }], + }); + + expect(executeSpy).not.toHaveBeenCalled(); + expect(fs.existsSync(scratchFilePath)).toBe(false); + const denyMessage = result.messages.find( + (m) => (m as { name?: string }).name === 'edit_scratch', + ) as { content?: unknown; tool_call_id?: string } | undefined; + expect(denyMessage).toBeDefined(); + expect(String(denyMessage?.content).length).toBeGreaterThan(0); + expect(denyMessage?.tool_call_id).toBe('call_1'); + }); + + it('allowed path: wrapped tool execute called exactly once, file written', async () => { + const { scratchRepoRoot, scratchFilePath } = newScratchRepo(); + markScratchRepoShip(scratchRepoRoot); + const wrapped = makeScratchEditTool(scratchFilePath); + const executeSpy = vi.spyOn(wrapped, 'invoke'); + + const middleware = createWardenGateMiddleware( + 'plan-review-gate', + () => ({ + tool_name: 'Edit', + tool_input: { file_path: scratchFilePath }, + cwd: scratchRepoRoot, + }), + scratchRepoRoot, + ); + + const agent = createAgent({ + model: new FakeToolCallingModel({ + toolCalls: [ + [{ name: 'edit_scratch', args: { content: 'hello' }, id: 'call_1' }], + [], + ], + }), + tools: [wrapped], + middleware: [middleware], + }); + + await agent.invoke({ + messages: [{ role: 'user', content: 'write hello to the scratch file' }], + }); + + expect(executeSpy).toHaveBeenCalledTimes(1); + expect(fs.readFileSync(scratchFilePath, 'utf8')).toBe('hello'); + }); + + it('negative test: a forged extra path argument has zero effect on the write target', async () => { + const { scratchRepoRoot, scratchFilePath } = newScratchRepo(); + markScratchRepoShip(scratchRepoRoot); + const forgedPath = `${scratchRepoRoot}/forged.txt`; + const wrapped = makeScratchEditTool(scratchFilePath); + + // The tool's `additionalProperties: false` schema means LangChain's own + // validation rejects a forged extra path/file_path property BEFORE the + // call ever reaches our execute function — an even stronger proof of + // "no argument surface for redirection" than merely ignoring it would + // have been. Confirmed empirically (not assumed): this throws. + await expect( + wrapped.invoke({ + content: 'hello', + path: forgedPath, + file_path: forgedPath, + } as never), + ).rejects.toThrow(); + + expect(fs.existsSync(forgedPath)).toBe(false); + expect(fs.existsSync(scratchFilePath)).toBe(false); + }); + + it('missing/empty tool-call ID rejects before the gate is ever invoked', async () => { + const { scratchRepoRoot, scratchFilePath } = newScratchRepo(); + const wrapped = makeScratchEditTool(scratchFilePath); + const executeSpy = vi.spyOn(wrapped, 'invoke'); + const invokeGate = vi.fn(); + + const middleware = createWardenGateMiddleware( + 'plan-review-gate', + () => ({ + tool_name: 'Edit', + tool_input: { file_path: scratchFilePath }, + cwd: scratchRepoRoot, + }), + scratchRepoRoot, + invokeGate, + ); + const handler = vi.fn(); + const wrapToolCall = ( + middleware as unknown as { + wrapToolCall: (req: unknown, h: unknown) => Promise; + } + ).wrapToolCall; + + await expect( + wrapToolCall( + { toolCall: { name: 'edit_scratch', args: {}, id: '' } }, + handler, + ), + ).rejects.toThrow(/no tool-call ID/); + + expect(invokeGate).not.toHaveBeenCalled(); + expect(executeSpy).not.toHaveBeenCalled(); + expect(handler).not.toHaveBeenCalled(); + }); +}); + +describe('model-visible-feedback — FakeToolCallingModel, deterministic', () => { + it('the model actually receives the denied ToolMessage on its second invocation', async () => { + const { scratchRepoRoot, scratchFilePath } = newScratchRepo(); + const wrapped = makeScratchEditTool(scratchFilePath); + const executeSpy = vi.spyOn(wrapped, 'invoke'); + const generateSpy = vi.spyOn( + FakeToolCallingModel.prototype, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + '_generate' as any, + ); + + const middleware = createWardenGateMiddleware( + 'plan-review-gate', + () => ({ + tool_name: 'Edit', + tool_input: { file_path: scratchFilePath }, + cwd: scratchRepoRoot, + }), + scratchRepoRoot, + ); + + const agent = createAgent({ + model: new FakeToolCallingModel({ + toolCalls: [ + [{ name: 'edit_scratch', args: { content: 'hello' }, id: 'call_1' }], + [], + ], + }), + tools: [wrapped], + middleware: [middleware], + }); + + const result = await agent.invoke({ + messages: [{ role: 'user', content: 'write hello to the scratch file' }], + }); + + expect(executeSpy).not.toHaveBeenCalled(); + expect(generateSpy.mock.calls.length).toBeGreaterThanOrEqual(2); + const secondInvocationMessages = generateSpy.mock.calls[1]?.[0] as Array<{ + name?: string; + content?: unknown; + tool_call_id?: string; + }>; + const forwardedDenyMessage = secondInvocationMessages.find( + (m) => m.name === 'edit_scratch', + ); + expect(forwardedDenyMessage).toBeDefined(); + expect(forwardedDenyMessage?.tool_call_id).toBe('call_1'); + expect(String(forwardedDenyMessage?.content).length).toBeGreaterThan(0); + + const finalDenyMessage = result.messages.find( + (m) => (m as { name?: string }).name === 'edit_scratch', + ) as { content?: unknown } | undefined; + expect(finalDenyMessage?.content).toBe(forwardedDenyMessage?.content); + + generateSpy.mockRestore(); + }); +}); diff --git a/scripts/spikes/lia395_warden_veto_walking_skeleton.ts b/scripts/spikes/lia395_warden_veto_walking_skeleton.ts new file mode 100644 index 00000000..e25e0cd3 --- /dev/null +++ b/scripts/spikes/lia395_warden_veto_walking_skeleton.ts @@ -0,0 +1,404 @@ +/** + * Spike (LIA-395 / A2): can LangChain JS's real `wrapToolCall` middleware + * contract enforce an UNMODIFIED Deus warden gate (`scripts/codex_warden_hooks.py + * run plan-review-gate`) — a denied call never reaches its handler, an allowed + * call reaches it exactly once, and the model sees the gate's actual feedback + * as a `ToolMessage`? + * + * This is a host-side-only spike, same convention as A1 + * (lia394_langchain_walking_skeleton.ts). Unlike A1, none of A2's core proof + * needs a live model call — `invokeWardenGate`/`createWardenGateMiddleware` + * are proven against the real (unmodified) gate script and a synthetic tool + * in a throwaway scratch git repo, and the model-visible-feedback transport + * is proven deterministically via LangChain's `FakeToolCallingModel`. See the + * paired write-up (lia395_warden_veto_walking_skeleton.md) for the full + * question/method/verdict. + * + * Cross-platform: PYTHON_BIN resolved via src/platform.ts, no shell-outs + * beyond spawning the Python gate script itself. + */ + +import { execFileSync } from 'node:child_process'; +import { spawn, type ChildProcess } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { ToolMessage } from '@langchain/core/messages'; +import { tool, type StructuredTool } from '@langchain/core/tools'; +import { + createAgent, + createMiddleware, + FakeToolCallingModel, + type AgentMiddleware, + type ToolCallRequest, +} from 'langchain'; + +import { PYTHON_BIN } from '../../src/platform.js'; + +const WARDEN_SCRIPT = fileURLToPath( + new URL('../../scripts/codex_warden_hooks.py', import.meta.url), +); +const DEFAULT_TIMEOUT_MS = 15_000; + +// ── invokeWardenGate ──────────────────────────────────────────────────────── + +export interface WardenGateOptions { + /** Injectable seam for tests — defaults to the real node:child_process.spawn. */ + spawnFn?: typeof spawn; + /** Bounded wait before treating the gate subprocess as hung. */ + timeoutMs?: number; +} + +export type WardenGateDecision = + { decision: 'allow' } | { decision: 'deny'; reason: string }; + +/** + * Fail-closed FOR SUBPROCESS/PROTOCOL FAILURES ONLY. The unmodified gate + * script retains its own pre-existing fail-open paths (malformed stdin + * becomes `{}`, failed git invocations return `None` and are subsequently + * treated as permissive) — this wrapper cannot and does not paper over + * those; they are a residual limitation of the reused, unmodified gate + * script, recorded in the write-up rather than "fixed" here (fixing them + * would be new scope requiring its own review, not part of proving + * wrapToolCall). + */ +export async function invokeWardenGate( + gateName: string, + event: Record, + repoRoot: string, + opts: WardenGateOptions = {}, +): Promise { + const spawnFn = opts.spawnFn ?? spawn; + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + + return new Promise((resolve, reject) => { + const child: ChildProcess = spawnFn(PYTHON_BIN, [ + WARDEN_SCRIPT, + 'run', + gateName, + '--repo-root', + repoRoot, + ]); + + let settled = false; + let stdout = ''; + let stderr = ''; + let timer: ReturnType | undefined; + + const settle = (fn: () => void): void => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + fn(); + }; + + timer = setTimeout(() => { + settle(() => { + child.kill(); + reject( + new Error( + `invokeWardenGate: gate "${gateName}" timed out after ${timeoutMs}ms`, + ), + ); + }); + }, timeoutMs); + + // Broken-pipe/EPIPE on stdin surfaces via the eventual close/error event + // below — swallow here only to avoid an unhandled 'error' crash on the + // stream itself. + child.stdin?.on('error', () => {}); + + child.stdout?.on('data', (chunk: Buffer) => { + stdout += chunk.toString('utf8'); + }); + child.stderr?.on('data', (chunk: Buffer) => { + stderr += chunk.toString('utf8'); + }); + + child.on('error', (err: Error) => { + settle(() => { + reject( + new Error( + `invokeWardenGate: failed to spawn gate "${gateName}": ${err.message}`, + ), + ); + }); + }); + + child.on('close', (code: number | null, signal: NodeJS.Signals | null) => { + settle(() => { + if (signal) { + reject( + new Error( + `invokeWardenGate: gate "${gateName}" was killed by signal ${signal}`, + ), + ); + return; + } + if (code !== 0) { + reject( + new Error( + `invokeWardenGate: gate "${gateName}" exited with code ${code}. stderr: ${stderr}`, + ), + ); + return; + } + const trimmed = stdout.trim(); + if (trimmed === '') { + resolve({ decision: 'allow' }); + return; + } + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + reject( + new Error( + `invokeWardenGate: gate "${gateName}" produced non-empty, non-JSON stdout: ${trimmed}`, + ), + ); + return; + } + const hookOutput = (parsed as Record | null) + ?.hookSpecificOutput as Record | undefined; + const reason = hookOutput?.permissionDecisionReason; + const isValidDeny = + hookOutput?.hookEventName === 'PreToolUse' && + hookOutput?.permissionDecision === 'deny' && + typeof reason === 'string' && + reason.trim() !== ''; + if (!isValidDeny) { + reject( + new Error( + `invokeWardenGate: gate "${gateName}" produced an unrecognized stdout shape: ${trimmed}`, + ), + ); + return; + } + resolve({ decision: 'deny', reason: reason as string }); + }); + }); + + child.stdin?.write(JSON.stringify(event)); + child.stdin?.end(); + }); +} + +// ── createWardenGateMiddleware ────────────────────────────────────────────── + +/** + * Real LangChain middleware via createMiddleware's wrapToolCall hook — NOT a + * Decorator around .invoke() (that would prove a different mechanism than + * the one LIA-395 names). `invokeGate` is an injectable seam defaulting to + * the real invokeWardenGate: wrapToolCall closes over the module-local + * binding directly, and a spy on the export would not reliably intercept + * that internal call — tests that need to assert "the gate was never + * called" inject a mock here instead of spying. + */ +export function createWardenGateMiddleware( + gateName: string, + toEvent: (request: ToolCallRequest) => Record, + repoRoot: string, + invokeGate: typeof invokeWardenGate = invokeWardenGate, +): AgentMiddleware { + return createMiddleware({ + name: 'WardenGate', + wrapToolCall: async (request, handler) => { + // Checked before any subprocess is spawned: ToolCallRequest.toolCall.id + // is string | undefined, but ToolMessage.tool_call_id requires string. + // Synthesizing an empty id with `?? ''` would compile but silently + // break the model-visible-feedback transport this spike exists to + // prove. + const toolCallId = request.toolCall.id; + if (!toolCallId?.trim()) { + throw new Error('Warden-denied tool call has no tool-call ID'); + } + const decision = await invokeGate(gateName, toEvent(request), repoRoot); + if (decision.decision === 'deny') { + return new ToolMessage({ + content: decision.reason, + tool_call_id: toolCallId, + name: request.toolCall.name, + status: 'error', + }); + } + return handler(request); + }, + }); +} + +// ── Synthetic scratch-repo test tool ──────────────────────────────────────── + +/** + * Schema is `{ content: string }` only — deliberately NO path argument. + * Closes over a fixed, server-chosen scratchFilePath at construction time, + * so there is no argument surface for a model (real or fake) to redirect + * the write anywhere else. A forged extra `path`/`file_path` property in the + * call args is simply never read. + */ +export function makeScratchEditTool(scratchFilePath: string): StructuredTool { + return tool( + async (args: Record) => { + const content = typeof args.content === 'string' ? args.content : ''; + fs.writeFileSync(scratchFilePath, content, 'utf8'); + return JSON.stringify({ ok: true, path: scratchFilePath }); + }, + { + name: 'edit_scratch', + description: + 'Writes content to a fixed scratch file for the A2 warden-veto spike.', + schema: { + type: 'object', + properties: { content: { type: 'string' } }, + required: ['content'], + additionalProperties: false, + }, + }, + ); +} + +// ── Scratch repo helpers ──────────────────────────────────────────────────── + +export function setupScratchRepo(): { + scratchRepoRoot: string; + scratchFilePath: string; +} { + const scratchRepoRoot = fs.mkdtempSync( + path.join(os.tmpdir(), 'lia395-scratch-'), + ); + execFileSync('git', ['init', '--quiet'], { cwd: scratchRepoRoot }); + return { + scratchRepoRoot, + scratchFilePath: path.join(scratchRepoRoot, 'scratch.txt'), + }; +} + +export function teardownScratchRepo(scratchRepoRoot: string): void { + fs.rmSync(scratchRepoRoot, { recursive: true, force: true }); +} + +export function markScratchRepoShip(scratchRepoRoot: string): void { + execFileSync( + PYTHON_BIN, + [ + WARDEN_SCRIPT, + 'mark', + 'plan-reviewed', + 'SHIP', + 'test', + '--repo-root', + scratchRepoRoot, + ], + { cwd: scratchRepoRoot }, + ); +} + +function toEditEvent(scratchRepoRoot: string, scratchFilePath: string) { + return (_request: ToolCallRequest): Record => ({ + tool_name: 'Edit', + tool_input: { file_path: scratchFilePath }, + cwd: scratchRepoRoot, + }); +} + +// ── Direct-execution smoke run ────────────────────────────────────────────── + +async function runBlockedPathDemo(): Promise { + const { scratchRepoRoot, scratchFilePath } = setupScratchRepo(); + try { + const tool_ = makeScratchEditTool(scratchFilePath); + const middleware = createWardenGateMiddleware( + 'plan-review-gate', + toEditEvent(scratchRepoRoot, scratchFilePath), + scratchRepoRoot, + ); + const agent = createAgent({ + model: new FakeToolCallingModel({ + toolCalls: [ + [{ name: 'edit_scratch', args: { content: 'hello' }, id: 'call_1' }], + [], + ], + }), + tools: [tool_], + middleware: [middleware], + }); + const result = await agent.invoke({ + messages: [{ role: 'user', content: 'write hello to the scratch file' }], + }); + const wroteFile = fs.existsSync(scratchFilePath); + console.log( + `Blocked path: file written = ${wroteFile} (expected false), ` + + `messages = ${result.messages.length}`, + ); + const denyMessage = result.messages.find( + (m) => (m as { name?: string }).name === 'edit_scratch', + ) as { content?: unknown } | undefined; + console.log( + `Deny message content: ${JSON.stringify(denyMessage?.content)}`, + ); + } finally { + teardownScratchRepo(scratchRepoRoot); + } +} + +async function runAllowedPathDemo(): Promise { + const { scratchRepoRoot, scratchFilePath } = setupScratchRepo(); + try { + markScratchRepoShip(scratchRepoRoot); + const tool_ = makeScratchEditTool(scratchFilePath); + const middleware = createWardenGateMiddleware( + 'plan-review-gate', + toEditEvent(scratchRepoRoot, scratchFilePath), + scratchRepoRoot, + ); + const agent = createAgent({ + model: new FakeToolCallingModel({ + toolCalls: [ + [{ name: 'edit_scratch', args: { content: 'hello' }, id: 'call_1' }], + [], + ], + }), + tools: [tool_], + middleware: [middleware], + }); + await agent.invoke({ + messages: [{ role: 'user', content: 'write hello to the scratch file' }], + }); + const wroteFile = fs.existsSync(scratchFilePath); + const content = wroteFile + ? fs.readFileSync(scratchFilePath, 'utf8') + : undefined; + console.log( + `Allowed path: file written = ${wroteFile} (expected true), content = ${JSON.stringify(content)}`, + ); + } finally { + teardownScratchRepo(scratchRepoRoot); + } +} + +async function main(): Promise { + console.log('=== A2 walking skeleton: blocked path ==='); + await runBlockedPathDemo(); + console.log('\n=== A2 walking skeleton: allowed path ==='); + await runAllowedPathDemo(); + console.log( + '\nSee scripts/spikes/lia395_warden_veto_walking_skeleton.test.ts for the ' + + 'full assertion suite (deny/allow/model-visible-feedback/fail-closed cases).', + ); +} + +// Only run when executed directly (not when imported by the unit tests). +// See lia394_langchain_walking_skeleton.ts for why this compares resolved +// filesystem paths rather than raw strings (Windows file:// vs backslash +// path mismatch). +const invokedDirectly = + process.argv[1] !== undefined && + fileURLToPath(import.meta.url) === path.resolve(process.argv[1]); +if (invokedDirectly) { + main().catch((err) => { + console.error('A2 spike failed:', err instanceof Error ? err.message : err); + process.exit(1); + }); +} diff --git a/vitest.config.ts b/vitest.config.ts index 46ba2703..2ada0c2b 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,7 +2,11 @@ import { defineConfig } from 'vitest/config'; export default defineConfig({ test: { - include: ['src/**/*.test.ts', 'setup/**/*.test.ts'], + include: [ + 'src/**/*.test.ts', + 'setup/**/*.test.ts', + 'scripts/spikes/**/*.test.ts', + ], coverage: { provider: 'v8', reporter: ['text', 'text-summary', 'lcov'],