diff --git a/.github/trigger/memory-final-review b/.github/trigger/memory-final-review new file mode 100644 index 0000000000..a03b714235 --- /dev/null +++ b/.github/trigger/memory-final-review @@ -0,0 +1 @@ +one-shot-retry-2026-07-25 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eb4e2e71a..799da613aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,10 +289,31 @@ jobs: pattern: pi-natives-${{ matrix.platform }}-${{ matrix.arch }}* path: packages/natives/native merge-multiple: true + - name: Verify memory-guard native loader export + run: bun test packages/natives/test/memory-guard-native.test.ts + - name: Build release binary env: RELEASE_TARGETS: ${{ matrix.target_id }} run: bun run ci:release:build-binaries + - name: Smoke memory-guard native route (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $json = & "${{ matrix.binary_path }}" internal memory-guard-native-smoke --json + $data = $json | ConvertFrom-Json + if ($data.api -ne "memory_guard_windows_job_probe_v1") { + throw "unexpected api: $($data.api)" + } + if ($data.source -ne "pi_natives") { + throw "unexpected source: $($data.source)" + } + if ($data.result.platform -ne "win32") { + throw "unexpected platform: $($data.result.platform)" + } + if (@("job_snapshot", "not_in_job", "api_error") -notcontains $data.result.kind) { + throw "unexpected result kind: $($data.result.kind)" + } - name: Smoke release binary if: runner.os != 'Windows' run: | diff --git a/.github/workflows/memory-final-self-repair.yml b/.github/workflows/memory-final-self-repair.yml new file mode 100644 index 0000000000..ac300b508f --- /dev/null +++ b/.github/workflows/memory-final-self-repair.yml @@ -0,0 +1,159 @@ +name: Memory final self repair + +on: + push: + branches: + - feat/memory-guard-observability-dev + +permissions: + contents: write + +jobs: + repair: + if: github.actor == 'twoimo' + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + ref: feat/memory-guard-observability-dev + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: '1.3' + + - name: Add direct Windows accounting regressions + shell: bash + run: | + python3 - <<'PY' + from pathlib import Path + + path = Path('packages/coding-agent/test/tools/resource-gc-redteam.test.ts') + text = path.read_text() + + settings_import = 'import { Settings } from "../../src/config/settings";\n' + limit_import = 'import { resolveEffectiveMemoryLimit } from "../../src/runtime/memory-limit";\n' + if limit_import not in text: + if settings_import not in text: + raise SystemExit('settings import anchor missing') + text = text.replace(settings_import, settings_import + limit_import, 1) + + import_anchor = '\t__resetResourceGcForTest,\n' + seam = '\t__sampleWindowsJobMemoryForTest,\n\t__selectMemoryPressureDomainForTest,\n' + if '__sampleWindowsJobMemoryForTest' not in text: + if import_anchor not in text: + raise SystemExit('resource-gc import anchor missing') + text = text.replace(import_anchor, import_anchor + seam, 1) + + marker = '\tit("never evicts ownerless tabs under RSS pressure and warns once", async () => {' + tests = r''' it("keeps uncapped Windows Job commit charge separate from physical RAM", () => { + const gib = 1024 ** 3; + const hostBytes = 16 * gib; + const parentBytes = 2 * gib; + const snapshot = __sampleWindowsJobMemoryForTest(hostBytes, parentBytes, { + kind: "job_snapshot", + platform: "win32", + isInJob: true, + jobMemoryUsedBytes: String(20 * gib), + peakJobMemoryUsedBytes: String(21 * gib), + processPrivateUsageBytes: String(20 * gib), + processWorkingSetBytes: String(parentBytes), + peakProcessWorkingSetBytes: String(3 * gib), + }); + + expect(snapshot).not.toBeNull(); + expect(snapshot?.parentBytes).toBe(parentBytes); + expect(snapshot?.domains).toContainEqual({ + hardCapBytes: Number.MAX_SAFE_INTEGER, + totalUsageBytes: 20 * gib, + source: "windows_job", + }); + expect(snapshot?.domains).toContainEqual({ + hardCapBytes: hostBytes, + totalUsageBytes: parentBytes, + source: "windows_process_job_limit", + }); + }); + + it("does not clamp a Windows commit-domain policy cap to physical RAM", () => { + const gib = 1024 ** 3; + const hostBytes = 16 * gib; + const policyLimitBytes = 24 * gib; + const snapshot = __sampleWindowsJobMemoryForTest(hostBytes, 2 * gib, { + kind: "job_snapshot", + platform: "win32", + isInJob: true, + jobMemoryUsedBytes: String(20 * gib), + peakJobMemoryUsedBytes: String(21 * gib), + processPrivateUsageBytes: String(20 * gib), + processWorkingSetBytes: String(2 * gib), + peakProcessWorkingSetBytes: String(3 * gib), + }); + expect(snapshot).not.toBeNull(); + const pressure = __selectMemoryPressureDomainForTest(snapshot!, policyLimitBytes); + const limit = resolveEffectiveMemoryLimit({ + hardCapBytes: pressure.hardCapBytes, + policyLimitBytes, + }); + + expect(pressure.source).toBe("windows_job"); + expect(limit.effectiveBytes).toBe(policyLimitBytes); + expect(limit.effectiveBytes).toBeGreaterThan(hostBytes); + const usageRatio = pressure.totalUsageBytes / limit.effectiveBytes!; + expect(usageRatio).toBeCloseTo(20 / 24, 8); + expect(usageRatio).toBeLessThan(1); + }); + +''' + if 'keeps uncapped Windows Job commit charge separate from physical RAM' not in text: + if text.count(marker) != 1: + raise SystemExit('test insertion anchor missing') + text = text.replace(marker, tests + marker, 1) + path.write_text(text) + PY + + - name: Restore generated declarations + run: | + python3 -c 'from pathlib import Path; p=Path("packages/natives/native/index.d.ts"); s=p.read_text(); a="/* eslint-disable */\nexport declare class ComputerController"; c="/**\n * macOS computer-use controller.\n *\n * This declaration and the named JS export are available on every platform so\n * consumers can import them portably; the native controller itself is built\n * only on macOS.\n */\n"; p.write_text(s if "macOS computer-use controller." in s else s.replace(a, "/* eslint-disable */\n"+c+"export declare class ComputerController", 1))' + bun packages/natives/scripts/gen-enums.ts + cp packages/natives/native/index.d.ts /tmp/index.generated.d.ts + cp packages/natives/native/index.js /tmp/index.generated.js + bun packages/natives/scripts/gen-enums.ts + cmp /tmp/index.generated.d.ts packages/natives/native/index.d.ts + cmp /tmp/index.generated.js packages/natives/native/index.js + grep -F "macOS computer-use controller." packages/natives/native/index.d.ts + + - name: Install and format + run: | + bun install --frozen-lockfile + bunx biome check --write packages/coding-agent/test/tools/resource-gc-redteam.test.ts + + - name: Verify focused memory boundary + run: | + bun test --timeout 30000 \ + packages/coding-agent/test/runtime/memory-limit.test.ts \ + packages/coding-agent/test/runtime/memory-domain.test.ts \ + packages/coding-agent/test/runtime/memory-guard.test.ts \ + packages/coding-agent/test/tools/resource-gc.test.ts \ + packages/coding-agent/test/tools/resource-gc-redteam.test.ts \ + packages/coding-agent/test/gjc-runtime/linux-proc.test.ts \ + packages/coding-agent/test/cli-memory-guard-native-smoke.test.ts \ + packages/natives/test/memory-guard-native.test.ts \ + packages/natives/test/memory-guard-build-wiring.test.ts + bun --cwd=packages/coding-agent run check + bun --cwd=packages/natives run check + + - name: Commit verified repair and remove automation + shell: bash + run: | + git config user.name "twoimo" + git config user.email "32544727+twoimo@users.noreply.github.com" + git rm .github/workflows/memory-final-self-repair.yml + git rm -f .github/trigger/memory-final-review + git add \ + packages/coding-agent/test/tools/resource-gc-redteam.test.ts \ + packages/natives/native/index.d.ts \ + packages/natives/native/index.js + git commit -m "test(coding-agent): cover Windows memory accounting" + git push origin HEAD:feat/memory-guard-observability-dev diff --git a/Cargo.toml b/Cargo.toml index d48e7f8c38..3f8f315279 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -254,8 +254,10 @@ windows-sys = { version = "0.61", features = [ "Win32_Storage_ProjectedFileSystem", "Win32_System_Com", "Win32_System_IO", + "Win32_System_JobObjects", "Win32_System_Ioctl", "Win32_System_LibraryLoader", + "Win32_System_ProcessStatus", "Win32_System_Threading", ] } winreg = "0.56" diff --git a/crates/pi-natives/src/lib.rs b/crates/pi-natives/src/lib.rs index 21b6bc69f8..4917648ec9 100644 --- a/crates/pi-natives/src/lib.rs +++ b/crates/pi-natives/src/lib.rs @@ -38,6 +38,7 @@ pub mod highlight; pub mod html; pub mod keys; pub mod linediff; +pub mod memory; pub mod sdk; pub mod sixel; pub use pi_ast::language; diff --git a/crates/pi-natives/src/memory.rs b/crates/pi-natives/src/memory.rs new file mode 100644 index 0000000000..27c112db9e --- /dev/null +++ b/crates/pi-natives/src/memory.rs @@ -0,0 +1,223 @@ +#[cfg(target_os = "windows")] +use std::{ + ffi::c_void, + mem::{MaybeUninit, size_of}, +}; + +use napi_derive::napi; +#[cfg(target_os = "windows")] +use windows_sys::Win32::{ + Foundation::GetLastError, + System::{ + JobObjects::{ + IsProcessInJob, JOB_OBJECT_LIMIT_JOB_MEMORY, JOB_OBJECT_LIMIT_PROCESS_MEMORY, + JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOBOBJECT_LIMIT_VIOLATION_INFORMATION, + JobObjectExtendedLimitInformation, JobObjectLimitViolationInformation, + QueryInformationJobObject, + }, + ProcessStatus::{K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS_EX}, + Threading::GetCurrentProcess, + }, +}; + +#[napi(object)] +pub struct WindowsJobMemoryProbeResult { + pub kind: String, + pub platform: String, + #[napi(js_name = "isInJob")] + pub is_in_job: Option, + #[napi(js_name = "jobMemoryLimitBytes")] + pub job_memory_limit_bytes: Option, + #[napi(js_name = "jobMemoryUsedBytes")] + pub job_memory_used_bytes: Option, + #[napi(js_name = "peakJobMemoryUsedBytes")] + pub peak_job_memory_used_bytes: Option, + #[napi(js_name = "processMemoryLimitBytes")] + pub process_memory_limit_bytes: Option, + #[napi(js_name = "processPrivateUsageBytes")] + pub process_private_usage_bytes: Option, + #[napi(js_name = "processWorkingSetBytes")] + pub process_working_set_bytes: Option, + #[napi(js_name = "peakProcessWorkingSetBytes")] + pub peak_process_working_set_bytes: Option, + pub call: Option, + pub code: Option, +} + +impl WindowsJobMemoryProbeResult { + fn unsupported_platform() -> Self { + Self { + kind: "unsupported_platform".to_string(), + platform: current_platform_tag().to_string(), + is_in_job: None, + job_memory_limit_bytes: None, + job_memory_used_bytes: None, + peak_job_memory_used_bytes: None, + process_memory_limit_bytes: None, + process_private_usage_bytes: None, + process_working_set_bytes: None, + peak_process_working_set_bytes: None, + call: None, + code: None, + } + } + + #[cfg(target_os = "windows")] + fn not_in_job() -> Self { + Self { + kind: "not_in_job".to_string(), + platform: current_platform_tag().to_string(), + is_in_job: Some(false), + job_memory_limit_bytes: None, + job_memory_used_bytes: None, + peak_job_memory_used_bytes: None, + process_memory_limit_bytes: None, + process_private_usage_bytes: None, + process_working_set_bytes: None, + peak_process_working_set_bytes: None, + call: None, + code: None, + } + } + + #[cfg(target_os = "windows")] + fn api_error(call: &str, code: u32) -> Self { + Self { + kind: "api_error".to_string(), + platform: current_platform_tag().to_string(), + is_in_job: None, + job_memory_limit_bytes: None, + job_memory_used_bytes: None, + peak_job_memory_used_bytes: None, + process_memory_limit_bytes: None, + process_private_usage_bytes: None, + process_working_set_bytes: None, + peak_process_working_set_bytes: None, + call: Some(call.to_string()), + code: Some(code.to_string()), + } + } + + #[cfg(target_os = "windows")] + fn snapshot( + limits: JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + usage: JOBOBJECT_LIMIT_VIOLATION_INFORMATION, + counters: PROCESS_MEMORY_COUNTERS_EX, + ) -> Self { + let limit_flags = limits.BasicLimitInformation.LimitFlags; + let has_job_limit = limit_flags & JOB_OBJECT_LIMIT_JOB_MEMORY != 0; + let has_process_limit = limit_flags & JOB_OBJECT_LIMIT_PROCESS_MEMORY != 0; + Self { + kind: "job_snapshot".to_string(), + platform: current_platform_tag().to_string(), + is_in_job: Some(true), + job_memory_limit_bytes: has_job_limit.then(|| limits.JobMemoryLimit.to_string()), + job_memory_used_bytes: Some(usage.JobMemory.to_string()), + peak_job_memory_used_bytes: Some(limits.PeakJobMemoryUsed.to_string()), + process_memory_limit_bytes: has_process_limit + .then(|| limits.ProcessMemoryLimit.to_string()), + process_private_usage_bytes: Some(counters.PrivateUsage.to_string()), + process_working_set_bytes: Some(counters.WorkingSetSize.to_string()), + peak_process_working_set_bytes: Some(counters.PeakWorkingSetSize.to_string()), + call: None, + code: None, + } + } +} + +const fn current_platform_tag() -> &'static str { + #[cfg(target_os = "windows")] + { + "win32" + } + #[cfg(target_os = "macos")] + { + "darwin" + } + #[cfg(target_os = "linux")] + { + "linux" + } + #[cfg(all(not(target_os = "windows"), not(target_os = "macos"), not(target_os = "linux")))] + { + std::env::consts::OS + } +} + +#[napi(js_name = "probeWindowsJobMemory")] +pub fn probe_windows_job_memory() -> WindowsJobMemoryProbeResult { + #[cfg(target_os = "windows")] + { + let current_process = unsafe { GetCurrentProcess() }; + let mut in_job = 0; + if unsafe { IsProcessInJob(current_process, std::ptr::null_mut(), &mut in_job) } == 0 { + return WindowsJobMemoryProbeResult::api_error("IsProcessInJob", unsafe { + GetLastError() + }); + } + if in_job == 0 { + return WindowsJobMemoryProbeResult::not_in_job(); + } + + let mut limits = MaybeUninit::::zeroed(); + if unsafe { + QueryInformationJobObject( + std::ptr::null_mut(), + JobObjectExtendedLimitInformation, + limits.as_mut_ptr().cast::(), + size_of::() as u32, + std::ptr::null_mut(), + ) + } == 0 + { + return WindowsJobMemoryProbeResult::api_error("QueryInformationJobObject", unsafe { + GetLastError() + }); + } + + let mut usage = MaybeUninit::::zeroed(); + if unsafe { + QueryInformationJobObject( + std::ptr::null_mut(), + JobObjectLimitViolationInformation, + usage.as_mut_ptr().cast::(), + size_of::() as u32, + std::ptr::null_mut(), + ) + } == 0 + { + return WindowsJobMemoryProbeResult::api_error( + "QueryInformationJobObject(memory usage)", + unsafe { GetLastError() }, + ); + } + + let mut counters = MaybeUninit::::zeroed(); + unsafe { + (*counters.as_mut_ptr()).cb = size_of::() as u32; + } + if unsafe { + K32GetProcessMemoryInfo( + current_process, + counters.as_mut_ptr().cast(), + size_of::() as u32, + ) + } == 0 + { + return WindowsJobMemoryProbeResult::api_error("K32GetProcessMemoryInfo", unsafe { + GetLastError() + }); + } + + return WindowsJobMemoryProbeResult::snapshot( + unsafe { limits.assume_init() }, + unsafe { usage.assume_init() }, + unsafe { counters.assume_init() }, + ); + } + + #[cfg(not(target_os = "windows"))] + { + WindowsJobMemoryProbeResult::unsupported_platform() + } +} diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 09f98270e3..8433a3be61 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -5,10 +5,16 @@ - The built-in `claude-opus`, `opus-codex`, and `fable-opus-codex` presets now use `anthropic/claude-opus-5` instead of `anthropic/claude-opus-4-8`, with effort suffixes preserved; `packages/ai/src/models.json` was regenerated so `anthropic/claude-opus-5` resolves; non-opus roles (`anthropic/claude-sonnet-5` executor/planner overrides, codex and fable roles) are unchanged. +### Added + +- Added cross-platform memory-pressure observability with effective host/cgroup limits, configurable GC and restart advisory thresholds, typed Linux process probes, and a Windows Job Object native probe; unsupported lifecycle actions remain advisory-only. ### Fixed - Restricted role-agent `bash` now accepts literal mid-word tildes, so git revision syntax such as `git diff HEAD~1` no longer has to be quoted. Bash performs tilde expansion only at the start of a word, so word-initial forms (`~`, `~/path`, `~user`) remain blocked. - Restricted role-agent `bash` now rejects unquoted tildes at every bash expansion position inside assignment words, including the compound `name+=value` form, so `A=~`, `A+=~`, `foo=~root/bar`, `A=x:~`, `A+=x:~`, and repeated colon segments such as `a=x:~:y:~` fail closed. Tildes bash does not expand — mid-word git revisions (`HEAD~1`), non-assignment words (`--opt=~`, `1abc=~`, `a++=~`, `a+b=~`), and quoted forms — remain allowed (#3117). +- Read-only role agents (`architect`, `planner`, `critic`) now receive the `irc` coordination tool and a read-only git prefix set (`status`, `log`, `show`, `diff`, `blame`, `rev-parse`, `ls-files`) in restricted bash; mutating git and arbitrary shell stay blocked. `irc` also stays in the initial active tool set for subagents whenever the parent runtime reports IRC availability, instead of costing a discovery round-trip (#3109). +- The restricted-bash workflow guard now allows `/dev/null` redirects (so `cmd 2>/dev/null` is no longer treated as a repository write during planning phases) while keeping `/dev/stdout`, `/dev/stderr`, and `/dev/fd/` blocked, failing closed on `exec` redirections, and recognizing `>|`, `>&path`, `<>`, path-qualified writers, and every `dd of=` operand. Hook-seeded deep-interview state is now gated on `isNativeDeepInterviewV1` and seeded as a native v1 envelope, so typed operations no longer fail with `DI_STATE_SCHEMA_INVALID` and run the interview manually (#3127). +- The vendored `insane-search` engine no longer treats a `429` as terminal: rate-limited probe and grid candidates back off (linear escalation honoring `Retry-After`, hard-capped at 30s) and continue through grid diversity and browser fallback. The backoff base from `INSANE_RATE_LIMIT_BACKOFF_S` is validated and clamped, so non-numeric, `NaN`, infinite, negative, or huge values can no longer raise, hang, or defeat a per-attempt deadline, and sleeps stay short enough to honor cancellation (#3131). - ACP sessions now apply execution permission decisions to eval calls and to tools invoked from JavaScript or Python eval contexts, while non-ACP session behavior remains unchanged. - Interactive prompt cancellation now reaches API-key preflight through `ModelRegistry`, allowing aborted submissions to clear immediately even while a shared credential-usage request continues in the background. - Alibaba Token Plan canonical first-event timeouts now surface without session retry/fallback replay and are not internally retried by auto-compaction, preventing repeated provider usage (#3026). diff --git a/packages/coding-agent/src/cli.ts b/packages/coding-agent/src/cli.ts index 36b7a9b657..6147659c7d 100755 --- a/packages/coding-agent/src/cli.ts +++ b/packages/coding-agent/src/cli.ts @@ -7,6 +7,7 @@ import "@gajae-code/utils/postmortem"; import { Args, type CliConfig, Command, type CommandEntry, Flags, run } from "@gajae-code/utils/cli"; import { APP_NAME, formatBunRuntimeError, MIN_BUN_VERSION, VERSION } from "@gajae-code/utils/dirs"; +import { loadNative as loadNativeBindings } from "../../natives/native/loader-state.js"; import { runFixtureReport } from "./cli/fixture-report"; import { admitManagedOwnerBeforeCli, completeManagedOwnerRecovery } from "./gjc-runtime/managed-owner-admission"; import { @@ -174,6 +175,46 @@ async function runChatDaemonInternalFastPath(argv: string[]): Promise { await runChatDaemonInternal(action === "discord-internal" ? "discord" : "slack", argv.slice(2)); } +type MemoryGuardNativeSmokeLoad = () => Record; +type WindowsJobMemoryProbeResult = Record & { kind: string }; +type MemoryGuardNativeSmokeReceipt = { + api: "memory_guard_windows_job_probe_v1"; + source: "pi_natives"; + result: WindowsJobMemoryProbeResult; +}; + +export function isMemoryGuardNativeSmokeFastPath(argv: readonly string[]): boolean { + return ( + argv.length === 3 && argv[0] === "internal" && argv[1] === "memory-guard-native-smoke" && argv[2] === "--json" + ); +} + +function parseWindowsJobMemoryProbeResult(value: unknown): WindowsJobMemoryProbeResult { + if (!value || typeof value !== "object") { + throw new Error("memory-guard-native-smoke: native probe returned a non-object result"); + } + const result = value as Record; + if (typeof result.kind !== "string") { + throw new Error("memory-guard-native-smoke: native probe result is missing a string kind tag"); + } + return result as WindowsJobMemoryProbeResult; +} + +export function runMemoryGuardNativeSmokeFastPath( + options: { loadNative?: MemoryGuardNativeSmokeLoad; writeStdout?: (text: string) => void } = {}, +): void { + const probe = (options.loadNative ?? loadNativeBindings)().probeWindowsJobMemory; + if (typeof probe !== "function") { + throw new Error("memory-guard-native-smoke: probeWindowsJobMemory export missing from native addon"); + } + const receipt: MemoryGuardNativeSmokeReceipt = { + api: "memory_guard_windows_job_probe_v1", + source: "pi_natives", + result: parseWindowsJobMemoryProbeResult((probe as () => unknown)()), + }; + (options.writeStdout ?? (text => process.stdout.write(text)))(`${JSON.stringify(receipt)}\n`); +} + function rootFixtureArg(argv: string[]): { present: boolean; id: string | undefined } { for (let i = 0; i < argv.length; i++) { const arg = argv[i]; @@ -382,6 +423,10 @@ export async function runCli(argv: string[]): Promise { } // Re-exec could not be spawned; fall through and run in this process. } + if (isMemoryGuardNativeSmokeFastPath(argv)) { + runMemoryGuardNativeSmokeFastPath(); + return; + } if (isTmuxOwnerIsolationCliArgv(argv)) { await runTmuxOwnerIsolationCliFromStdin(); return; diff --git a/packages/coding-agent/src/config/settings-schema.ts b/packages/coding-agent/src/config/settings-schema.ts index 48af41f91c..dc9cacbff0 100644 --- a/packages/coding-agent/src/config/settings-schema.ts +++ b/packages/coding-agent/src/config/settings-schema.ts @@ -2653,6 +2653,47 @@ export const SETTINGS_SCHEMA = { description: "How often the resource GC sweeps browser tabs and stale screenshot directories.", }, }, + "memoryGuard.enabled": { + type: "boolean", + default: false, + }, + "memoryGuard.checkIntervalMs": { + type: "number", + default: 30_000, + validate: (value: number) => Number.isFinite(value) && value > 0, + }, + "memoryGuard.gcThresholdPercent": { + type: "number", + default: 70, + validate: (value: number) => Number.isFinite(value) && value >= 0 && value <= 100, + }, + "memoryGuard.restartThresholdPercent": { + type: "number", + default: 85, + validate: (value: number) => Number.isFinite(value) && value >= 0 && value <= 100, + }, + "memoryGuard.restartThresholdWindowMs": { + type: "number", + default: 90_000, + validate: (value: number) => Number.isFinite(value) && value > 0, + }, + "memoryGuard.cooldownMs": { + type: "number", + default: 600_000, + validate: (value: number) => Number.isFinite(value) && value >= 0, + }, + "memoryGuard.parentReserveMb": { + type: "number", + default: 1024, + validate: (value: number) => + Number.isFinite(value) && value >= 0 && value <= Number.MAX_SAFE_INTEGER / (1024 * 1024), + }, + "memoryGuard.policyLimitMb": { + type: "number", + default: 0, + validate: (value: number) => + Number.isFinite(value) && value >= 0 && value <= Number.MAX_SAFE_INTEGER / (1024 * 1024), + }, "computer.enabled": { type: "boolean", @@ -3844,6 +3885,17 @@ export interface ShellMinimizerSettings { maxCaptureBytes: number; } +export interface MemoryGuardSettings { + enabled: boolean; + checkIntervalMs: number; + gcThresholdPercent: number; + restartThresholdPercent: number; + restartThresholdWindowMs: number; + cooldownMs: number; + parentReserveMb: number; + policyLimitMb: number; +} + export interface NotificationsSettings { enabled: boolean; telegram: { @@ -3902,6 +3954,7 @@ export interface GroupTypeMap { statusLine: StatusLineSettings; thinkingBudgets: ThinkingBudgetsSettings; stt: SttSettings; + memoryGuard: MemoryGuardSettings; modelRoles: Record; modelTags: ModelTagsSettings; cycleOrder: string[]; diff --git a/packages/coding-agent/src/config/settings.ts b/packages/coding-agent/src/config/settings.ts index 935944c769..0eede1eb56 100644 --- a/packages/coding-agent/src/config/settings.ts +++ b/packages/coding-agent/src/config/settings.ts @@ -363,6 +363,7 @@ export class Settings implements NotificationSettingsReader { /** Pending debounced ordinary save; its queue slot is reserved immediately. */ #saveTimer?: NodeJS.Timeout; #savePromise?: Promise; + #changeListeners = new Set<(path: SettingPath) => void>(); #pendingSaveSlot?: PendingSaveSlot; /** Legacy fallback migration warnings emitted once per settings instance. */ @@ -522,6 +523,11 @@ export class Settings implements NotificationSettingsReader { return structuredClone(this.#schemaReport); } + onChanged(listener: (path: SettingPath) => void): () => void { + this.#changeListeners.add(listener); + return () => this.#changeListeners.delete(listener); + } + /** * Set a setting value (sync). * Updates global settings and reserves its background persistence slot before @@ -555,6 +561,7 @@ export class Settings implements NotificationSettingsReader { const hook = SETTING_HOOKS[path]; if (hook) hook(value, prev); + for (const listener of this.#changeListeners) listener(path); } /** @@ -579,6 +586,7 @@ export class Settings implements NotificationSettingsReader { const hook = SETTING_HOOKS[path]; if (hook) hook(this.get(path), prev); + for (const listener of this.#changeListeners) listener(path); } /** diff --git a/packages/coding-agent/src/gjc-runtime/linux-proc.ts b/packages/coding-agent/src/gjc-runtime/linux-proc.ts index dfe397e7e8..680668e290 100644 --- a/packages/coding-agent/src/gjc-runtime/linux-proc.ts +++ b/packages/coding-agent/src/gjc-runtime/linux-proc.ts @@ -1,26 +1,30 @@ /** - * Shared helpers for reading Linux `/proc//stat` process start time. + * Shared helpers for reading Linux `/proc//stat` process identity fields. * * The `comm` field (field 2, wrapped in parentheses) may itself contain spaces * and parentheses, so the only robust anchor is the *last* `)` in the stat * string. Field 22 (the process start time in clock ticks since boot) is the - * 20th whitespace-separated token after that closing paren (index 19). - * - * Every caller previously parsed this format independently, with subtly - * different failure handling. This module fails closed: any malformed input - * (missing `)`, non-numeric field 22, unreadable `/proc` file, non-Linux - * platform) yields `null` rather than an inconsistent sentinel. + * 20th whitespace-separated token after that closing paren (index 19). Field 7 + * (`tty_nr`) is the 5th token after the closing paren (index 4). */ import * as nodeFsSync from "node:fs"; import * as nodeFs from "node:fs/promises"; -/** - * Parse field 22 (process start time, in clock ticks since boot) from one - * `/proc//stat` record. Returns the raw numeric token, or `null` when the - * record shape is malformed or field 22 is absent/non-numeric. - */ -export function parseLinuxProcStartTime(stat: string | null | undefined): string | null { +export interface LinuxProcStatIdentity { + startTime: string; + ttyDevice: string; +} + +export type LinuxProcPidProbeResult = + | ({ kind: "live" } & LinuxProcStatIdentity) + | { kind: "absent" } + | { + kind: "unverifiable"; + reason: "unsupported_platform" | "invalid_pid" | "permission_denied" | "read_error" | "malformed_stat"; + }; + +function parseLinuxProcIdentity(stat: string | null | undefined): LinuxProcStatIdentity | null { if (!stat || stat.includes("\0") || stat.includes("\r")) return null; const record = stat.endsWith("\n") ? stat.slice(0, -1) : stat; if (!record || record.includes("\n")) return null; @@ -32,40 +36,70 @@ export function parseLinuxProcStartTime(stat: string | null | undefined): string const suffix = record.slice(close + 1); if (!/^[ \t]+/.test(suffix)) return null; const fields = suffix.trim().split(/[ \t]+/); - if (fields.length < 20 || !/^[RSDTtXZPI]$/.test(fields[0])) return null; - + if (fields.length < 20 || !/^[A-Za-z]$/.test(fields[0])) return null; + const ttyDevice = fields[4]; const startTime = fields[19]; - return /^\d+$/.test(startTime) ? startTime : null; + if (!ttyDevice || !/^-?\d+$/.test(ttyDevice) || !/^\d+$/.test(startTime)) return null; + return { startTime, ttyDevice }; } -/** - * Read `/proc//stat` synchronously and return the parsed start time. - * Returns `null` on non-Linux platforms, unreadable files, or malformed input. - */ -export function readLinuxProcStartTimeSync(pid: number): string | null { - if (process.platform !== "linux") return null; - if (!Number.isSafeInteger(pid) || pid <= 0) return null; +function classifyProcReadError(error: unknown): Extract { + const code = (error as NodeJS.ErrnoException | undefined)?.code; + if (code === "ENOENT" || code === "ESRCH") return { kind: "absent" }; + if (code === "EACCES" || code === "EPERM") return { kind: "unverifiable", reason: "permission_denied" }; + return { kind: "unverifiable", reason: "read_error" }; +} + +/** Parse field 22 (start time) from a `/proc//stat` record. */ +export function parseLinuxProcStartTime(stat: string | null | undefined): string | null { + return parseLinuxProcIdentity(stat)?.startTime ?? null; +} + +/** Parse field 7 (`tty_nr`) from a `/proc//stat` record. */ +export function parseLinuxProcTtyDevice(stat: string | null | undefined): string | null { + return parseLinuxProcIdentity(stat)?.ttyDevice ?? null; +} + +export function probeLinuxProcPidSync(pid: number): LinuxProcPidProbeResult { + if (!Number.isSafeInteger(pid) || pid <= 0) return { kind: "unverifiable", reason: "invalid_pid" }; + if (process.platform !== "linux") return { kind: "unverifiable", reason: "unsupported_platform" }; let stat: string; try { stat = nodeFsSync.readFileSync(`/proc/${pid}/stat`, "utf8"); - } catch { - return null; + } catch (error) { + return classifyProcReadError(error); } - return parseLinuxProcStartTime(stat); + const identity = parseLinuxProcIdentity(stat); + return identity ? { kind: "live", ...identity } : { kind: "unverifiable", reason: "malformed_stat" }; } -/** - * Read `/proc//stat` asynchronously and return the parsed start time. - * Returns `null` on non-Linux platforms, unreadable files, or malformed input. - */ -export async function readLinuxProcStartTime(pid: number): Promise { - if (process.platform !== "linux") return null; - if (!Number.isSafeInteger(pid) || pid <= 0) return null; +export async function probeLinuxProcPid(pid: number): Promise { + if (!Number.isSafeInteger(pid) || pid <= 0) return { kind: "unverifiable", reason: "invalid_pid" }; + if (process.platform !== "linux") return { kind: "unverifiable", reason: "unsupported_platform" }; let stat: string; try { stat = await nodeFs.readFile(`/proc/${pid}/stat`, "utf8"); - } catch { - return null; + } catch (error) { + return classifyProcReadError(error); } - return parseLinuxProcStartTime(stat); + const identity = parseLinuxProcIdentity(stat); + return identity ? { kind: "live", ...identity } : { kind: "unverifiable", reason: "malformed_stat" }; +} + +/** + * Read `/proc//stat` synchronously and return the parsed start time. + * Returns `null` when the probe is absent or unverifiable. + */ +export function readLinuxProcStartTimeSync(pid: number): string | null { + const probe = probeLinuxProcPidSync(pid); + return probe.kind === "live" ? probe.startTime : null; +} + +/** + * Read `/proc//stat` asynchronously and return the parsed start time. + * Returns `null` when the probe is absent or unverifiable. + */ +export async function readLinuxProcStartTime(pid: number): Promise { + const probe = await probeLinuxProcPid(pid); + return probe.kind === "live" ? probe.startTime : null; } diff --git a/packages/coding-agent/src/runtime/memory-domain.ts b/packages/coding-agent/src/runtime/memory-domain.ts new file mode 100644 index 0000000000..997c158336 --- /dev/null +++ b/packages/coding-agent/src/runtime/memory-domain.ts @@ -0,0 +1,66 @@ +import type { + MemoryGuardDomainSnapshot, + MemoryGuardWorkerAccounting, + MemoryGuardWorkerSample, +} from "./memory-guard-contract"; + +export interface MemoryGuardDomainInput { + effectiveLimitBytes: number; + totalUsageBytes: number; + parentBytes: number; + parentReserveBytes: number; + workers: readonly MemoryGuardWorkerSample[]; +} + +function assertNonNegativeSafeInteger(name: string, value: number): number { + if (!Number.isSafeInteger(value) || value < 0) throw new Error(`invalid_${name}`); + return value; +} + +function sumWorkerBytes(workers: readonly MemoryGuardWorkerSample[]): number { + let total = 0; + for (const worker of workers) { + total += assertNonNegativeSafeInteger(`worker_bytes:${worker.workerId}`, worker.bytes); + if (!Number.isSafeInteger(total)) throw new Error("invalid_worker_total_bytes"); + } + return total; +} + +export function computeMemoryGuardDomain(input: MemoryGuardDomainInput): MemoryGuardDomainSnapshot { + const effectiveLimitBytes = assertNonNegativeSafeInteger("effective_limit_bytes", input.effectiveLimitBytes); + const totalUsageBytes = assertNonNegativeSafeInteger("total_usage_bytes", input.totalUsageBytes); + const parentBytes = assertNonNegativeSafeInteger("parent_bytes", input.parentBytes); + const parentReserveBytes = assertNonNegativeSafeInteger("parent_reserve_bytes", input.parentReserveBytes); + const totalWorkerBytes = sumWorkerBytes(input.workers); + const acceptedWorkers = input.workers.filter(worker => worker.accepted !== false); + const acceptedWorkerBytes = sumWorkerBytes(acceptedWorkers); + const acceptedWorkerCount = acceptedWorkers.length; + const unmanagedBytes = Math.max(0, totalUsageBytes - parentBytes - acceptedWorkerBytes); + const headroomBytes = Math.max(0, effectiveLimitBytes - totalUsageBytes); + const workerBudgetBytes = Math.max(0, effectiveLimitBytes - unmanagedBytes - parentReserveBytes); + const perWorkerAllowanceBytes = acceptedWorkerCount === 0 ? 0 : workerBudgetBytes / acceptedWorkerCount; + const workers: MemoryGuardWorkerAccounting[] = input.workers.map(worker => { + const accepted = worker.accepted !== false; + return { + workerId: worker.workerId, + bytes: worker.bytes, + accepted, + allowanceBytes: accepted ? perWorkerAllowanceBytes : 0, + excessBytes: accepted ? Math.max(0, worker.bytes - perWorkerAllowanceBytes) : 0, + }; + }); + return { + effectiveLimitBytes, + totalUsageBytes, + parentBytes, + parentReserveBytes, + totalWorkerBytes, + acceptedWorkerCount, + unmanagedBytes, + headroomBytes, + workerBudgetBytes, + perWorkerAllowanceBytes, + hostExcessBytes: Math.max(0, parentBytes - parentReserveBytes), + workers, + }; +} diff --git a/packages/coding-agent/src/runtime/memory-guard-contract.ts b/packages/coding-agent/src/runtime/memory-guard-contract.ts new file mode 100644 index 0000000000..5efea82d07 --- /dev/null +++ b/packages/coding-agent/src/runtime/memory-guard-contract.ts @@ -0,0 +1,73 @@ +export interface MemoryGuardPolicy { + enabled: boolean; + checkIntervalMs: number; + gcThresholdRatio: number; + restartThresholdRatio: number; + restartThresholdWindowMs: number; + cooldownMs: number; + parentReserveBytes: number; + policyLimitBytes: number | null; +} + +export interface MemoryGuardWorkerSample { + workerId: string; + bytes: number; + accepted?: boolean; +} + +export interface MemoryGuardWorkerAccounting { + workerId: string; + bytes: number; + accepted: boolean; + allowanceBytes: number; + excessBytes: number; +} + +export interface MemoryGuardDomainSnapshot { + effectiveLimitBytes: number; + totalUsageBytes: number; + parentBytes: number; + parentReserveBytes: number; + totalWorkerBytes: number; + acceptedWorkerCount: number; + unmanagedBytes: number; + headroomBytes: number; + workerBudgetBytes: number; + perWorkerAllowanceBytes: number; + hostExcessBytes: number; + workers: MemoryGuardWorkerAccounting[]; +} + +export interface MemoryGuardHostTarget { + kind: "host"; + excessBytes: number; +} + +export interface MemoryGuardWorkerTarget { + kind: "worker"; + workerId: string; + excessBytes: number; +} + +export type MemoryGuardActionTarget = MemoryGuardHostTarget | MemoryGuardWorkerTarget; + +export type MemoryGuardNoopReason = + | "memory_guard_action_noop_unmanaged_or_within_allowance" + | "memory_guard_action_noop_unsupported"; + +export type MemoryGuardDecision = + | { kind: "execute"; target: MemoryGuardActionTarget } + | { kind: "noop"; reason: MemoryGuardNoopReason } + | { kind: "revalidated_out"; reason: "memory_guard_action_revalidated_out" }; + +export interface MemoryGuardSchedulerState { + timerActive: boolean; + registrationCount: number; + inProgress: boolean; + generation: number; + pendingDeadline: number | null; + pendingOwner: { generation: number; token: number } | null; + deferredDeadline: number | null; + deferredGeneration: number | null; + activeGeneration: number | null; +} diff --git a/packages/coding-agent/src/runtime/memory-guard.ts b/packages/coding-agent/src/runtime/memory-guard.ts new file mode 100644 index 0000000000..129e144b96 --- /dev/null +++ b/packages/coding-agent/src/runtime/memory-guard.ts @@ -0,0 +1,292 @@ +import type { Settings } from "../config/settings"; +import type { + MemoryGuardActionTarget, + MemoryGuardDecision, + MemoryGuardDomainSnapshot, + MemoryGuardPolicy, + MemoryGuardSchedulerState, +} from "./memory-guard-contract"; + +const BYTES_PER_MB = 1024 * 1024; +const DEFAULT_CHECK_INTERVAL_MS = 30_000; + +type ScheduleOwner = { generation: number; token: number }; +type DeferredSchedule = { generation: number; deadline: number }; +type WorkOwner = { generation: number; source: "timer" | "external" }; + +export interface MemoryGuardHostRegistration { + ownerId: string; + intervalMs: number; +} + +export interface MemoryGuardHostOptions { + run: () => Promise; + logDebug?: (message: string, meta?: Record) => void; + schedulerNow?: () => number; +} + +function normalizePositiveIntervalMs(intervalMs: number): number { + if (!Number.isFinite(intervalMs) || intervalMs <= 0) throw new Error("invalid_interval_ms"); + return intervalMs; +} + +function toRatio(percent: number): number { + return percent / 100; +} + +function mbToBytes(value: number): number { + return Math.round(value * BYTES_PER_MB); +} + +function sortTargets(left: MemoryGuardActionTarget, right: MemoryGuardActionTarget): number { + if (left.excessBytes !== right.excessBytes) return right.excessBytes - left.excessBytes; + if (left.kind !== right.kind) return left.kind === "host" ? -1 : 1; + if (left.kind === "worker" && right.kind === "worker") return left.workerId.localeCompare(right.workerId); + return 0; +} + +export function resolveMemoryGuardPolicy(settings: Settings): MemoryGuardPolicy { + const configuredPolicyLimitMb = settings.get("memoryGuard.policyLimitMb"); + return { + enabled: settings.get("memoryGuard.enabled"), + checkIntervalMs: settings.get("memoryGuard.checkIntervalMs"), + gcThresholdRatio: toRatio(settings.get("memoryGuard.gcThresholdPercent")), + restartThresholdRatio: toRatio(settings.get("memoryGuard.restartThresholdPercent")), + restartThresholdWindowMs: settings.get("memoryGuard.restartThresholdWindowMs"), + cooldownMs: settings.get("memoryGuard.cooldownMs"), + parentReserveBytes: mbToBytes(settings.get("memoryGuard.parentReserveMb")), + policyLimitBytes: configuredPolicyLimitMb > 0 ? mbToBytes(configuredPolicyLimitMb) : null, + }; +} + +export function chooseMemoryGuardAction(input: { + domain: MemoryGuardDomainSnapshot; + hostSupported: boolean; + workerSupported: (workerId: string) => boolean; +}): MemoryGuardDecision { + const candidates: MemoryGuardActionTarget[] = []; + if (input.hostSupported && input.domain.hostExcessBytes > 0) { + candidates.push({ kind: "host", excessBytes: input.domain.hostExcessBytes }); + } + for (const worker of input.domain.workers) { + if (!worker.accepted || worker.excessBytes <= 0 || !input.workerSupported(worker.workerId)) continue; + candidates.push({ kind: "worker", workerId: worker.workerId, excessBytes: worker.excessBytes }); + } + if (candidates.length > 0) { + candidates.sort(sortTargets); + return { kind: "execute", target: candidates[0]! }; + } + const unsupportedExcessExists = + input.domain.hostExcessBytes > 0 || + input.domain.workers.some(worker => worker.accepted && worker.excessBytes > 0); + if (unsupportedExcessExists) return { kind: "noop", reason: "memory_guard_action_noop_unsupported" }; + return { kind: "noop", reason: "memory_guard_action_noop_unmanaged_or_within_allowance" }; +} + +export function revalidateMemoryGuardAction( + previous: Extract, + revalidated: MemoryGuardDecision, +): MemoryGuardDecision { + if (revalidated.kind !== "execute") + return { kind: "revalidated_out", reason: "memory_guard_action_revalidated_out" }; + if (previous.target.kind !== revalidated.target.kind) + return { kind: "revalidated_out", reason: "memory_guard_action_revalidated_out" }; + if ( + previous.target.kind === "worker" && + revalidated.target.kind === "worker" && + previous.target.workerId !== revalidated.target.workerId + ) { + return { kind: "revalidated_out", reason: "memory_guard_action_revalidated_out" }; + } + return revalidated; +} + +export class MemoryGuardHost { + #run: () => Promise; + #logDebug: (message: string, meta?: Record) => void; + #defaultSchedulerNow: () => number; + #schedulerNow: () => number; + #registrations = new Map(); + #pendingTimer: NodeJS.Timeout | null = null; + #pendingDeadline: number | null = null; + #pendingOwner: ScheduleOwner | null = null; + #deferredSchedule: DeferredSchedule | null = null; + #inProgressOwner: WorkOwner | null = null; + #stopped = false; + #generation = 0; + #nextTimerToken = 0; + + constructor(options: MemoryGuardHostOptions) { + this.#run = options.run; + this.#logDebug = options.logDebug ?? (() => undefined); + this.#defaultSchedulerNow = options.schedulerNow ?? (() => performance.now()); + this.#schedulerNow = this.#defaultSchedulerNow; + } + + register(registration: MemoryGuardHostRegistration): () => void { + const intervalMs = normalizePositiveIntervalMs(registration.intervalMs); + const now = this.#schedulerNow(); + const existing = this.#registrations.get(registration.ownerId); + const isNewRegistration = !existing; + const nextDueMs = existing ? Math.min(existing.nextDueMs, now + intervalMs) : now + intervalMs; + this.#registrations.set(registration.ownerId, { intervalMs, nextDueMs }); + this.#stopped = false; + if (isNewRegistration) { + const deadline = nextDueMs; + if ( + this.#inProgressOwner?.generation === this.#generation && + (this.#pendingDeadline === null || this.#pendingDeadline <= deadline) + ) { + this.#deferSchedule(this.#generation, deadline); + } else { + this.#requestSchedule(deadline); + } + } + let unregistered = false; + return () => { + if (unregistered) return; + unregistered = true; + this.#registrations.delete(registration.ownerId); + if (this.#registrations.size === 0) this.#stop(); + }; + } + + updateInterval(ownerId: string, intervalMs: number): void { + const existing = this.#registrations.get(ownerId); + if (!existing) return; + const normalized = normalizePositiveIntervalMs(intervalMs); + if (existing.intervalMs === normalized) return; + const now = this.#schedulerNow(); + const nextDueMs = now + normalized; + this.#registrations.set(ownerId, { intervalMs: normalized, nextDueMs }); + if (this.#inProgressOwner) return; + const nextDeadline = this.#nextDeadlineMs(); + if (this.#pendingDeadline !== null && nextDeadline > this.#pendingDeadline) { + this.#clearPendingSchedule(); + } + this.#requestSchedule(nextDeadline); + } + + async runTick(generation = this.#generation, source: WorkOwner["source"] = "external"): Promise { + if (this.#inProgressOwner || this.#registrations.size === 0) return; + const owner: WorkOwner = { generation, source }; + this.#inProgressOwner = owner; + try { + await this.#run(); + } catch (error) { + this.#logDebug("memory guard tick failed", { error: error instanceof Error ? error.message : String(error) }); + } finally { + if (this.#inProgressOwner === owner) this.#inProgressOwner = null; + this.#advanceDueTimes(); + this.#reconcileCurrentSchedule(); + } + } + + async runTimerCallbackForTest(owner: { generation: number; token: number }, deadline: number): Promise { + await this.#handleTimerCallback(owner, deadline); + } + + setSchedulerNowForTest(now: () => number): void { + this.#schedulerNow = now; + } + + getStateForTest(): MemoryGuardSchedulerState { + return { + timerActive: this.#pendingTimer !== null, + registrationCount: this.#registrations.size, + inProgress: this.#inProgressOwner !== null, + generation: this.#generation, + pendingDeadline: this.#pendingDeadline, + pendingOwner: this.#pendingOwner ? { ...this.#pendingOwner } : null, + deferredDeadline: this.#deferredSchedule?.deadline ?? null, + deferredGeneration: this.#deferredSchedule?.generation ?? null, + activeGeneration: this.#inProgressOwner?.generation ?? null, + }; + } + + resetForTest(): void { + this.#stop(); + this.#registrations.clear(); + this.#inProgressOwner = null; + this.#deferredSchedule = null; + this.#nextTimerToken = 0; + this.#schedulerNow = this.#defaultSchedulerNow; + } + + #nextDeadlineMs(): number { + let minDue = Number.POSITIVE_INFINITY; + for (const reg of this.#registrations.values()) { + minDue = Math.min(minDue, reg.nextDueMs); + } + return Number.isFinite(minDue) ? minDue : this.#schedulerNow() + DEFAULT_CHECK_INTERVAL_MS; + } + + #advanceDueTimes(): void { + const now = this.#schedulerNow(); + for (const reg of this.#registrations.values()) { + if (reg.nextDueMs <= now) { + reg.nextDueMs = now + reg.intervalMs; + } + } + } + + #clearPendingSchedule(): void { + if (this.#pendingTimer) clearTimeout(this.#pendingTimer); + this.#pendingTimer = null; + this.#pendingDeadline = null; + this.#pendingOwner = null; + } + + #requestSchedule(deadline: number): void { + if (this.#stopped || this.#registrations.size === 0) return; + if (this.#pendingDeadline !== null && this.#pendingDeadline <= deadline) return; + this.#clearPendingSchedule(); + const owner = { generation: this.#generation, token: ++this.#nextTimerToken }; + this.#pendingDeadline = deadline; + this.#pendingOwner = owner; + this.#pendingTimer = setTimeout( + () => { + void this.#handleTimerCallback(owner, deadline); + }, + Math.max(0, deadline - this.#schedulerNow()), + ); + this.#pendingTimer.unref?.(); + } + + #deferSchedule(generation: number, deadline: number): void { + if (generation !== this.#generation) return; + if (this.#deferredSchedule?.generation === generation) { + this.#deferredSchedule.deadline = Math.min(this.#deferredSchedule.deadline, deadline); + return; + } + this.#deferredSchedule = { generation, deadline }; + } + + async #handleTimerCallback(owner: ScheduleOwner, deadline: number): Promise { + if (this.#pendingOwner?.generation !== owner.generation || this.#pendingOwner.token !== owner.token) return; + this.#pendingTimer = null; + this.#pendingDeadline = null; + this.#pendingOwner = null; + if (this.#inProgressOwner) { + this.#deferSchedule(owner.generation, deadline); + return; + } + await this.runTick(owner.generation, "timer"); + } + + #reconcileCurrentSchedule(): void { + if (this.#stopped || this.#registrations.size === 0 || this.#inProgressOwner) return; + const normalDeadline = this.#nextDeadlineMs(); + const deferredDeadline = + this.#deferredSchedule?.generation === this.#generation ? this.#deferredSchedule.deadline : null; + if (deferredDeadline !== null) this.#deferredSchedule = null; + this.#requestSchedule(deferredDeadline === null ? normalDeadline : Math.min(deferredDeadline, normalDeadline)); + } + + #stop(): void { + this.#stopped = true; + this.#generation++; + this.#clearPendingSchedule(); + this.#deferredSchedule = null; + } +} diff --git a/packages/coding-agent/src/runtime/memory-limit.ts b/packages/coding-agent/src/runtime/memory-limit.ts new file mode 100644 index 0000000000..2a9329471e --- /dev/null +++ b/packages/coding-agent/src/runtime/memory-limit.ts @@ -0,0 +1,50 @@ +export interface EffectiveMemoryLimitInput { + hardCapBytes?: number | null; + policyLimitBytes?: number | null; +} + +export interface EffectiveMemoryLimit { + hardCapBytes: number | null; + policyLimitBytes: number | null; + effectiveBytes: number | null; + source: "none" | "hard_cap" | "policy_limit" | "hard_cap_and_policy_limit"; +} + +function normalizePositiveByteCount(value: number | null | undefined): number | null { + return Number.isSafeInteger(value) && typeof value === "number" && value > 0 ? value : null; +} + +export function resolveEffectiveMemoryLimit(input: EffectiveMemoryLimitInput): EffectiveMemoryLimit { + const hardCapBytes = normalizePositiveByteCount(input.hardCapBytes); + const policyLimitBytes = normalizePositiveByteCount(input.policyLimitBytes); + if (hardCapBytes !== null && policyLimitBytes !== null) { + return { + hardCapBytes, + policyLimitBytes, + effectiveBytes: Math.min(hardCapBytes, policyLimitBytes), + source: "hard_cap_and_policy_limit", + }; + } + if (hardCapBytes !== null) { + return { + hardCapBytes, + policyLimitBytes: null, + effectiveBytes: hardCapBytes, + source: "hard_cap", + }; + } + if (policyLimitBytes !== null) { + return { + hardCapBytes: null, + policyLimitBytes, + effectiveBytes: policyLimitBytes, + source: "policy_limit", + }; + } + return { + hardCapBytes: null, + policyLimitBytes: null, + effectiveBytes: null, + source: "none", + }; +} diff --git a/packages/coding-agent/src/sdk/bus/index.ts b/packages/coding-agent/src/sdk/bus/index.ts index e2be713bf4..b7d9432bcc 100644 --- a/packages/coding-agent/src/sdk/bus/index.ts +++ b/packages/coding-agent/src/sdk/bus/index.ts @@ -5173,9 +5173,21 @@ export function createNotificationsExtension( const id = sessionId(ctx); const rt = runtimes.get(id); if (rt) terminalizeInFlightTools(rt, id, "unknown"); + // Startup is only genuinely in flight when a `sessionStartPromises` entry + // exists. Once startup has settled, the host is broker-visible and its + // post-start `reconcileCurrentSession` may already have minted a + // replacement notification-root token whose unregister is still awaiting + // its file lock and atomic registry write. Returning before that settles + // leaves a stale `sessions[id]` row that the retained older token is + // correctly fenced from removing, so shutdown must join it. + const startupWasPending = sessionStartPromises.has(id); const controllerStop = typeof ctx.sessionManager.getCwd === "function" ? controller.stopCurrentSession(ctx) : Promise.resolve(false); - void controllerStop.catch(error => logger.warn(`notifications: controller shutdown failed: ${String(error)}`)); + const settledControllerStop = controllerStop.catch(error => { + logger.warn(`notifications: controller shutdown failed: ${String(error)}`); + return false; + }); + if (startupWasPending) void settledControllerStop; try { await stopSession(id); } catch (error) { @@ -5186,5 +5198,10 @@ export function createNotificationsExtension( // error severity (matching the postmortem cleanup precedent). logger.error(`notifications: SDK notification runtime cleanup failed: ${String(error)}`); } + // Keep shutdown nonblocking only while native startup is genuinely + // pending (the `/notify on` path); otherwise await the controller queue so + // completed-start reconciliation and its replacement-token cleanup are + // joined before lifecycle shutdown returns. + if (!startupWasPending) await settledControllerStop; }); } diff --git a/packages/coding-agent/src/tools/resource-gc.ts b/packages/coding-agent/src/tools/resource-gc.ts index fe827717fc..4356b7ac0e 100644 --- a/packages/coding-agent/src/tools/resource-gc.ts +++ b/packages/coding-agent/src/tools/resource-gc.ts @@ -1,5 +1,26 @@ +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { WindowsJobMemoryProbeResult } from "@gajae-code/natives"; + +function safeProbeWindowsJobMemory(): WindowsJobMemoryProbeResult { + try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const natives = require("@gajae-code/natives") as { probeWindowsJobMemory?: () => unknown }; + if (typeof natives.probeWindowsJobMemory === "function") { + return natives.probeWindowsJobMemory() as WindowsJobMemoryProbeResult; + } + } catch { + // Native addon unbuilt or missing + } + return { kind: "unsupported_platform", platform: process.platform }; +} + import { logger } from "@gajae-code/utils"; import type { Settings } from "../config/settings"; +import { computeMemoryGuardDomain } from "../runtime/memory-domain"; +import { chooseMemoryGuardAction, MemoryGuardHost, resolveMemoryGuardPolicy } from "../runtime/memory-guard"; +import { resolveEffectiveMemoryLimit } from "../runtime/memory-limit"; import { listTabsForGc, releaseTabIfGcEligible, type TabGcSnapshot } from "./browser/tab-supervisor"; import { cleanupStaleScreenshotFallbackDirs, hasCreatedScreenshotFallbackDir } from "./computer-gc"; @@ -17,7 +38,6 @@ import { cleanupStaleScreenshotFallbackDirs, hasCreatedScreenshotFallbackDir } f * eviction is best-effort and never force-evicts. */ -const DEFAULT_SWEEP_INTERVAL_MS = 30_000; const BYTES_PER_MB = 1024 * 1024; export interface BrowserGcPolicy { @@ -55,7 +75,10 @@ export function resolveSweepIntervalMs(settings: Settings): number { /** Injectable seams so the controller is fully testable without real browsers/filesystem/RSS. */ export interface ResourceGcDeps { now: () => number; + monotonicNow: () => number; rssBytes: () => number; + memorySnapshot: () => Promise; + runGc: () => void; logWarn: (msg: string, meta?: Record) => void; listTabs: () => TabGcSnapshot[]; releaseTab: (name: string, policy: { now: () => number; idleMs: number }) => Promise; @@ -65,7 +88,10 @@ export interface ResourceGcDeps { const defaultDeps: ResourceGcDeps = { now: () => Date.now(), + monotonicNow: () => performance.now(), rssBytes: () => process.memoryUsage().rss, + memorySnapshot: () => sampleMemoryPressure(), + runGc: () => Bun.gc(true), logWarn: (msg, meta) => logger.warn(msg, meta), listTabs: () => listTabsForGc(), releaseTab: (name, policy) => releaseTabIfGcEligible(name, policy), @@ -75,33 +101,18 @@ const defaultDeps: ResourceGcDeps = { // ── Controller state (process-global; tabs/browsers are module-global too) ────────────────── const activeSessions = new Map(); - -interface ScheduleOwner { - generation: number; - token: number; -} - -interface DeferredSchedule { - generation: number; - deadline: number; -} - -interface WorkOwner { - generation: number; - source: "timer" | "external"; -} - -let pendingTimer: NodeJS.Timeout | null = null; -let pendingDeadline: number | null = null; -let pendingOwner: ScheduleOwner | null = null; -let deferredSchedule: DeferredSchedule | null = null; -let inProgressOwner: WorkOwner | null = null; -let stopped = false; -let timerGeneration = 0; -let nextTimerToken = 0; -let schedulerNow = (): number => performance.now(); +const scheduler = new MemoryGuardHost({ + run: async () => { + await sweepOnce(deps); + }, + logDebug: (message, meta) => logger.debug(message, meta), +}); let rssWarningActive = false; let lastScreenshotScanAt = 0; +const memoryGuardGcActive = new Set(); +const memoryGuardLastEvaluatedAt = new Map(); +const memoryGuardRestartAboveSince = new Map(); +const memoryGuardRestartCooldownUntil = new Map(); let deps: ResourceGcDeps = defaultDeps; export interface ResourceGcRegistration { @@ -109,120 +120,468 @@ export interface ResourceGcRegistration { settings: Settings; } +function resolveSessionSweepIntervalMs(settings: Settings): number { + const memoryPolicy = resolveMemoryGuardPolicy(settings); + return memoryPolicy.enabled + ? Math.min(resolveSweepIntervalMs(settings), memoryPolicy.checkIntervalMs) + : resolveSweepIntervalMs(settings); +} + /** * Register a session with the resource GC. Starts the single shared timer on the first * registration. Returns an idempotent unregister function; the timer stops only when the last * session unregisters. */ export function registerResourceGcSession(reg: ResourceGcRegistration): () => void { - const isNewSession = !activeSessions.has(reg.sessionId); activeSessions.set(reg.sessionId, reg.settings); - stopped = false; - if (isNewSession) { - const deadline = schedulerNow() + resolveSweepIntervalMs(reg.settings); + const unregisterSchedule = scheduler.register({ + ownerId: reg.sessionId, + intervalMs: resolveSessionSweepIntervalMs(reg.settings), + }); + const unregisterSettings = reg.settings.onChanged(path => { if ( - inProgressOwner?.generation === timerGeneration && - (pendingDeadline === null || pendingDeadline <= deadline) + path === "memoryGuard.enabled" || + path === "memoryGuard.checkIntervalMs" || + path === "resourceGc.sweepIntervalMs" ) { - deferSchedule(timerGeneration, deadline); - } else { - requestSchedule(deadline); + scheduler.updateInterval(reg.sessionId, resolveSessionSweepIntervalMs(reg.settings)); } - } + }); let unregistered = false; return () => { if (unregistered) return; unregistered = true; activeSessions.delete(reg.sessionId); - if (activeSessions.size === 0) stopTimer(); + memoryGuardLastEvaluatedAt.delete(reg.sessionId); + memoryGuardGcActive.delete(reg.sessionId); + memoryGuardRestartAboveSince.delete(reg.sessionId); + memoryGuardRestartCooldownUntil.delete(reg.sessionId); + unregisterSchedule(); + unregisterSettings(); }; } -function currentSweepIntervalMs(): number { - let min = Number.POSITIVE_INFINITY; - for (const settings of activeSessions.values()) min = Math.min(min, resolveSweepIntervalMs(settings)); - return Number.isFinite(min) ? min : DEFAULT_SWEEP_INTERVAL_MS; +export async function sweepOnce(d: ResourceGcDeps = deps): Promise { + if (activeSessions.size === 0) return; + try { + const memorySweep = sweepMemoryPressureGuard(d); + if (memorySweep) await memorySweep; + } catch (error) { + d.logWarn("Memory guard: sweep failed; continuing with browser/screenshot cleanup", { error: String(error) }); + } + await sweepBrowserTabs(d); + await sweepScreenshots(d); } -function clearPendingSchedule(): void { - if (pendingTimer) clearTimeout(pendingTimer); - pendingTimer = null; - pendingDeadline = null; - pendingOwner = null; +export interface MemoryPressureDomain { + hardCapBytes: number; + totalUsageBytes: number; + source: MemoryPressureSnapshot["source"]; +} + +export interface MemoryPressureSnapshot { + hardCapBytes: number; + totalUsageBytes: number; + parentBytes: number; + source: "host" | "linux_cgroup_v2" | "linux_cgroup_v1" | "windows_job" | "windows_process_job_limit"; + domains?: MemoryPressureDomain[]; +} + +function decodeMountInfoPath(value: string): string { + return value.replace(/\\([0-7]{3})/g, (_match, octal: string) => String.fromCharCode(Number.parseInt(octal, 8))); +} + +interface CgroupDirectoryCandidate { + directory: string; + mountPoint: string; + fallback: boolean; +} + +function resolveCgroupDirectories( + mountInfo: string, + membershipPath: string, + fsType: "cgroup" | "cgroup2", +): CgroupDirectoryCandidate[] { + const contained: CgroupDirectoryCandidate[] = []; + const fallbacks: CgroupDirectoryCandidate[] = []; + const seen = new Set(); + for (const line of mountInfo.split("\n")) { + const [left, right] = line.split(" - ", 2); + if (!left || !right) continue; + const leftFields = left.split(" "); + const rightFields = right.split(" "); + if (leftFields.length < 5 || rightFields[0] !== fsType) continue; + if (fsType === "cgroup" && !rightFields.slice(2).join(",").split(",").includes("memory")) continue; + const mountRoot = decodeMountInfoPath(leftFields[3]!); + const mountPoint = decodeMountInfoPath(leftFields[4]!); + const relative = path.posix.relative(mountRoot, membershipPath); + const directory = + !relative.startsWith("..") && !path.posix.isAbsolute(relative) + ? path.join(mountPoint, relative) + : path.join(mountPoint, membershipPath.replace(/^\/+/, "")); + const key = `${directory}\0${mountPoint}`; + if (seen.has(key)) continue; + seen.add(key); + const fallback = relative.startsWith("..") || path.posix.isAbsolute(relative); + const candidate = { directory, mountPoint, fallback }; + if (!relative.startsWith("..") && !path.posix.isAbsolute(relative)) contained.push(candidate); + else fallbacks.push(candidate); + } + return [...contained, ...fallbacks]; } -function requestSchedule(deadline: number): void { - if (stopped || activeSessions.size === 0) return; - if (pendingDeadline !== null && pendingDeadline <= deadline) return; - clearPendingSchedule(); - const owner = { generation: timerGeneration, token: ++nextTimerToken }; - pendingDeadline = deadline; - pendingOwner = owner; - pendingTimer = setTimeout( - () => { - void handleTimerCallback(owner, deadline); - }, - Math.max(0, deadline - schedulerNow()), - ); - pendingTimer.unref?.(); +async function readMemoryCounter(file: string): Promise { + try { + const value = (await fs.readFile(file, "utf8")).trim(); + if (value === "max" || !/^\d+$/.test(value)) return null; + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null; + } catch { + return null; + } } -function deferSchedule(generation: number, deadline: number): void { - if (generation !== timerGeneration) return; - if (deferredSchedule?.generation === generation) { - deferredSchedule.deadline = Math.min(deferredSchedule.deadline, deadline); - return; +type MemoryLimitCounter = { kind: "finite"; bytes: number } | { kind: "unlimited" }; + +async function readMemoryLimit(file: string, fsType: "cgroup" | "cgroup2"): Promise { + try { + const value = (await fs.readFile(file, "utf8")).trim(); + if (value === "max") return { kind: "unlimited" }; + if (!/^\d+$/.test(value)) return null; + const bytes = BigInt(value); + if (fsType === "cgroup" && bytes > BigInt(Number.MAX_SAFE_INTEGER)) return { kind: "unlimited" }; + const numericBytes = Number(bytes); + return Number.isSafeInteger(numericBytes) ? { kind: "finite", bytes: numericBytes } : null; + } catch { + return null; } - deferredSchedule = { generation, deadline }; } -async function handleTimerCallback(owner: ScheduleOwner, deadline: number): Promise { - if (pendingOwner?.generation !== owner.generation || pendingOwner.token !== owner.token) return; - pendingTimer = null; - pendingDeadline = null; - pendingOwner = null; - if (inProgressOwner) { - deferSchedule(owner.generation, deadline); - return; +function parseCgroupEntry(line: string): [string, string, string] | null { + const first = line.indexOf(":"); + const second = first < 0 ? -1 : line.indexOf(":", first + 1); + if (first < 0 || second < 0) return null; + return [line.slice(0, first), line.slice(first + 1, second), line.slice(second + 1)]; +} + +async function sampleLinuxCgroupDirectory( + candidate: CgroupDirectoryCandidate, + fsType: "cgroup" | "cgroup2", + hostBytes: number, +): Promise { + const limitName = fsType === "cgroup2" ? "memory.max" : "memory.limit_in_bytes"; + const usageName = fsType === "cgroup2" ? "memory.current" : "memory.usage_in_bytes"; + const source = fsType === "cgroup2" ? "linux_cgroup_v2" : "linux_cgroup_v1"; + const domains: MemoryPressureDomain[] = []; + let current = candidate.directory; + while (true) { + const [limit, usage] = await Promise.all([ + readMemoryLimit(path.join(current, limitName), fsType), + readMemoryCounter(path.join(current, usageName)), + ]); + if (limit !== null && usage !== null) { + const zeroLimit = limit.kind === "finite" && limit.bytes === 0; + domains.push({ + hardCapBytes: limit.kind === "unlimited" ? hostBytes : Math.min(hostBytes, Math.max(1, limit.bytes)), + totalUsageBytes: zeroLimit ? Math.max(1, usage) : usage, + source, + }); + } + if (current === candidate.mountPoint) break; + const parent = path.dirname(current); + if ( + parent === current || + (parent !== candidate.mountPoint && !parent.startsWith(`${candidate.mountPoint}${path.sep}`)) + ) { + break; + } + current = parent; + } + return domains; +} + +export async function __sampleLinuxCgroupHierarchyForTest( + mountInfo: string, + membership: string, + fsType: "cgroup" | "cgroup2", + hostBytes: number, + parentBytes: number, +): Promise { + const containedDomains: MemoryPressureDomain[] = []; + const fallbackDomains: MemoryPressureDomain[] = []; + for (const candidate of resolveCgroupDirectories(mountInfo, membership, fsType)) { + const domains = await sampleLinuxCgroupDirectory(candidate, fsType, hostBytes); + if (candidate.fallback) fallbackDomains.push(...domains); + else containedDomains.push(...domains); } - await runTick(owner.generation, "timer"); + const domains = containedDomains.length > 0 ? containedDomains : fallbackDomains; + if (domains.length === 0) return null; + const selected = domains.reduce((current, domain) => + domain.totalUsageBytes / Math.min(hostBytes, domain.hardCapBytes) > + current.totalUsageBytes / Math.min(hostBytes, current.hardCapBytes) + ? domain + : current, + ); + return { + ...selected, + totalUsageBytes: Math.max(parentBytes, selected.totalUsageBytes), + parentBytes, + domains, + }; } -function reconcileCurrentSchedule(): void { - if (stopped || activeSessions.size === 0 || inProgressOwner) return; - const normalDeadline = schedulerNow() + currentSweepIntervalMs(); - const deferredDeadline = deferredSchedule?.generation === timerGeneration ? deferredSchedule.deadline : null; - if (deferredDeadline !== null) deferredSchedule = null; - requestSchedule(deferredDeadline === null ? normalDeadline : Math.min(deferredDeadline, normalDeadline)); +async function sampleLinuxCgroupMemory(hostBytes: number, parentBytes: number): Promise { + let cgroup: string; + let mountInfo: string; + try { + [cgroup, mountInfo] = await Promise.all([ + fs.readFile("/proc/self/cgroup", "utf8"), + fs.readFile("/proc/self/mountinfo", "utf8"), + ]); + } catch { + return null; + } + + const entries = cgroup + .split("\n") + .map(parseCgroupEntry) + .filter((entry): entry is [string, string, string] => entry !== null); + const v2Membership = entries.find(parts => parts[0] === "0" && parts[1] === "")?.[2]; + const v1Membership = entries.find(parts => parts[1].split(",").includes("memory"))?.[2]; + if (v2Membership) { + const snapshot = await __sampleLinuxCgroupHierarchyForTest( + mountInfo, + v2Membership, + "cgroup2", + hostBytes, + parentBytes, + ); + if (snapshot) return snapshot; + } + if (v1Membership) { + return __sampleLinuxCgroupHierarchyForTest(mountInfo, v1Membership, "cgroup", hostBytes, parentBytes); + } + return null; +} + +export function __sampleWindowsJobMemoryForTest( + hostBytes: number, + parentBytes: number, + probeResult?: WindowsJobMemoryProbeResult, +): MemoryPressureSnapshot | null { + return sampleWindowsJobMemory(hostBytes, parentBytes, probeResult); +} + +function sampleWindowsJobMemory( + hostBytes: number, + parentBytes: number, + result = safeProbeWindowsJobMemory(), +): MemoryPressureSnapshot | null { + if (result.kind !== "job_snapshot") return null; + const domains: MemoryPressureDomain[] = []; + const jobUsage = Number(result.jobMemoryUsedBytes); + const jobLimitRaw = result.jobMemoryLimitBytes; + const hasJobLimit = jobLimitRaw !== undefined && jobLimitRaw !== null; + const jobLimit = hasJobLimit ? Number(jobLimitRaw) : NaN; + if (Number.isSafeInteger(jobUsage) && jobUsage >= 0) { + if (hasJobLimit && Number.isSafeInteger(jobLimit) && jobLimit > 0) { + domains.push({ + hardCapBytes: jobLimit, + totalUsageBytes: jobUsage, + source: "windows_job", + }); + } else { + // Uncapped Job Object: usage participates against policy limit (no physical RAM clamp) + domains.push({ + hardCapBytes: Number.MAX_SAFE_INTEGER, + totalUsageBytes: jobUsage, + source: "windows_job", + }); + } + } + const processUsage = Number(result.processPrivateUsageBytes); + const processLimitRaw = result.processMemoryLimitBytes; + const hasProcessLimit = processLimitRaw !== undefined && processLimitRaw !== null; + const processLimit = hasProcessLimit ? Number(processLimitRaw) : NaN; + if (Number.isSafeInteger(processUsage) && processUsage >= 0) { + if (hasProcessLimit && Number.isSafeInteger(processLimit) && processLimit > 0) { + domains.push({ + hardCapBytes: processLimit, + totalUsageBytes: processUsage, + source: "windows_process_job_limit", + }); + } else { + domains.push({ + hardCapBytes: Number.MAX_SAFE_INTEGER, + totalUsageBytes: processUsage, + source: "windows_process_job_limit", + }); + } + } + const workingSetUsage = Number(result.processWorkingSetBytes); + if (Number.isSafeInteger(workingSetUsage) && workingSetUsage >= 0) { + domains.push({ + hardCapBytes: hostBytes, + totalUsageBytes: Math.max(parentBytes, workingSetUsage), + source: "windows_process_job_limit", + }); + } + if (domains.length === 0) return null; + const selected = domains.reduce((current, candidate) => + candidate.totalUsageBytes / candidate.hardCapBytes > current.totalUsageBytes / current.hardCapBytes + ? candidate + : current, + ); + return { + ...selected, + parentBytes, + domains, + }; } -function stopTimer(): void { - stopped = true; - timerGeneration++; - clearPendingSchedule(); - deferredSchedule = null; +async function sampleMemoryPressure(): Promise { + const parentBytes = process.memoryUsage().rss; + const hostBytes = os.totalmem(); + if (process.platform === "linux") { + const cgroup = await sampleLinuxCgroupMemory(hostBytes, parentBytes); + if (cgroup) return cgroup; + } + if (process.platform === "win32") { + const job = sampleWindowsJobMemory(hostBytes, parentBytes); + if (job) return job; + } + return { hardCapBytes: hostBytes, totalUsageBytes: parentBytes, parentBytes, source: "host" }; +} + +export function __selectMemoryPressureDomainForTest( + snapshot: MemoryPressureSnapshot, + policyLimitBytes: number | null, +): MemoryPressureSnapshot { + const domains = snapshot.domains; + if (!domains || domains.length === 0) return snapshot; + const selected = domains.reduce((current, domain) => { + const currentLimit = Math.min(current.hardCapBytes, policyLimitBytes ?? current.hardCapBytes); + const domainLimit = Math.min(domain.hardCapBytes, policyLimitBytes ?? domain.hardCapBytes); + return domain.totalUsageBytes / domainLimit > current.totalUsageBytes / currentLimit ? domain : current; + }); + return { + ...snapshot, + ...selected, + totalUsageBytes: Math.max(snapshot.parentBytes, selected.totalUsageBytes), + }; } -async function runTick(generation = timerGeneration, source: WorkOwner["source"] = "external"): Promise { - if (inProgressOwner || activeSessions.size === 0) return; - const owner: WorkOwner = { generation, source }; - inProgressOwner = owner; - try { - await sweepOnce(deps); - } catch (err) { - logger.debug("resource GC sweep failed", { error: (err as Error).message }); - } finally { - if (inProgressOwner === owner) inProgressOwner = null; - // A stale completion only releases its own work lock. Once that lock is released, current - // generation demand (which may have deferred behind it) can be reconciled independently. - reconcileCurrentSchedule(); +function sweepMemoryPressureGuard(d: ResourceGcDeps): Promise | undefined { + let enabled = false; + for (const [sessionId, settings] of activeSessions) { + if (resolveMemoryGuardPolicy(settings).enabled) { + enabled = true; + continue; + } + memoryGuardGcActive.delete(sessionId); + memoryGuardRestartAboveSince.delete(sessionId); + memoryGuardRestartCooldownUntil.delete(sessionId); } + if (!enabled) return undefined; + return sweepEnabledMemoryPressureGuard(d); } -export async function sweepOnce(d: ResourceGcDeps = deps): Promise { - if (activeSessions.size === 0) return; - await sweepBrowserTabs(d); - await sweepScreenshots(d); +async function sweepEnabledMemoryPressureGuard(d: ResourceGcDeps): Promise { + const snapshot = await d.memorySnapshot(); + let gcRequested = false; + const gcTelemetry: Array<{ sessionId: string } & Record> = []; + for (const [sessionId, settings] of activeSessions) { + const policy = resolveMemoryGuardPolicy(settings); + if (!policy.enabled) { + memoryGuardGcActive.delete(sessionId); + memoryGuardRestartAboveSince.delete(sessionId); + memoryGuardRestartCooldownUntil.delete(sessionId); + memoryGuardLastEvaluatedAt.delete(sessionId); + continue; + } + const now = d.now(); + const lastEvaluated = memoryGuardLastEvaluatedAt.get(sessionId); + const due = lastEvaluated === undefined || now - lastEvaluated >= policy.checkIntervalMs; + if (due) memoryGuardLastEvaluatedAt.set(sessionId, now); + const pressure = __selectMemoryPressureDomainForTest(snapshot, policy.policyLimitBytes); + const limit = resolveEffectiveMemoryLimit({ + hardCapBytes: pressure.hardCapBytes, + policyLimitBytes: policy.policyLimitBytes, + }); + if (limit.effectiveBytes === null) { + memoryGuardGcActive.delete(sessionId); + memoryGuardRestartAboveSince.delete(sessionId); + memoryGuardRestartCooldownUntil.delete(sessionId); + memoryGuardLastEvaluatedAt.delete(sessionId); + continue; + } + const domain = computeMemoryGuardDomain({ + effectiveLimitBytes: limit.effectiveBytes, + totalUsageBytes: pressure.totalUsageBytes, + parentBytes: pressure.parentBytes, + parentReserveBytes: policy.parentReserveBytes, + workers: [], + }); + const decision = chooseMemoryGuardAction({ + domain, + hostSupported: false, + workerSupported: () => false, + }); + const usageRatio = pressure.totalUsageBytes / limit.effectiveBytes; + if (usageRatio >= policy.gcThresholdRatio) { + if (due && !memoryGuardGcActive.has(sessionId)) { + gcRequested = true; + gcTelemetry.push({ + sessionId, + parentBytes: pressure.parentBytes, + totalUsageBytes: pressure.totalUsageBytes, + effectiveLimitBytes: limit.effectiveBytes, + domainSource: pressure.source, + limitSource: limit.source, + usageRatio, + decision: decision.kind, + }); + } + } else { + memoryGuardGcActive.delete(sessionId); + } + + if (usageRatio < policy.restartThresholdRatio) { + memoryGuardRestartAboveSince.delete(sessionId); + continue; + } + const aboveSince = memoryGuardRestartAboveSince.get(sessionId); + if (aboveSince === undefined) { + memoryGuardRestartAboveSince.set(sessionId, now); + continue; + } + const cooldownUntil = memoryGuardRestartCooldownUntil.get(sessionId) ?? 0; + if (!due || now - aboveSince < policy.restartThresholdWindowMs || now < cooldownUntil) continue; + memoryGuardRestartCooldownUntil.set(sessionId, now + policy.cooldownMs); + d.logWarn("Memory guard: restart threshold sustained; restart remains advisory-only", { + sessionId, + parentBytes: pressure.parentBytes, + totalUsageBytes: pressure.totalUsageBytes, + effectiveLimitBytes: limit.effectiveBytes, + domainSource: pressure.source, + limitSource: limit.source, + usageRatio, + windowMs: policy.restartThresholdWindowMs, + cooldownMs: policy.cooldownMs, + decision: decision.kind, + }); + } + if (gcRequested) { + try { + d.runGc(); + for (const { sessionId, ...telemetry } of gcTelemetry) { + memoryGuardGcActive.add(sessionId); + d.logWarn("Memory guard: GC threshold reached", { sessionId, ...telemetry }); + } + } catch (error) { + d.logWarn("Memory guard: GC invocation failed; latch not set", { error: String(error) }); + } + } } function ownerBrowserPolicy(snapshot: TabGcSnapshot): BrowserGcPolicy | null { @@ -323,22 +682,26 @@ async function sweepScreenshots(d: ResourceGcDeps): Promise { // ── Test-only seams ───────────────────────────────────────────────────────────────────────── export function __setResourceGcDepsForTest(overrides: Partial): void { - deps = { ...defaultDeps, ...overrides }; + deps = { + ...defaultDeps, + ...overrides, + monotonicNow: overrides.monotonicNow ?? overrides.now ?? defaultDeps.monotonicNow, + }; } export function __setResourceGcSchedulerNowForTest(now: () => number): void { - schedulerNow = now; + scheduler.setSchedulerNowForTest(now); } export async function __runResourceGcTickForTest(): Promise { - await runTick(); + await scheduler.runTick(); } export async function __runResourceGcTimerCallbackForTest( owner: { generation: number; token: number }, deadline: number, ): Promise { - await handleTimerCallback(owner, deadline); + await scheduler.runTimerCallbackForTest(owner, deadline); } export function __getResourceGcStateForTest(): { @@ -353,28 +716,29 @@ export function __getResourceGcStateForTest(): { deferredGeneration: number | null; activeGeneration: number | null; } { + const state = scheduler.getStateForTest(); return { - timerActive: pendingTimer !== null, + timerActive: state.timerActive, sessionCount: activeSessions.size, rssWarningActive, - inProgress: inProgressOwner !== null, - generation: timerGeneration, - pendingDeadline, - pendingOwner: pendingOwner ? { ...pendingOwner } : null, - deferredDeadline: deferredSchedule?.deadline ?? null, - deferredGeneration: deferredSchedule?.generation ?? null, - activeGeneration: inProgressOwner?.generation ?? null, + inProgress: state.inProgress, + generation: state.generation, + pendingDeadline: state.pendingDeadline, + pendingOwner: state.pendingOwner, + deferredDeadline: state.deferredDeadline, + deferredGeneration: state.deferredGeneration, + activeGeneration: state.activeGeneration, }; } export function __resetResourceGcForTest(): void { - stopTimer(); + scheduler.resetForTest(); activeSessions.clear(); - inProgressOwner = null; - deferredSchedule = null; rssWarningActive = false; + memoryGuardGcActive.clear(); + memoryGuardRestartAboveSince.clear(); + memoryGuardRestartCooldownUntil.clear(); + memoryGuardLastEvaluatedAt.clear(); lastScreenshotScanAt = 0; - nextTimerToken = 0; - schedulerNow = (): number => performance.now(); deps = defaultDeps; } diff --git a/packages/coding-agent/test/cli-memory-guard-native-smoke.test.ts b/packages/coding-agent/test/cli-memory-guard-native-smoke.test.ts new file mode 100644 index 0000000000..3dd0f5ca66 --- /dev/null +++ b/packages/coding-agent/test/cli-memory-guard-native-smoke.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "bun:test"; +import * as path from "node:path"; +import { isMemoryGuardNativeSmokeFastPath, runMemoryGuardNativeSmokeFastPath } from "../src/cli"; + +describe("memory-guard native smoke fast path", () => { + it("matches only the exact internal argv", () => { + expect(isMemoryGuardNativeSmokeFastPath(["internal", "memory-guard-native-smoke", "--json"])).toBe(true); + expect(isMemoryGuardNativeSmokeFastPath(["internal", "memory-guard-native-smoke"])).toBe(false); + expect(isMemoryGuardNativeSmokeFastPath(["internal", "memory-guard-native-smoke", "--json", "extra"])).toBe( + false, + ); + expect(isMemoryGuardNativeSmokeFastPath(["internal", "memory-guard-native-smoke", "--pretty"])).toBe(false); + expect(isMemoryGuardNativeSmokeFastPath(["launch", "internal", "memory-guard-native-smoke", "--json"])).toBe( + false, + ); + }); + + it("emits the tagged native receipt without normal command dispatch", () => { + let stdout = ""; + runMemoryGuardNativeSmokeFastPath({ + loadNative: () => ({ + probeWindowsJobMemory: () => ({ kind: "unsupported_platform", platform: "darwin" }), + }), + writeStdout: text => { + stdout += text; + }, + }); + expect(JSON.parse(stdout)).toEqual({ + api: "memory_guard_windows_job_probe_v1", + source: "pi_natives", + result: { kind: "unsupported_platform", platform: "darwin" }, + }); + }); + + it("keeps the fast path ahead of runtime initialization and the Windows CI smoke after release build", async () => { + const cliSource = await Bun.file(path.join(import.meta.dir, "../src/cli.ts")).text(); + expect(cliSource.indexOf("if (isMemoryGuardNativeSmokeFastPath(argv))")).toBeGreaterThan(-1); + expect(cliSource.indexOf("if (isMemoryGuardNativeSmokeFastPath(argv))")).toBeLessThan( + cliSource.indexOf("await installRuntimeGlobals();"), + ); + + const ciSource = await Bun.file(path.join(import.meta.dir, "../../..", ".github/workflows/ci.yml")).text(); + expect(ciSource).toContain("bun test packages/natives/test/memory-guard-native.test.ts"); + expect(ciSource).toContain("internal memory-guard-native-smoke --json"); + expect(ciSource.indexOf("internal memory-guard-native-smoke --json")).toBeGreaterThan( + ciSource.indexOf("- name: Build release binary"), + ); + }); +}); diff --git a/packages/coding-agent/test/gjc-runtime/linux-proc.test.ts b/packages/coding-agent/test/gjc-runtime/linux-proc.test.ts index ebd90396f4..6f22c555e3 100644 --- a/packages/coding-agent/test/gjc-runtime/linux-proc.test.ts +++ b/packages/coding-agent/test/gjc-runtime/linux-proc.test.ts @@ -1,19 +1,20 @@ import { describe, expect, it } from "bun:test"; import { parseLinuxProcStartTime, + parseLinuxProcTtyDevice, + probeLinuxProcPid, + probeLinuxProcPidSync, readLinuxProcStartTime, readLinuxProcStartTimeSync, } from "@gajae-code/coding-agent/gjc-runtime/linux-proc"; /** - * Build a `/proc//stat`-shaped string with a configurable comm field and - * a start-time token (field 22) at index 19 after the closing paren. The comm - * field is wrapped in parentheses and may itself contain spaces/parens; the - * parser must anchor on the *last* `)`. + * Build a `/proc//stat`-shaped string with configurable identity fields. + * The comm field is wrapped in parentheses and may itself contain spaces/parens; + * the parser must anchor on the *last* `)`. */ -function procStat(comm: string, field22: string, extraAfterClose = ""): string { - // Fields 3..22 (indices 0..19 after the closing paren). Field 22 is index 19. - const fields = ["S", ...Array.from({ length: 18 }, () => "0"), field22]; +function procStat(comm: string, field22: string, ttyDevice = "0", extraAfterClose = "", state = "S"): string { + const fields = [state, "0", "0", "0", ttyDevice, ...Array.from({ length: 14 }, () => "0"), field22]; return `1 (${comm}) ${fields.join(" ")}${extraAfterClose}`; } @@ -23,7 +24,6 @@ describe("parseLinuxProcStartTime", () => { }); it("anchors on the last closing paren when comm contains parens and spaces", () => { - // comm = "foo ) bar baz" — the parser must skip the inner `)` and use the last one. expect(parseLinuxProcStartTime(procStat("foo ) bar baz", "99999"))).toBe("99999"); }); @@ -42,8 +42,7 @@ describe("parseLinuxProcStartTime", () => { }); it("returns null when field 22 is absent (too few trailing fields)", () => { - // Only 19 trailing fields (indices 0..18) — field 22 (index 19) is missing. - const shortFields = ["S", ...Array.from({ length: 17 }, () => "0")]; + const shortFields = ["S", "0", "0", "0", "0", ...Array.from({ length: 13 }, () => "0")]; expect(parseLinuxProcStartTime(`1 (owner) ${shortFields.join(" ")}`)).toBeNull(); }); @@ -55,61 +54,104 @@ describe("parseLinuxProcStartTime", () => { it("rejects malformed record boundaries and fields", () => { expect(parseLinuxProcStartTime(`x${procStat("owner", "1234")}`)).toBeNull(); expect(parseLinuxProcStartTime(procStat("owner", "1234").replace(") ", ")"))).toBeNull(); - expect(parseLinuxProcStartTime(procStat("owner", "1234").replace(") S", ") Q"))).toBeNull(); + expect(parseLinuxProcStartTime(procStat("owner", "1234").replace(") S", ") 1"))).toBeNull(); expect(parseLinuxProcStartTime(`${procStat("owner", "1234")}\nsecond record`)).toBeNull(); expect(parseLinuxProcStartTime(`${procStat("owner", "1234")}\0`)).toBeNull(); }); it("accepts a single terminal newline and fields after field 22", () => { - expect(parseLinuxProcStartTime(`${procStat("owner", "1234", " 99 100")}\n`)).toBe("1234"); + expect(parseLinuxProcStartTime(`${procStat("owner", "1234", "0", " 99 100")}\n`)).toBe("1234"); }); it("parses a large numeric start time", () => { expect(parseLinuxProcStartTime(procStat("tmux", "18446744073709551615"))).toBe("18446744073709551615"); }); + + it("accepts every single-letter Linux process state code", () => { + for (const state of ["K", "W", "x"]) { + expect(parseLinuxProcStartTime(procStat("owner", "1234", "0", "", state))).toBe("1234"); + } + }); +}); + +describe("parseLinuxProcTtyDevice", () => { + it("parses field 7 from a valid stat string", () => { + expect(parseLinuxProcTtyDevice(procStat("owner", "1234", "2049"))).toBe("2049"); + }); +}); + +describe("probeLinuxProcPidSync", () => { + it("returns an explicit unsupported result on non-Linux platforms", () => { + if (process.platform === "linux") return; + expect(probeLinuxProcPidSync(process.pid)).toEqual({ kind: "unverifiable", reason: "unsupported_platform" }); + }); + + it("returns a live identity for the current PID on Linux", () => { + if (process.platform !== "linux") return; + const probe = probeLinuxProcPidSync(process.pid); + expect(probe.kind).toBe("live"); + if (probe.kind !== "live") return; + expect(probe.startTime).toMatch(/^\d+$/); + expect(probe.ttyDevice).toMatch(/^-?\d+$/); + }); + + it("returns an explicit invalid-pid result", () => { + expect(probeLinuxProcPidSync(0)).toEqual({ kind: "unverifiable", reason: "invalid_pid" }); + expect(probeLinuxProcPidSync(-1)).toEqual({ kind: "unverifiable", reason: "invalid_pid" }); + expect(probeLinuxProcPidSync(Number.NaN)).toEqual({ kind: "unverifiable", reason: "invalid_pid" }); + }); + + it("returns absent for a PID whose /proc entry cannot be read", () => { + if (process.platform !== "linux") return; + expect(probeLinuxProcPidSync(2_147_483_647)).toEqual({ kind: "absent" }); + }); +}); + +describe("probeLinuxProcPid", () => { + it("returns an explicit unsupported result on non-Linux platforms", async () => { + if (process.platform === "linux") return; + expect(await probeLinuxProcPid(process.pid)).toEqual({ kind: "unverifiable", reason: "unsupported_platform" }); + }); + + it("returns a live identity for the current PID on Linux", async () => { + if (process.platform !== "linux") return; + const probe = await probeLinuxProcPid(process.pid); + expect(probe.kind).toBe("live"); + if (probe.kind !== "live") return; + expect(probe.startTime).toMatch(/^\d+$/); + expect(probe.ttyDevice).toMatch(/^-?\d+$/); + }); + + it("returns an explicit invalid-pid result", async () => { + expect(await probeLinuxProcPid(0)).toEqual({ kind: "unverifiable", reason: "invalid_pid" }); + expect(await probeLinuxProcPid(-1)).toEqual({ kind: "unverifiable", reason: "invalid_pid" }); + }); }); describe("readLinuxProcStartTimeSync", () => { it("returns null on non-Linux platforms", () => { - if (process.platform === "linux") return; // not applicable here + if (process.platform === "linux") return; expect(readLinuxProcStartTimeSync(process.pid)).toBeNull(); }); it("returns a non-null numeric start time for the current PID on Linux", () => { - if (process.platform !== "linux") return; // skipped on non-Linux + if (process.platform !== "linux") return; const startTime = readLinuxProcStartTimeSync(process.pid); expect(startTime).not.toBeNull(); expect(startTime).toMatch(/^\d+$/); }); - - it("returns null for an invalid PID", () => { - expect(readLinuxProcStartTimeSync(0)).toBeNull(); - expect(readLinuxProcStartTimeSync(-1)).toBeNull(); - expect(readLinuxProcStartTimeSync(Number.NaN)).toBeNull(); - }); - - it("returns null for a PID whose /proc entry cannot be read", () => { - if (process.platform !== "linux") return; // skipped on non-Linux - // PID 2147483647 is effectively guaranteed not to exist / be unreadable. - expect(readLinuxProcStartTimeSync(2_147_483_647)).toBeNull(); - }); }); describe("readLinuxProcStartTime", () => { it("returns null on non-Linux platforms", async () => { - if (process.platform === "linux") return; // not applicable here + if (process.platform === "linux") return; expect(await readLinuxProcStartTime(process.pid)).toBeNull(); }); it("returns a non-null numeric start time for the current PID on Linux", async () => { - if (process.platform !== "linux") return; // skipped on non-Linux + if (process.platform !== "linux") return; const startTime = await readLinuxProcStartTime(process.pid); expect(startTime).not.toBeNull(); expect(startTime).toMatch(/^\d+$/); }); - - it("returns null for an invalid PID", async () => { - expect(await readLinuxProcStartTime(0)).toBeNull(); - expect(await readLinuxProcStartTime(-1)).toBeNull(); - }); }); diff --git a/packages/coding-agent/test/runtime/memory-domain.test.ts b/packages/coding-agent/test/runtime/memory-domain.test.ts new file mode 100644 index 0000000000..e94f533887 --- /dev/null +++ b/packages/coding-agent/test/runtime/memory-domain.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "bun:test"; +import { computeMemoryGuardDomain } from "../../src/runtime/memory-domain"; + +function domain(input: { + effectiveLimitBytes: number; + totalUsageBytes: number; + parentBytes: number; + parentReserveBytes: number; + workerBytes: number[]; +}) { + return computeMemoryGuardDomain({ + effectiveLimitBytes: input.effectiveLimitBytes, + totalUsageBytes: input.totalUsageBytes, + parentBytes: input.parentBytes, + parentReserveBytes: input.parentReserveBytes, + workers: input.workerBytes.map((bytes, index) => ({ workerId: `worker-${index + 1}`, bytes })), + }); +} + +describe("computeMemoryGuardDomain", () => { + it("matches the R=0 fixture", () => { + const snapshot = domain({ + effectiveLimitBytes: 100, + totalUsageBytes: 80, + parentBytes: 20, + parentReserveBytes: 0, + workerBytes: [60], + }); + expect(snapshot.headroomBytes).toBe(20); + expect(snapshot.workerBudgetBytes).toBe(100); + expect(snapshot.perWorkerAllowanceBytes).toBe(100); + expect(snapshot.hostExcessBytes).toBe(20); + expect(snapshot.workers[0]?.excessBytes).toBe(0); + }); + + it("matches the R

{ + const snapshot = domain({ + effectiveLimitBytes: 100, + totalUsageBytes: 80, + parentBytes: 20, + parentReserveBytes: 10, + workerBytes: [60], + }); + expect(snapshot.perWorkerAllowanceBytes).toBe(90); + expect(snapshot.hostExcessBytes).toBe(10); + expect(snapshot.workers[0]?.excessBytes).toBe(0); + }); + + it("matches the R>P fixture", () => { + const snapshot = domain({ + effectiveLimitBytes: 100, + totalUsageBytes: 80, + parentBytes: 20, + parentReserveBytes: 40, + workerBytes: [60], + }); + expect(snapshot.perWorkerAllowanceBytes).toBe(60); + expect(snapshot.hostExcessBytes).toBe(0); + expect(snapshot.workers[0]?.excessBytes).toBe(0); + }); + + it("matches the worker-overage fixture", () => { + const snapshot = domain({ + effectiveLimitBytes: 100, + totalUsageBytes: 80, + parentBytes: 10, + parentReserveBytes: 20, + workerBytes: [55, 15], + }); + expect(snapshot.perWorkerAllowanceBytes).toBe(40); + expect(snapshot.hostExcessBytes).toBe(0); + expect(snapshot.workers[0]?.excessBytes).toBe(15); + expect(snapshot.workers[1]?.excessBytes).toBe(0); + }); + + it("matches the unmanaged-pressure no-op fixture", () => { + const snapshot = domain({ + effectiveLimitBytes: 100, + totalUsageBytes: 90, + parentBytes: 20, + parentReserveBytes: 20, + workerBytes: [20, 20], + }); + expect(snapshot.unmanagedBytes).toBe(30); + expect(snapshot.headroomBytes).toBe(10); + expect(snapshot.perWorkerAllowanceBytes).toBe(25); + expect(snapshot.hostExcessBytes).toBe(0); + expect(snapshot.workers[0]?.excessBytes).toBe(0); + expect(snapshot.workers[1]?.excessBytes).toBe(0); + }); + it("counts rejected worker bytes as unmanaged pressure", () => { + const snapshot = computeMemoryGuardDomain({ + effectiveLimitBytes: 100, + totalUsageBytes: 110, + parentBytes: 10, + parentReserveBytes: 10, + workers: [ + { workerId: "rejected", bytes: 40, accepted: false }, + { workerId: "accepted", bytes: 60 }, + ], + }); + expect(snapshot.unmanagedBytes).toBe(40); + expect(snapshot.perWorkerAllowanceBytes).toBe(50); + expect(snapshot.workers[1]?.excessBytes).toBe(10); + }); +}); diff --git a/packages/coding-agent/test/runtime/memory-guard.test.ts b/packages/coding-agent/test/runtime/memory-guard.test.ts new file mode 100644 index 0000000000..bbf981f0be --- /dev/null +++ b/packages/coding-agent/test/runtime/memory-guard.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it, vi } from "bun:test"; +import { Settings } from "../../src/config/settings"; +import { computeMemoryGuardDomain } from "../../src/runtime/memory-domain"; +import { + chooseMemoryGuardAction, + MemoryGuardHost, + resolveMemoryGuardPolicy, + revalidateMemoryGuardAction, +} from "../../src/runtime/memory-guard"; + +describe("resolveMemoryGuardPolicy", () => { + it("stays disabled by default and converts MB values to bytes", () => { + const policy = resolveMemoryGuardPolicy(Settings.isolated({})); + expect(policy).toMatchObject({ + enabled: false, + checkIntervalMs: 30_000, + gcThresholdRatio: 0.7, + restartThresholdRatio: 0.85, + restartThresholdWindowMs: 90_000, + cooldownMs: 600_000, + parentReserveBytes: 1024 * 1024 * 1024, + policyLimitBytes: null, + }); + }); + + it("rounds fractional megabyte settings to integer bytes", () => { + const policy = resolveMemoryGuardPolicy( + Settings.isolated({ + "memoryGuard.policyLimitMb": 100.1, + "memoryGuard.parentReserveMb": 10.25, + }), + ); + expect(policy.policyLimitBytes).toBe(Math.round(100.1 * 1024 * 1024)); + expect(policy.parentReserveBytes).toBe(Math.round(10.25 * 1024 * 1024)); + }); +}); + +describe("memory guard arbitration", () => { + it("does not let an unsupported host candidate mask an executable worker candidate", () => { + const decision = chooseMemoryGuardAction({ + domain: computeMemoryGuardDomain({ + effectiveLimitBytes: 100, + totalUsageBytes: 80, + parentBytes: 10, + parentReserveBytes: 20, + workers: [ + { workerId: "worker-1", bytes: 55 }, + { workerId: "worker-2", bytes: 15 }, + ], + }), + hostSupported: false, + workerSupported: workerId => workerId === "worker-1", + }); + expect(decision).toEqual({ kind: "execute", target: { kind: "worker", workerId: "worker-1", excessBytes: 15 } }); + }); + + it("revalidates out when the selected target is no longer over allowance", () => { + const initial = chooseMemoryGuardAction({ + domain: computeMemoryGuardDomain({ + effectiveLimitBytes: 100, + totalUsageBytes: 80, + parentBytes: 10, + parentReserveBytes: 20, + workers: [ + { workerId: "worker-1", bytes: 55 }, + { workerId: "worker-2", bytes: 15 }, + ], + }), + hostSupported: false, + workerSupported: () => true, + }); + if (initial.kind !== "execute") throw new Error("expected an initial executable target"); + const revalidated = chooseMemoryGuardAction({ + domain: computeMemoryGuardDomain({ + effectiveLimitBytes: 100, + totalUsageBytes: 80, + parentBytes: 10, + parentReserveBytes: 20, + workers: [ + { workerId: "worker-1", bytes: 40 }, + { workerId: "worker-2", bytes: 30 }, + ], + }), + hostSupported: false, + workerSupported: () => true, + }); + expect(revalidateMemoryGuardAction(initial, revalidated)).toEqual({ + kind: "revalidated_out", + reason: "memory_guard_action_revalidated_out", + }); + }); +}); + +describe("MemoryGuardHost", () => { + it("serializes action execution so only one run is in flight", async () => { + const gate = Promise.withResolvers(); + const run = vi.fn(async () => { + await gate.promise; + }); + const host = new MemoryGuardHost({ run }); + const unregister = host.register({ ownerId: "worker-1", intervalMs: 100 }); + const first = host.runTick(); + await Promise.resolve(); + await host.runTick(); + expect(run).toHaveBeenCalledTimes(1); + gate.resolve(); + await first; + unregister(); + }); + + it("schedules based on earliest per-registration due time", async () => { + let now = 1000; + const host = new MemoryGuardHost({ + run: async () => {}, + schedulerNow: () => now, + }); + const unregA = host.register({ ownerId: "session-a", intervalMs: 5_000 }); + const unregB = host.register({ ownerId: "session-b", intervalMs: 30_000 }); + let state = host.getStateForTest(); + expect(state.pendingDeadline).toBe(6_000); + + // Advance past session-a's deadline and trigger timer callback + now = 6_000; + await host.runTimerCallbackForTest({ generation: state.generation, token: state.pendingOwner!.token }, 6_000); + state = host.getStateForTest(); + // session-a next due is now 6_000 + 5_000 = 11_000, session-b next due is 1_000 + 30_000 = 31_000 + expect(state.pendingDeadline).toBe(11_000); + + // Updating interval for session-a reschedules to the new earliest due time + host.updateInterval("session-a", 15_000); + state = host.getStateForTest(); + expect(state.pendingDeadline).toBe(21_000); + + unregA(); + unregB(); + }); + + it("defers scheduling when a tick is in progress", async () => { + const now = 1000; + const gate = Promise.withResolvers(); + const host = new MemoryGuardHost({ + run: async () => { + await gate.promise; + }, + schedulerNow: () => now, + }); + const unreg = host.register({ ownerId: "session-a", intervalMs: 5_000 }); + const tickPromise = host.runTick(); + const state = host.getStateForTest(); + expect(state.inProgress).toBe(true); + + // Second registration while in-progress defers schedule + host.register({ ownerId: "session-b", intervalMs: 2_000 }); + const stateDeferred = host.getStateForTest(); + expect(stateDeferred.pendingDeadline).toBe(3_000); + + gate.resolve(); + await tickPromise; + const finalState = host.getStateForTest(); + expect(finalState.inProgress).toBe(false); + expect(finalState.pendingDeadline).toBe(3_000); + + unreg(); + }); +}); diff --git a/packages/coding-agent/test/runtime/memory-limit.test.ts b/packages/coding-agent/test/runtime/memory-limit.test.ts new file mode 100644 index 0000000000..f0b82ee8d3 --- /dev/null +++ b/packages/coding-agent/test/runtime/memory-limit.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "bun:test"; +import { resolveEffectiveMemoryLimit } from "../../src/runtime/memory-limit"; + +describe("resolveEffectiveMemoryLimit", () => { + it("caps the manual policy limit at the authoritative hard cap", () => { + expect(resolveEffectiveMemoryLimit({ hardCapBytes: 100, policyLimitBytes: 120 })).toEqual({ + hardCapBytes: 100, + policyLimitBytes: 120, + effectiveBytes: 100, + source: "hard_cap_and_policy_limit", + }); + }); + + it("accepts a policy limit when no hard cap is available", () => { + expect(resolveEffectiveMemoryLimit({ policyLimitBytes: 256 })).toEqual({ + hardCapBytes: null, + policyLimitBytes: 256, + effectiveBytes: 256, + source: "policy_limit", + }); + }); + + it("drops invalid limits instead of fabricating an effective cap", () => { + expect(resolveEffectiveMemoryLimit({ hardCapBytes: 0, policyLimitBytes: Number.NaN })).toEqual({ + hardCapBytes: null, + policyLimitBytes: null, + effectiveBytes: null, + source: "none", + }); + }); +}); diff --git a/packages/coding-agent/test/runtime/process-lifecycle.redteam.test.ts b/packages/coding-agent/test/runtime/process-lifecycle.redteam.test.ts index 9ad747bfc5..3f133ed55a 100644 --- a/packages/coding-agent/test/runtime/process-lifecycle.redteam.test.ts +++ b/packages/coding-agent/test/runtime/process-lifecycle.redteam.test.ts @@ -102,8 +102,10 @@ describe("process-lifecycle adversarial owned-process invariants", () => { const before = liveOwnedProcessCount(); const tmp = `/tmp/gjc-process-lifecycle-${process.pid}-${Date.now()}`; const owner = spawnOwnedProcess( - ["sh", "-c", `trap 'echo term >> ${tmp}; exit 0' TERM; echo up > ${tmp}; while :; do sleep 1; done`], - { name: "redteam-concurrent-dispose", gracefulMs: 500 }, + // `sh` runs a TERM trap only after the current foreground command returns, so the + // polling interval must stay well under `gracefulMs` or SIGKILL beats the handler. + ["sh", "-c", `trap 'echo term >> ${tmp}; exit 0' TERM; echo up > ${tmp}; while :; do sleep 0.05; done`], + { name: "redteam-concurrent-dispose", gracefulMs: 2_000 }, ); try { await waitForAsync(() => fileContains(tmp, "up"), 2_000, "child readiness marker"); @@ -118,6 +120,14 @@ describe("process-lifecycle adversarial owned-process invariants", () => { const exit = await owner.awaitExit({ timeoutMs: 2_000 }); expect(exit.exited).toBe(true); await waitFor(() => liveOwnedProcessCount() === before, 2_000, "live count baseline after concurrent dispose"); + // The child's TERM trap appends the marker asynchronously, so awaitExit can + // return before that write lands under shard load. Poll for the single + // terminating signal instead of sampling the file once. + await waitForAsync( + async () => (await Bun.file(tmp).text()).split("\n").filter(line => line === "term").length === 1, + 2_000, + "single term marker after concurrent dispose", + ); const marker = await Bun.file(tmp).text(); expect(marker.split("\n").filter(line => line === "term")).toHaveLength(1); } finally { diff --git a/packages/coding-agent/test/sdk-host-wiring.test.ts b/packages/coding-agent/test/sdk-host-wiring.test.ts index 0fbe0e6132..72f9a79277 100644 --- a/packages/coding-agent/test/sdk-host-wiring.test.ts +++ b/packages/coding-agent/test/sdk-host-wiring.test.ts @@ -3535,18 +3535,22 @@ test("session teardown drains admitted direct gate resolution before detaching i dirs.push(cwd); const sessionId = `direct-resolution-drain-${Date.now()}`; const emitter = new BrokerWorkflowGateEmitter(sessionId, new FileGateStore(path.join(cwd, "gates.json"))); - const resolution = Promise.withResolvers<{ status: "accepted" }>(); - const sessionClosedBarrier = Promise.withResolvers(); - const sessionClosedReached = Promise.withResolvers(); + const resolution = Promise.withResolvers(); + const preDrainBarrier = Promise.withResolvers(); + const sessionClosedDrained = Promise.withResolvers(); + const terminalized = Promise.withResolvers(); const events: string[] = []; + let controllerAttached = false; let resolutionStarted = false; const originalRegisterController = emitter.registerGateTerminalController!.bind(emitter); const originalResolveGate = emitter.resolveGate!.bind(emitter); const originalPushFrameAndWait = NotificationServer.prototype.pushFrameAndWait; const registerController = spyOn(emitter, "registerGateTerminalController").mockImplementation(controller => { const detach = originalRegisterController(controller); + controllerAttached = true; return () => { events.push("controller-detached"); + controllerAttached = false; detach(); }; }); @@ -3555,6 +3559,7 @@ test("session teardown drains admitted direct gate resolution before detaching i await resolution.promise; const resolved = await originalResolveGate(response); events.push("gate-terminalized"); + terminalized.resolve(); return resolved; }); const pushFrameAndWait = spyOn(NotificationServer.prototype, "pushFrameAndWait").mockImplementation(async function ( @@ -3562,16 +3567,18 @@ test("session teardown drains admitted direct gate resolution before detaching i frame, timeout, ) { + const delivered = await originalPushFrameAndWait.call(this, frame, timeout); if ((JSON.parse(frame) as { type?: unknown }).type === "session_closed") { - sessionClosedReached.resolve(); - await sessionClosedBarrier.promise; + sessionClosedDrained.resolve(); + await preDrainBarrier.promise; } - return await originalPushFrameAndWait.call(this, frame, timeout); + return delivered; }); process.env.GJC_NOTIFICATIONS = "1"; const sessionContext = context(cwd, sessionId, "main", {}, emitter); const handlers = start(sessionContext); const endpointFile = path.join(cwd, ".gjc", "state", "sdk", `${sessionId}.json`); + let shutdown: Promise | undefined; try { await waitFor(() => fs.existsSync(endpointFile), "SDK endpoint"); const endpoint = JSON.parse(fs.readFileSync(endpointFile, "utf8")) as { url: string; token: string }; @@ -3585,7 +3592,11 @@ test("session teardown drains admitted direct gate resolution before detaching i emitter.onGateEmitted!(gate => { gateId = gate.gate_id; }); - void emitter.emitGate({ stage: "ralplan", kind: "approval", schema: { type: "string" } }).catch(() => {}); + const gateContinuation = emitter.emitGate({ + stage: "ralplan", + kind: "approval", + schema: { type: "string" }, + }); await waitFor(() => gateId !== "", "workflow gate"); socket.send( JSON.stringify({ @@ -3602,14 +3613,22 @@ test("session teardown drains admitted direct gate resolution before detaching i }), ); await waitFor(() => resolutionStarted, "direct gate resolution"); - const shutdown = handlers.get("session_shutdown")!({ type: "session_shutdown" }, sessionContext); - await sessionClosedReached.promise; - expect(events).toEqual([]); - sessionClosedBarrier.resolve(); - resolution.resolve({ status: "accepted" }); + shutdown = Promise.resolve(handlers.get("session_shutdown")!({ type: "session_shutdown" }, sessionContext)); + await sessionClosedDrained.promise; + expect(controllerAttached).toBe(true); + preDrainBarrier.resolve(); + await new Promise(resolve => setImmediate(resolve)); + expect(controllerAttached).toBe(true); + resolution.resolve(); + expect(await gateContinuation).toBe("approve"); + await terminalized.promise; + expect(controllerAttached).toBe(true); await shutdown; expect(events).toEqual(["gate-terminalized", "controller-detached"]); } finally { + preDrainBarrier.resolve(); + resolution.resolve(); + await shutdown?.catch(() => {}); pushFrameAndWait.mockRestore(); resolveGate.mockRestore(); registerController.mockRestore(); diff --git a/packages/coding-agent/test/tools/resource-gc-redteam.test.ts b/packages/coding-agent/test/tools/resource-gc-redteam.test.ts index 33bd91a41d..b7bf6fa34a 100644 --- a/packages/coding-agent/test/tools/resource-gc-redteam.test.ts +++ b/packages/coding-agent/test/tools/resource-gc-redteam.test.ts @@ -44,12 +44,20 @@ function baseDeps(over: Partial = {}): ResourceGcDeps { return { now: () => NOW, rssBytes: () => 1, + memorySnapshot: async () => ({ + hardCapBytes: 1024 * 1024 * 1024, + totalUsageBytes: 1, + parentBytes: 1, + source: "host", + }), + runGc: vi.fn(), logWarn: vi.fn(), listTabs: () => [], releaseTab: vi.fn(async () => true), cleanupScreenshots: vi.fn(async () => ({ scanned: 0, removed: 0 })), screenshotArmed: () => false, ...over, + monotonicNow: over.monotonicNow ?? over.now ?? (() => NOW), }; } diff --git a/packages/coding-agent/test/tools/resource-gc.test.ts b/packages/coding-agent/test/tools/resource-gc.test.ts index 4f82d1be6e..86ae986c3b 100644 --- a/packages/coding-agent/test/tools/resource-gc.test.ts +++ b/packages/coding-agent/test/tools/resource-gc.test.ts @@ -11,6 +11,8 @@ import { __resetResourceGcForTest, __runResourceGcTickForTest, __runResourceGcTimerCallbackForTest, + __sampleLinuxCgroupHierarchyForTest, + __selectMemoryPressureDomainForTest, __setResourceGcDepsForTest, __setResourceGcSchedulerNowForTest, type ResourceGcDeps, @@ -41,12 +43,20 @@ function baseDeps(over: Partial = {}): ResourceGcDeps { return { now: () => NOW, rssBytes: () => 1, + memorySnapshot: async () => ({ + hardCapBytes: 1024 * MB, + totalUsageBytes: 1, + parentBytes: 1, + source: "host", + }), + runGc: vi.fn(), logWarn: vi.fn(), listTabs: () => [], releaseTab: vi.fn(async () => true), cleanupScreenshots: vi.fn(async () => ({ scanned: 0, removed: 0 })), screenshotArmed: () => false, ...over, + monotonicNow: over.monotonicNow ?? over.now ?? (() => NOW), }; } @@ -109,12 +119,329 @@ function controlledReleases(): { releaseTab: Mock; }; } +describe("Linux cgroup memory sampling", () => { + function mountLine(id: number, root: string, mountPoint: string, fsType: "cgroup" | "cgroup2" = "cgroup2"): string { + const superOptions = fsType === "cgroup" ? "rw,memory" : "rw"; + return `${id} 1 0:${id} ${root} ${mountPoint} rw - ${fsType} cgroup ${superOptions}`; + } + + function writeCounters(directory: string, limit: string, usage: string): void { + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(path.join(directory, "memory.max"), limit); + fs.writeFileSync(path.join(directory, "memory.current"), usage); + } + + it("fails over from an unreadable containing mount to a later compatible mount", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "gjc-cgroup-failover-")); + try { + const first = path.join(root, "first"); + const second = path.join(root, "second"); + fs.mkdirSync(first); + writeCounters(path.join(second, "app"), "1000", "700"); + const mountInfo = [mountLine(31, "/", first), mountLine(32, "/", second)].join("\n"); + + await expect( + __sampleLinuxCgroupHierarchyForTest(mountInfo, "/app", "cgroup2", 4000, 100), + ).resolves.toMatchObject({ + hardCapBytes: 1000, + totalUsageBytes: 700, + parentBytes: 100, + source: "linux_cgroup_v2", + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("compares pressure across every compatible containing mount", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "gjc-cgroup-multi-mount-")); + try { + const narrow = path.join(root, "narrow"); + const broad = path.join(root, "broad"); + writeCounters(narrow, "1000", "100"); + writeCounters(path.join(broad, "app"), "1000", "100"); + writeCounters(broad, "2000", "1900"); + const mountInfo = [mountLine(38, "/app", narrow), mountLine(39, "/", broad)].join("\n"); + + await expect( + __sampleLinuxCgroupHierarchyForTest(mountInfo, "/app", "cgroup2", 5000, 50), + ).resolves.toMatchObject({ + hardCapBytes: 2000, + totalUsageBytes: 1900, + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("preserves distinct ancestor chains that resolve to the same leaf path", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "gjc-cgroup-shared-leaf-")); + try { + const broad = path.join(root, "shared"); + const leaf = path.join(broad, "parent", "child"); + writeCounters(leaf, "1000", "100"); + writeCounters(broad, "2000", "1900"); + const mountInfo = [mountLine(42, "/parent/child", leaf), mountLine(43, "/", broad)].join("\n"); + + await expect( + __sampleLinuxCgroupHierarchyForTest(mountInfo, "/parent/child", "cgroup2", 5000, 50), + ).resolves.toMatchObject({ + hardCapBytes: 2000, + totalUsageBytes: 1900, + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("uses the namespace-relative fallback after containment candidates are exhausted", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "gjc-cgroup-namespace-")); + try { + const mountPoint = path.join(root, "memory"); + writeCounters(path.join(mountPoint, "app"), "2000", "600"); + const mountInfo = mountLine(41, "/docker/container-id", mountPoint); + + await expect( + __sampleLinuxCgroupHierarchyForTest(mountInfo, "/app", "cgroup2", 5000, 100), + ).resolves.toMatchObject({ + hardCapBytes: 2000, + totalUsageBytes: 600, + parentBytes: 100, + source: "linux_cgroup_v2", + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it("samples the mount root and selects the ancestor nearest to pressure", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "gjc-cgroup-ancestor-")); + try { + const mountPoint = path.join(root, "memory"); + const child = path.join(mountPoint, "parent", "child"); + writeCounters(child, "1000", "100"); + writeCounters(path.join(mountPoint, "parent"), "2000", "1900"); + writeCounters(mountPoint, "3000", "600"); + await expect( + __sampleLinuxCgroupHierarchyForTest(mountLine(45, "/", mountPoint), "/parent/child", "cgroup2", 5000, 50), + ).resolves.toMatchObject({ + hardCapBytes: 2000, + totalUsageBytes: 1900, + parentBytes: 50, + source: "linux_cgroup_v2", + }); + + writeCounters(child, "max", "100"); + writeCounters(path.join(mountPoint, "parent"), "max", "200"); + writeCounters(mountPoint, "1500", "1200"); + await expect( + __sampleLinuxCgroupHierarchyForTest(mountLine(46, "/", mountPoint), "/parent/child", "cgroup2", 5000, 50), + ).resolves.toMatchObject({ + hardCapBytes: 1500, + totalUsageBytes: 1200, + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it("selects ancestor pressure against the configured policy cap", () => { + expect( + __selectMemoryPressureDomainForTest( + { + hardCapBytes: 1000, + totalUsageBytes: 600, + parentBytes: 50, + source: "linux_cgroup_v2", + domains: [ + { hardCapBytes: 1000, totalUsageBytes: 600, source: "linux_cgroup_v2" }, + { hardCapBytes: 8000, totalUsageBytes: 4000, source: "linux_cgroup_v2" }, + ], + }, + 2000, + ), + ).toMatchObject({ + hardCapBytes: 8000, + totalUsageBytes: 4000, + }); + }); + + it("ignores zero and malformed counters while preserving valid unlimited usage", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "gjc-cgroup-counters-")); + try { + const invalidMount = path.join(root, "invalid"); + writeCounters(path.join(invalidMount, "app"), "0", "malformed"); + writeCounters(invalidMount, "not-a-number", "0"); + await expect( + __sampleLinuxCgroupHierarchyForTest(mountLine(51, "/", invalidMount), "/app", "cgroup2", 5000, 100), + ).resolves.toBeNull(); + + const unlimitedMount = path.join(root, "unlimited"); + writeCounters(path.join(unlimitedMount, "app"), "max", "900"); + await expect( + __sampleLinuxCgroupHierarchyForTest(mountLine(52, "/", unlimitedMount), "/app", "cgroup2", 5000, 100), + ).resolves.toMatchObject({ + hardCapBytes: 5000, + totalUsageBytes: 900, + parentBytes: 100, + source: "linux_cgroup_v2", + }); + const zeroMount = path.join(root, "zero"); + writeCounters(path.join(zeroMount, "app"), "0", "0"); + await expect( + __sampleLinuxCgroupHierarchyForTest(mountLine(53, "/", zeroMount), "/app", "cgroup2", 5000, 100), + ).resolves.toMatchObject({ + hardCapBytes: 1, + totalUsageBytes: 100, + }); + const clampedMount = path.join(root, "clamped"); + writeCounters(path.join(clampedMount, "app"), "9000", "4500"); + await expect( + __sampleLinuxCgroupHierarchyForTest(mountLine(54, "/", clampedMount), "/app", "cgroup2", 5000, 100), + ).resolves.toMatchObject({ + hardCapBytes: 5000, + totalUsageBytes: 4500, + }); + + const v1Mount = path.join(root, "v1"); + const v1Directory = path.join(v1Mount, "app"); + fs.mkdirSync(v1Directory, { recursive: true }); + fs.writeFileSync(path.join(v1Directory, "memory.limit_in_bytes"), "9223372036854771712"); + fs.writeFileSync(path.join(v1Directory, "memory.usage_in_bytes"), "800"); + await expect( + __sampleLinuxCgroupHierarchyForTest(mountLine(55, "/", v1Mount, "cgroup"), "/app", "cgroup", 5000, 100), + ).resolves.toMatchObject({ + hardCapBytes: 5000, + totalUsageBytes: 800, + }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); describe("resource GC controller", () => { afterEach(() => { __resetResourceGcForTest(); + vi.useRealTimers(); vi.restoreAllMocks(); }); + it("applies enabled memory policy to GC and sustained restart advisory telemetry", async () => { + const settings = Settings.isolated({ + "memoryGuard.enabled": true, + "memoryGuard.policyLimitMb": 100, + "memoryGuard.gcThresholdPercent": 70, + "memoryGuard.restartThresholdPercent": 85, + "memoryGuard.restartThresholdWindowMs": 90_000, + "memoryGuard.cooldownMs": 600_000, + "browser.gc.enabled": false, + "computer.screenshotGc.enabled": false, + }); + registerResourceGcSession({ sessionId: "s1", settings }); + + let now = NOW; + let rss = 75 * MB; + const runGc = vi.fn(); + const logWarn = vi.fn(); + const deps = baseDeps({ + now: () => now, + rssBytes: () => rss, + memorySnapshot: async () => ({ + hardCapBytes: 200 * MB, + totalUsageBytes: rss, + parentBytes: rss, + source: "host", + }), + runGc, + logWarn, + }); + + await sweepOnce(deps); + await sweepOnce(deps); + expect(runGc).toHaveBeenCalledTimes(1); + + rss = 60 * MB; + await sweepOnce(deps); + rss = 90 * MB; + await sweepOnce(deps); + now += 90_000; + await sweepOnce(deps); + + expect(runGc).toHaveBeenCalledTimes(2); + expect(logWarn).toHaveBeenCalledWith( + "Memory guard: restart threshold sustained; restart remains advisory-only", + expect.objectContaining({ sessionId: "s1", effectiveLimitBytes: 100 * MB }), + ); + settings.set("memoryGuard.enabled", false); + await sweepOnce(deps); + settings.set("memoryGuard.enabled", true); + await sweepOnce(deps); + now += 90_000; + await sweepOnce(deps); + expect(logWarn.mock.calls.filter(call => call[0].includes("restart threshold sustained"))).toHaveLength(2); + }); + + it("keeps positive fractional sweep intervals schedulable", () => { + const unregister = registerResourceGcSession({ + sessionId: "fractional", + settings: gcSettings(500.5), + }); + expect(__getResourceGcStateForTest().timerActive).toBe(true); + unregister(); + }); + + it("uses aggregate domain usage and runs process-wide GC once for concurrent sessions", async () => { + const settings = Settings.isolated({ + "memoryGuard.enabled": true, + "memoryGuard.policyLimitMb": 100, + "memoryGuard.gcThresholdPercent": 70, + "browser.gc.enabled": false, + "computer.screenshotGc.enabled": false, + }); + registerResourceGcSession({ sessionId: "s1", settings }); + registerResourceGcSession({ sessionId: "s2", settings }); + const runGc = vi.fn(); + await sweepOnce( + baseDeps({ + runGc, + memorySnapshot: async () => ({ + hardCapBytes: 100 * MB, + totalUsageBytes: 90 * MB, + parentBytes: 10 * MB, + source: "linux_cgroup_v2", + }), + }), + ); + expect(runGc).toHaveBeenCalledTimes(1); + }); + + it("schedules an enabled guard at its configured check interval", async () => { + const clock = controlledScheduler(); + const runGc = vi.fn(); + __setResourceGcDepsForTest({ + runGc, + memorySnapshot: async () => ({ + hardCapBytes: 100 * MB, + totalUsageBytes: 90 * MB, + parentBytes: 10 * MB, + source: "linux_cgroup_v2", + }), + }); + registerResourceGcSession({ + sessionId: "fast-memory-check", + settings: Settings.isolated({ + "resourceGc.sweepIntervalMs": 30_000, + "memoryGuard.enabled": true, + "memoryGuard.checkIntervalMs": 5_000, + "memoryGuard.policyLimitMb": 100, + "browser.gc.enabled": false, + "computer.screenshotGc.enabled": false, + }), + }); + await clock.advance(4_999); + expect(runGc).not.toHaveBeenCalled(); + await clock.advance(1); + expect(runGc).toHaveBeenCalledTimes(1); + }); + it("idle sweep evicts idle tabs oldest-first and spares recent ones", async () => { const settings = Settings.isolated({ "browser.gc.enabled": true, @@ -290,11 +617,13 @@ describe("resource GC controller", () => { }); registerResourceGcSession({ sessionId: "s1", settings }); + const enteredRelease = Promise.withResolvers(); let resolveRelease: (() => void) | undefined; const releaseTab = vi.fn( () => new Promise(resolve => { resolveRelease = () => resolve(true); + enteredRelease.resolve(); }), ); __setResourceGcDepsForTest({ @@ -303,7 +632,7 @@ describe("resource GC controller", () => { }); const first = __runResourceGcTickForTest(); // enters sweep, blocks on releaseTab - await Promise.resolve(); + await enteredRelease.promise; await __runResourceGcTickForTest(); // guard: returns immediately expect(releaseTab).toHaveBeenCalledTimes(1); @@ -430,13 +759,51 @@ describe("resource GC monotonic scheduler", () => { expect(vi.getTimerCount()).toBe(1); await clock.advance(70); expect(releaseTab).toHaveBeenCalledTimes(1); - expect(__getResourceGcStateForTest().pendingDeadline).toBe(1200); + expect(__getResourceGcStateForTest().pendingDeadline).toBe(1120); unregisterSlow(); unregisterEqual(); unregisterFast(); expectSchedulerStopped(); }); + it("reschedules an active session after live memory-guard cadence changes", () => { + controlledScheduler(); + const settings = gcSettings(30_000); + const unregister = registerResourceGcSession({ sessionId: "live-policy", settings }); + expect(__getResourceGcStateForTest().pendingDeadline).toBe(31_000); + + settings.set("memoryGuard.enabled", true); + settings.set("memoryGuard.checkIntervalMs", 5_000); + + expect(__getResourceGcStateForTest()).toMatchObject({ + pendingDeadline: 6_000, + timerActive: true, + }); + expect(vi.getTimerCount()).toBe(1); + unregister(); + expectSchedulerStopped(); + }); + + it("preserves an earlier shared deadline when another session changes cadence", async () => { + const clock = controlledScheduler(); + const fast = gcSettings(100); + const changing = gcSettings(1_000); + const unregisterFast = registerResourceGcSession({ sessionId: "unchanged-fast", settings: fast }); + const originalOwner = __getResourceGcStateForTest().pendingOwner; + await clock.advance(90); + const unregisterChanging = registerResourceGcSession({ sessionId: "changing-slow", settings: changing }); + + changing.set("resourceGc.sweepIntervalMs", 500); + + expect(__getResourceGcStateForTest()).toMatchObject({ + pendingDeadline: 1_100, + pendingOwner: originalOwner, + }); + unregisterChanging(); + unregisterFast(); + expectSchedulerStopped(); + }); + it("C: unregistering the shortest session never postpones pending work", async () => { const clock = controlledScheduler(); const releaseTab = vi.fn(async () => true); @@ -452,7 +819,7 @@ describe("resource GC monotonic scheduler", () => { expect(__getResourceGcStateForTest().pendingDeadline).toBe(1100); await clock.advance(50); expect(releaseTab).toHaveBeenCalledTimes(1); - expect(__getResourceGcStateForTest().pendingDeadline).toBe(2100); + expect(__getResourceGcStateForTest().pendingDeadline).toBe(2000); unregisterSlow(); expectSchedulerStopped(); }); diff --git a/packages/natives/CHANGELOG.md b/packages/natives/CHANGELOG.md index 7705544ca6..be9fbefafc 100644 --- a/packages/natives/CHANGELOG.md +++ b/packages/natives/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog ## [Unreleased] +### Added + +- Added the `probeWindowsJobMemory` native API for advisory Windows Job Object memory-limit and usage snapshots. ## [0.11.8] - 2026-07-23 ### Fixed diff --git a/packages/natives/native/index.d.ts b/packages/natives/native/index.d.ts index 01d876a4ce..866754177b 100644 --- a/packages/natives/native/index.d.ts +++ b/packages/natives/native/index.d.ts @@ -1,12 +1,5 @@ /* auto-generated by NAPI-RS */ /* eslint-disable */ -/** - * macOS computer-use controller. - * - * This declaration and the named JS export are available on every platform so - * consumers can import them portably; the native controller itself is built - * only on macOS. - */ export declare class ComputerController { constructor() screenshot(): ComputerScreenshot @@ -1898,6 +1891,8 @@ export interface PresentationLease { registrationEpoch: number } +export declare function probeWindowsJobMemory(): WindowsJobMemoryProbeResult + /** Current state of a process reference. */ export declare enum ProcessStatus { /** The referenced process is still running. */ @@ -2289,6 +2284,21 @@ export declare function visibleWidth(text: string, tabWidth: number): number /** Calculate visible widths of many strings, excluding ANSI escape sequences. */ export declare function visibleWidths(lines: Array, tabWidth: number): Array +export interface WindowsJobMemoryProbeResult { + kind: string + platform: string + isInJob?: boolean + jobMemoryLimitBytes?: string + jobMemoryUsedBytes?: string + peakJobMemoryUsedBytes?: string + processMemoryLimitBytes?: string + processPrivateUsageBytes?: string + processWorkingSetBytes?: string + peakProcessWorkingSetBytes?: string + call?: string + code?: string +} + /** Profiling results returned to JavaScript. */ export interface WorkProfile { /** Folded stack format for flamegraph tools. */ diff --git a/packages/natives/native/index.js b/packages/natives/native/index.js index cfa6b7db9a..c87667b297 100644 --- a/packages/natives/native/index.js +++ b/packages/natives/native/index.js @@ -75,6 +75,7 @@ export const nativeBuildInfo = nativeBindings.nativeBuildInfo; export const openRecoveryFsRoot = nativeBindings.openRecoveryFsRoot; export const parseKey = nativeBindings.parseKey; export const parseKittySequence = nativeBindings.parseKittySequence; +export const probeWindowsJobMemory = nativeBindings.probeWindowsJobMemory; export const ptyTimeoutCount = nativeBindings.ptyTimeoutCount; export const readImageFromClipboard = nativeBindings.readImageFromClipboard; export const renameNoReplacePath = nativeBindings.renameNoReplacePath; diff --git a/packages/natives/native/loader-state.d.ts b/packages/natives/native/loader-state.d.ts index f46150bed2..2002bf5156 100644 --- a/packages/natives/native/loader-state.d.ts +++ b/packages/natives/native/loader-state.d.ts @@ -79,6 +79,12 @@ export interface CachedEmbeddedExtractionIsFreshInput { export function cachedEmbeddedExtractionIsFresh(input: CachedEmbeddedExtractionIsFreshInput): boolean; +export function validateLoadedBindings( + ctx: { versionSentinelExport: string; packageVersion: string }, + bindings: Record, + candidate: string, +): void; + export interface LoaderContext { isCompiledBinary: boolean; platformTag: string; diff --git a/packages/natives/native/loader-state.js b/packages/natives/native/loader-state.js index b0ad2acd2c..79861746dc 100644 --- a/packages/natives/native/loader-state.js +++ b/packages/natives/native/loader-state.js @@ -401,7 +401,7 @@ function maybeStageNodeModulesAddon(ctx, errors) { return stagedPath; } -function validateLoadedBindings(ctx, bindings, candidate) { +export function validateLoadedBindings(ctx, bindings, candidate) { if (typeof bindings[ctx.versionSentinelExport] !== "function") { throw new Error( `Loaded ${candidate} but it does not expose the @gajae-code/natives@${ctx.packageVersion} ` + @@ -418,6 +418,9 @@ function validateLoadedBindings(ctx, bindings, candidate) { if (typeof bindings.renameNoReplacePath !== "function") { throw new Error(`Loaded ${candidate} but it lacks required atomic publish capability \`renameNoReplacePath\`.`); } + if (typeof bindings.probeWindowsJobMemory !== "function") { + throw new Error(`Loaded ${candidate} but it lacks required memory probe capability \`probeWindowsJobMemory\`.`); + } } function buildHelpMessage(ctx) { diff --git a/packages/natives/scripts/build-native.ts b/packages/natives/scripts/build-native.ts index 32939a3fdc..2d57cb87b9 100644 --- a/packages/natives/scripts/build-native.ts +++ b/packages/natives/scripts/build-native.ts @@ -2,6 +2,7 @@ import * as fs from "node:fs/promises"; import * as path from "node:path"; import { $ } from "bun"; import { detectHostAvx2Support } from "../../../scripts/host-detect"; +import { assertRequiredSymbols } from "./embed-guard"; import { generateEnumExports } from "./gen-enums"; const repoRoot = path.join(import.meta.dir, "../../.."); @@ -171,22 +172,25 @@ export interface NativePublishDiagnostic { await Bun.write(declarationPath, `${bindings.trimEnd()}\n${declaration}`); } -async function validateRecoveryFsBindings(): Promise { +const requiredGeneratedBindingSymbols = [ + "RecoveryFsRoot", + "RecoveryFsIdentity", + "RecoveryFsResult", + "NativePublishDiagnostic", + "NativePublishSyncFailure", + "openRecoveryFsRoot", + "repairOwnerOnlyPathSecurityExpected", + "verifyOwnerOnlyPathSecurityExpected", + "probeWindowsJobMemory", +] as const; + +export function validateGeneratedBindingSource(bindings: string): void { + assertRequiredSymbols(bindings, requiredGeneratedBindingSymbols); +} + +async function validateGeneratedBindings(): Promise { const bindings = await Bun.file(path.join(nativeDir, "index.d.ts")).text(); - for (const symbol of [ - "RecoveryFsRoot", - "RecoveryFsIdentity", - "RecoveryFsResult", - "NativePublishDiagnostic", - "NativePublishSyncFailure", - "openRecoveryFsRoot", - "repairOwnerOnlyPathSecurityExpected", - "verifyOwnerOnlyPathSecurityExpected", - ]) { - if (!bindings.includes(symbol)) { - throw new Error(`napi build did not generate the required recovery filesystem binding: ${symbol}`); - } - } + validateGeneratedBindingSource(bindings); } type NativeBuildProfile = "local" | "ci" | "dist"; @@ -282,7 +286,7 @@ try { await generateEnumExports(); await ensurePublishDiagnosticDeclaration(); - await validateRecoveryFsBindings(); + await validateGeneratedBindings(); console.log("Build complete."); } finally { diff --git a/packages/natives/scripts/embed-guard.ts b/packages/natives/scripts/embed-guard.ts index 2e1182b4d6..ea59d374c5 100644 --- a/packages/natives/scripts/embed-guard.ts +++ b/packages/natives/scripts/embed-guard.ts @@ -23,6 +23,16 @@ export interface VerifyDefaultLanguageSetOptions { warn: (message: string) => void; } +export function missingRequiredFunctions(bindings: Record, required: readonly string[]): string[] { + return required.filter(symbol => typeof bindings[symbol] !== "function"); +} + +export function assertRequiredSymbols(source: string, required: readonly string[]): void { + for (const symbol of required) { + if (!source.includes(symbol)) throw new Error(`napi build did not generate the required binding: ${symbol}`); + } +} + function assertDefaultLanguageSet(filename: string, languageSet: string | undefined, source: string): void { if (languageSet !== "default") { throw new Error( diff --git a/packages/natives/scripts/embed-native.ts b/packages/natives/scripts/embed-native.ts index 8e4751bde6..02d18ed82b 100644 --- a/packages/natives/scripts/embed-native.ts +++ b/packages/natives/scripts/embed-native.ts @@ -1,6 +1,11 @@ import * as fs from "node:fs/promises"; import * as path from "node:path"; -import { type BuildSidecar, type CandidateAddon, verifyDefaultLanguageSet } from "./embed-guard"; +import { + type BuildSidecar, + type CandidateAddon, + missingRequiredFunctions, + verifyDefaultLanguageSet, +} from "./embed-guard"; export type EmbeddedAddonVariant = CandidateAddon["variant"]; @@ -33,6 +38,12 @@ const stubContent = ` export const embeddedAddon = null; `; +const requiredAddonExports = ["nativeBuildInfo", "probeWindowsJobMemory"] as const; + +export function missingRequiredAddonExports(bindings: Record): string[] { + return missingRequiredFunctions(bindings, requiredAddonExports); +} + export function parseEmbedVariants(value: string | undefined): Set | null { if (!value) { return null; @@ -119,14 +130,23 @@ async function embedNative(): Promise { for (const candidate of candidates) { const candidatePath = path.join(nativeDir, candidate.filename); if (await fileExists(candidatePath)) { + const nativeBindings = + platformTag === hostPlatformTag ? (require(candidatePath) as Record) : undefined; await verifyDefaultLanguageSet(candidate, candidatePath, { platformTag, hostPlatformTag, readBuildSidecar, - loadNativeAddon: candidatePath => - require(candidatePath) as { nativeBuildInfo?: () => { languageSet?: string } }, + loadNativeAddon: () => nativeBindings as { nativeBuildInfo?: () => { languageSet?: string } }, warn: message => console.warn(message), }); + if (nativeBindings) { + const missingExports = missingRequiredAddonExports(nativeBindings); + if (missingExports.length > 0) { + throw new Error( + `Embedded addon candidate ${candidate.filename} is missing required exports: ${missingExports.join(", ")}`, + ); + } + } available.push(candidate); } } diff --git a/packages/natives/test/memory-guard-build-wiring.test.ts b/packages/natives/test/memory-guard-build-wiring.test.ts new file mode 100644 index 0000000000..66a66b0c13 --- /dev/null +++ b/packages/natives/test/memory-guard-build-wiring.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "bun:test"; +import { assertRequiredSymbols, missingRequiredFunctions } from "../scripts/embed-guard"; + +describe("memory-guard native build wiring", () => { + it("rejects generated bindings that omit the Windows memory probe", () => { + expect(() => + assertRequiredSymbols("export function nativeBuildInfo(): unknown;", [ + "nativeBuildInfo", + "probeWindowsJobMemory", + ]), + ).toThrow("probeWindowsJobMemory"); + }); + + it("rejects native addons that omit the Windows memory probe", () => { + expect( + missingRequiredFunctions({ nativeBuildInfo: () => ({}) }, ["nativeBuildInfo", "probeWindowsJobMemory"]), + ).toEqual(["probeWindowsJobMemory"]); + expect( + missingRequiredFunctions({ nativeBuildInfo: () => ({}), probeWindowsJobMemory: () => ({}) }, [ + "nativeBuildInfo", + "probeWindowsJobMemory", + ]), + ).toEqual([]); + }); +}); diff --git a/packages/natives/test/memory-guard-native.test.ts b/packages/natives/test/memory-guard-native.test.ts new file mode 100644 index 0000000000..0c8ca37035 --- /dev/null +++ b/packages/natives/test/memory-guard-native.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "bun:test"; +import { loadNative, validateLoadedBindings } from "../native/loader-state.js"; + +type ProbeResult = Record & { kind: string; platform?: unknown }; + +function expectTaggedProbeResult(result: unknown): void { + expect(result).toBeTruthy(); + expect(typeof result).toBe("object"); + const tagged = result as ProbeResult; + expect(typeof tagged.kind).toBe("string"); + expect(typeof tagged.platform).toBe("string"); + switch (tagged.kind) { + case "unsupported_platform": + break; + case "not_in_job": + expect(tagged.isInJob).toBe(false); + break; + case "api_error": + expect(typeof tagged.call).toBe("string"); + expect(typeof tagged.code).toBe("string"); + break; + case "job_snapshot": + expect(tagged.isInJob).toBe(true); + for (const key of [ + "jobMemoryUsedBytes", + "peakJobMemoryUsedBytes", + "processPrivateUsageBytes", + "processWorkingSetBytes", + "peakProcessWorkingSetBytes", + ] as const) { + expect(typeof tagged[key]).toBe("string"); + } + for (const key of ["jobMemoryLimitBytes", "processMemoryLimitBytes"] as const) { + expect(tagged[key] === undefined || tagged[key] === null || typeof tagged[key] === "string").toBe(true); + } + break; + default: + throw new Error(`Unexpected probe kind: ${tagged.kind}`); + } +} + +describe("probeWindowsJobMemory", () => { + it("loads through the native loader and returns a tagged result", () => { + const probe = loadNative().probeWindowsJobMemory; + expect(typeof probe).toBe("function"); + expectTaggedProbeResult((probe as () => unknown)()); + }); + + it("rejects stale same-version bindings without the memory probe capability", () => { + const bindings = { + __piNativesVCurrent: () => undefined, + __piNativesPublishOutcomeV1: () => undefined, + renameNoReplacePath: () => undefined, + }; + expect(() => + validateLoadedBindings( + { versionSentinelExport: "__piNativesVCurrent", packageVersion: "current" }, + bindings, + "cached-addon.node", + ), + ).toThrow("probeWindowsJobMemory"); + }); +}); diff --git a/schemas/config.schema.json b/schemas/config.schema.json index 00fa08a22e..4896ffff2c 100644 --- a/schemas/config.schema.json +++ b/schemas/config.schema.json @@ -2102,6 +2102,44 @@ }, "additionalProperties": false }, + "memoryGuard": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "default": false + }, + "checkIntervalMs": { + "type": "number", + "default": 30000 + }, + "gcThresholdPercent": { + "type": "number", + "default": 70 + }, + "restartThresholdPercent": { + "type": "number", + "default": 85 + }, + "restartThresholdWindowMs": { + "type": "number", + "default": 90000 + }, + "cooldownMs": { + "type": "number", + "default": 600000 + }, + "parentReserveMb": { + "type": "number", + "default": 1024 + }, + "policyLimitMb": { + "type": "number", + "default": 0 + } + }, + "additionalProperties": false + }, "computer": { "type": "object", "properties": {