diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 000bb9a..08216d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,6 +119,11 @@ jobs: github.base_ref == 'main' || github.base_ref == 'pivot/sast-verifier' + env: + # GitHub-hosted runners often need more than local dev for image pulls + healthchecks. + SHIELDCLAW_COMPOSE_START_WAIT_SECONDS: "300" + SHIELDCLAW_COMPOSE_UP_TIMEOUT_SECONDS: "300" + steps: - uses: actions/checkout@v4 @@ -150,6 +155,12 @@ jobs: SHIELDCLAW_ATTACKER_IMAGE=shieldclaw-attacker:ci \ bash shield-claw/scripts/build_attacker_image.sh + - name: pull base images for integration tests + if: steps.docker_check.outputs.available == 'true' + run: | + docker pull nginx:alpine + docker pull postgres:15-alpine + - name: pytest integration if: steps.docker_check.outputs.available == 'true' env: @@ -160,4 +171,4 @@ jobs: -m integration --no-cov -v - --timeout=300 + --timeout=1200 diff --git a/shield-claw/docs/prd-sast-pipeline-v02.md b/shield-claw/docs/prd-sast-pipeline-v02.md new file mode 100644 index 0000000..d2b95d7 --- /dev/null +++ b/shield-claw/docs/prd-sast-pipeline-v02.md @@ -0,0 +1,238 @@ +# PRD: SAST Pipeline v0.2 Stabilization + +> Serialization of shared design alignment reached 2026-05-02. +> This is not a proposal — these decisions are made. + +--- + +## 1. Problem Statement + +ShieldClaw is a vulnerability verification pipeline that takes static analysis findings (Semgrep) and proves or disproves them by generating and detonating exploit code in isolated containers. + +The v0.2 SAST pipeline is architecturally complete — all 7 stages exist and connect — but has confirmed gaps in isolation, resumability, observability, and configurability that must be addressed before the pipeline can produce trustworthy results at scale. + +The legacy v0.1 diff-based pipeline is temporary scaffolding and will be retired once v0.2 is stable. No further investment in the legacy path. + +--- + +## 2. Solution Overview + +Address 13 confirmed gaps organized into three tiers: + +**Tier 1 — Security (must-fix before any real-target use):** +- Block outbound internet from attacker containers +- Add seccomp profile to attacker containers +- Validate LLM-generated `target_dns` against compose service names + +**Tier 2 — Correctness (must-fix for trustworthy results):** +- Mark interrupted detonations as INCONCLUSIVE on resume (no silent re-detonation) +- Wire `--timeout` CLI flag to `detonate()` (currently ignored) +- Surface observer failures in report output +- Enforce conservative multi-CWE conflict resolution (STATIC_ONLY wins) +- Warn on unmapped CWEs (don't silently drop to STATIC_ONLY) +- Retry once on LLM refusal before marking REFUSED + +**Tier 3 — Capability (post-stabilization):** +- Externalize CWE verdict map to config file +- CWE-specific log corroboration patterns +- In-process interactive HITL approval mode +- Pluggable report formats (JSON + SARIF + markdown) +- Agentic context enrichment (LLM tool-call for more source context) + +--- + +## 3. User Stories + +**Operator** = security engineer running ShieldClaw against a target application. + +| # | As an... | I want to... | So that... | +|---|----------|-------------|------------| +| 1 | Operator | Run ShieldClaw against a Semgrep report and get a verdict per finding | I know which findings are real vulnerabilities vs. noise | +| 2 | Operator | Trust that exploit code cannot phone home or escape the sandbox | I can run ShieldClaw against production-representative targets without risk | +| 3 | Operator | Resume an interrupted scan without re-detonating completed findings | Interrupted runs don't produce inconsistent results or side effects | +| 4 | Operator | See which observers failed in the report | I know when a verdict was reached with degraded evidence | +| 5 | Operator | Approve findings interactively in a terminal session | I don't need to run a separate CLI command between pipeline stages | +| 6 | Operator | Control detonation timeout via `--timeout` | I can adjust for slow-starting target services | +| 7 | Operator | Extend the CWE-to-verdict mapping without modifying code | I can add new CWEs as my Semgrep rules evolve | +| 8 | Consumer | Import ShieldClaw results into GitHub Code Scanning (SARIF) | Verified findings appear in my existing security workflow | + +--- + +## 4. Pipeline Architecture + +``` +Semgrep JSON ──> INGEST ──> TRIAGE ──> SCORE ──> APPROVE ──> POC GEN ──> DETONATE ──> VERDICT + (1) (2) (3) (4) (5) (6) (7) +``` + +All inter-stage data is persisted to SQLite. Each finding has a state that enables resumability. Actual states written by the orchestrator: `INGESTED → TRIAGED → SCORED → APPROVED → VERDICTED` (terminal). `REJECTED` is a terminal state when approval is denied. Note: the `shieldclaw approve` CLI subcommand queries for `AWAITING_APPROVAL` state, but the orchestrator currently leaves unapproved findings in `SCORED` — no code writes `AWAITING_APPROVAL`. This mismatch means the async HITL path is currently broken and must be fixed as part of issue #48. + +### Stage 1: Ingest +- **Module:** `ingest/semgrep.py` +- **Input:** Semgrep JSON file path +- **Output:** `List[Finding]` written to SQLite +- **Interface:** `parse_semgrep_json(path) → List[Finding]` +- **Notes:** Extracts CWE IDs from `metadata.cwe`, normalizes severity, generates UUID per finding. No source excerpt is stored at ingest time — the `Finding` dataclass and the SQLite schema have no excerpt field. The excerpt is reconstructed on-the-fly from disk at scoring and PoC generation time via `_extract_source_lines()` (orchestrator.py) using the finding's `path`, `start_line`, and `end_line`. Resumed scans therefore read the current file on disk, not the file as it was when Semgrep ran. + +### Stage 2: Triage +- **Module:** `triage/classifier.py` +- **Input:** `Finding` +- **Output:** `TriagedFinding(finding, verdict: TriageVerdict, reason: str)` +- **Interface:** `classify(finding) → TriagedFinding` +- **Verdicts:** `DYNAMICALLY_VERIFIABLE` | `STATIC_ONLY` | `OUT_OF_SCOPE` +- **Rules (pure, no LLM):** + - Prefix filter: dockerfile/terraform/kubernetes/secrets/license rules → OUT_OF_SCOPE + - INFO severity + no CWE → OUT_OF_SCOPE + - CWE lookup in `_CWE_VERDICTS` dict → DV or STATIC_ONLY + - Unmapped CWE fallback → STATIC_ONLY +- **Decision:** Multi-CWE conflict → STATIC_ONLY wins (conservative). Unmapped CWEs emit warning. + +### Stage 3: Score +- **Module:** `scoring/exploitability.py` +- **Input:** `Finding` + `source_excerpt` + `compose_yaml` (only DYNAMICALLY_VERIFIABLE findings) +- **Output:** `ExploitabilityScore(score: float 0-1, attack_surface: str, prerequisites: List[str])` +- **Interface:** `scorer.score(finding, excerpt, compose_yaml) → ExploitabilityScore` +- **Decision:** Score is stored in SQLite but does NOT influence the final verdict. Score-to-verdict modulation is deferred pending empirical accuracy data. + +### Stage 4: Approve +- **Module:** `approval/gate.py` +- **Input:** `Finding` in SCORED state +- **Output:** Finding state → APPROVED +- **Modes:** + - `SHIELDCLAW_AUTO_APPROVE=1`: auto-approve all (CI mode) — **fully implemented** + - Async (broken): orchestrator stops with findings in `SCORED` state; `shieldclaw approve` CLI exists but queries for `AWAITING_APPROVAL` which nothing writes — the transition `SCORED → AWAITING_APPROVAL` is missing from the orchestrator. Fix tracked in #48. + - Interactive (to build): pipeline blocks on stdin prompt — not yet implemented, tracked in #48 +- **Decision:** Both async and interactive modes needed. Async requires the orchestrator to write `AWAITING_APPROVAL` before stopping; interactive requires a blocking stdin loop. + +### Stage 5: PoC Generate +- **Module:** `intelligence/poc_generator.py` + `intelligence/parser.py` +- **Input:** `Finding` + `source_excerpt` + `compose_yaml` +- **Output:** `ExploitPayload(raw_code, target_dns, execution_command, language)` +- **Interface:** `poc_gen.generate(finding, excerpt, compose_yaml) → ExploitPayload` +- **Decision:** `target_dns` must be validated against compose service names before detonation. On LLM refusal, retry once with rephrased prompt; if second attempt fails, mark finding as REFUSED. +- **Future:** Agentic context enrichment — LLM tool-call to request more source context. + +### Stage 6: Detonate +- **Module:** `sandbox/docker_orchestrator.py` +- **Input:** `ExploitPayload` + target compose stack (already running) +- **Output:** `List[ObserverEvidence]` +- **Mechanism:** + - Compose project scoped by `sha256(result_id)[:20]` + - Attacker container: `--rm`, `--read-only`, `--user=1000:1000`, `--memory=256m`, `--cpus=0.5`, `--pids-limit=100`, `--tmpfs /tmp:rw,noexec,nosuid,size=32m` + - Exploit piped via stdin; timeout controlled by `--timeout` flag + - Observers: ExitCode (Tier-1), DockerDiff (Tier-2), TargetLogs (Tier-2) +- **Decisions:** + - Network must use `internal: true` to block egress + - Add Docker default seccomp profile + - Observer failures surfaced in report (not silent) + - Interrupted detonations → INCONCLUSIVE on resume (no re-run) + +### Stage 7: Verdict +- **Module:** `verdict/synthesizer.py` +- **Input:** `List[ObserverEvidence]` +- **Output:** `Verdict: TRUE_POSITIVE (0.95) | FALSE_POSITIVE | INCONCLUSIVE` +- **Interface:** `synthesize(evidence_list) → Verdict` +- **Rules (first-match-wins, deterministic):** + - exit_code=0 + Tier-2 corroboration → TRUE_POSITIVE + - exit_code != 0 → FALSE_POSITIVE + - otherwise → INCONCLUSIVE +- **Decision:** INCONCLUSIVE always means INCONCLUSIVE — LLM score does not tip it. CWE-specific log patterns replace generic keyword matching (future). + +--- + +## 5. Module Map + +### Deep (substantial logic, complex internals, simple interface) + +| Module | LOC | Role | Status | +|--------|-----|------|--------| +| `orchestrator.py` | ~578 | 7-stage state machine, resumability | Stable but needs resume fix (Q18) | +| `sandbox/docker_orchestrator.py` | ~683 | Docker lifecycle, isolation, detonation | Needs security hardening (Q12, Q15) | +| `persistence/store.py` | ~450 | SQLite schema, scan/finding lifecycle | Stable. Test WAL under concurrency | +| `models.py` | ~340 | All frozen dataclasses, ABCs | Needs REFUSED state added | +| `ingest/semgrep.py` | ~223 | JSON parsing, CWE extraction | Stable | +| `intelligence/parser.py` | ~171 | Refusal detection, JSON validation | Needs retry-on-refusal logic | + +### Medium (utility logic, will deepen) + +| Module | LOC | Role | Status | +|--------|-----|------|--------| +| `verdict/synthesizer.py` | ~156 | Evidence → verdict rules | Needs CWE-specific patterns, observer_warnings | +| `triage/classifier.py` | ~125 | CWE → verdict mapping | Needs config externalization, multi-CWE fix | +| `scoring/exploitability.py` | ~146 | LLM scoring prompt + parsing | Stable (score influence deferred) | +| `observer/docker_diff.py` | ~112 | Tier-2 filesystem diff | Stable | +| `observer/target_logs.py` | ~97 | Tier-2 log capture | Needs CWE-specific patterns | +| `context/aggregator.py` | ~161 | Git diff + compose reader | Legacy — will be retired | + +### Shallow (thin but correctly scoped — will deepen as gaps are addressed) + +| Module | LOC | Role | Status | +|--------|-----|------|--------| +| `approval/gate.py` | ~48 | HITL gate logic | Will deepen with interactive mode | +| `reporting/builder.py` | ~79 | JSON serializer | Will deepen with SARIF + markdown | +| `observer/exit_code.py` | ~49 | Tier-1 exit code capture | Stable, correctly small | +| `observer/base.py` | ~13 | Re-export | Consolidate into `observer/__init__.py` | +| `exceptions.py` | ~30 | Error hierarchy | Stable, correctly small | + +**Consolidation note:** Module boundaries align to pipeline stages — the right decomposition. Shallow modules are shallow because they haven't been built out, not because they're over-split. Only `observer/base.py` (13 LOC re-export) is a genuine consolidation candidate → fold into `observer/__init__.py`. + +--- + +## 6. Implementation Decisions Made + +| # | Decision | Rationale | +|---|----------|-----------| +| 1 | Legacy diff path will be retired | SAST pipeline is the target architecture | +| 2 | 1:1 finding-to-exploit cardinality | Deterministic baseline first; multi-variant is future work | +| 3 | INCONCLUSIVE means INCONCLUSIVE always | LLM score does not tip the verdict; needs empirical data first | +| 4 | Multi-CWE conflict → STATIC_ONLY wins | Conservative; don't detonate ambiguous findings | +| 5 | Egress blocked via `internal: true` compose network | No iptables needed; Docker-native | +| 6 | Seccomp: Docker default profile | Good baseline; custom profile is future work | +| 7 | `target_dns` validated against compose services | Cheap guard before expensive detonation | +| 8 | Interrupted detonation → INCONCLUSIVE | No silent re-detonation on resume | +| 9 | `--timeout` wires to `detonate()` | CLI flag must do what it says | +| 10 | Observer failures surfaced in report | Degraded evidence must be visible to reviewer | +| 11 | CWE map externalized to config | Extensible without code changes | +| 12 | Unmapped CWEs emit warning | Silent fallback to STATIC_ONLY is a gap | +| 13 | LLM refusal: retry once, then REFUSED state | Don't crash the scan; don't silently skip | +| 14 | Both HITL modes: async (CI) + interactive (dev) | Different workflows need different modes | +| 15 | Report formats: JSON + SARIF + markdown | Machines, CI, humans | +| 16 | Source context enrichment: agentic (LLM tool-call) | Deferred — requires multi-turn tool-use in intelligence/ | + +--- + +## 7. Testing Decisions and Test Boundaries + +### Unit test boundaries (per module, mocked dependencies) +- **Triage:** Given a Finding with known CWEs, assert correct verdict. Test multi-CWE conflict resolution. Test unmapped CWE warning emission. +- **Scoring:** Mock LLM provider. Assert prompt construction and response parsing. +- **Parser:** Test refusal detection against known refusal phrases. Test JSON fence stripping. Test malformed response handling. +- **Verdict synthesizer:** Given evidence lists, assert correct verdict. Test with missing observers (degraded evidence). +- **Ingest:** Parse known Semgrep JSON fixtures. Assert CWE extraction, severity normalization. +- **Approval gate:** Test auto-approve env var. Test interactive mode stdin (mock). + +### Integration test boundaries (real Docker, no LLM) +- **Sandbox isolation:** Launch attacker container, verify: no egress (ping external host fails), seccomp active, read-only FS, non-root UID, resource limits enforced. +- **Observer collection:** Detonate a known-good exploit against a test target. Assert all three observers return evidence. +- **Resume:** Start a scan, kill mid-detonation, resume. Assert interrupted findings are INCONCLUSIVE. +- **Concurrency:** Run two scans in parallel against different compose projects. Assert no container/network collisions. Verify SQLite WAL handles concurrent access. + +### What is NOT tested +- LLM output quality (non-deterministic; evaluated via accuracy benchmarks, not assertions) +- Semgrep itself (upstream tool; we test our parsing of its output) +- Docker internals (we test our orchestration of Docker, not Docker itself) + +--- + +## 8. Out of Scope + +| Item | Reason | +|------|--------| +| v0.3 patch generation / remediation output | Not implemented; specified in ADR-009 as v0.3 work (triple-verification patch loop) | +| LLM score influencing verdict | Deferred pending empirical accuracy data | +| Multi-variant exploit generation per finding | Future; 1:1 is the v0.2 model | +| Custom seccomp profiles | Docker default is sufficient for v0.2 | +| Legacy diff path improvements | Being retired | +| `ScoredFinding` dataclass usage in memory | Score lives in SQLite only; in-memory model not needed until score influences verdict | +| Agentic source context enrichment | Requires multi-turn tool-use refactor of intelligence/ | +| Multiple non-DB target service detection | Heuristic returns first match; multi-service targeting is future work | diff --git a/shield-claw/requirements-dev.txt b/shield-claw/requirements-dev.txt index 0f8b8c4..d4dfef4 100644 --- a/shield-claw/requirements-dev.txt +++ b/shield-claw/requirements-dev.txt @@ -3,5 +3,6 @@ pre-commit>=4.0 pytest>=8.0 pytest-cov>=5.0 pytest-mock>=3.14 +pytest-timeout>=0.5 ruff>=0.9.0 mypy>=1.11 diff --git a/shield-claw/src/shieldclaw/context/aggregator.py b/shield-claw/src/shieldclaw/context/aggregator.py index 818cd0d..5ce05db 100644 --- a/shield-claw/src/shieldclaw/context/aggregator.py +++ b/shield-claw/src/shieldclaw/context/aggregator.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import os import subprocess from datetime import UTC, datetime from pathlib import Path @@ -135,12 +136,16 @@ def _git_diff_head_minus_one(self, root: Path) -> str: """ command = ["git", "-C", str(root), "diff", "HEAD~1"] _LOG.debug("Running command: %s", command) + # Strip GIT_* env vars so the subprocess always reads the repository + # at ``root``, not a GIT_DIR inherited from an outer git worktree. + clean_env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} try: completed = subprocess.run( command, capture_output=True, text=True, timeout=self._git_timeout, + env=clean_env, ) except FileNotFoundError as exc: raise AggregationError("git executable not found on PATH.") from exc diff --git a/shield-claw/src/shieldclaw/sandbox/docker_orchestrator.py b/shield-claw/src/shieldclaw/sandbox/docker_orchestrator.py index 910fc63..d6ff756 100644 --- a/shield-claw/src/shieldclaw/sandbox/docker_orchestrator.py +++ b/shield-claw/src/shieldclaw/sandbox/docker_orchestrator.py @@ -1,8 +1,20 @@ -"""Manage compose-backed targets and locked-down attacker containers for detonation. - -Compose services receive ``shieldclaw.run`` labels via a generated override file so -engines that lack ``docker update --label-add`` (common on Windows Desktop) stay -compatible while still meeting labeling requirements. +""" +File: shield-claw/src/shieldclaw/sandbox/docker_orchestrator.py +Purpose: Manage compose-backed targets and locked-down attacker containers for detonation. +Public API: +- DockerOrchestrator(...) -> DockerOrchestrator +- compose_project_name(result_id: str) -> str +- label_override_path(compose_file: Path, result_id: str) -> Path +- compose_default_network(result_id: str) -> str +- compose_up_timeout_seconds() -> float +- resolve_compose_start_wait_seconds(local_default: float) -> float +Depends On: +- shieldclaw.exceptions (DetonationError, DockerNotAvailableError, SandboxStartError) +- shieldclaw.models (DetonationObserver, DetonationOutcome, ExploitPayload, ObserverEvidence) +Used By: +- shield-claw/src/shieldclaw/orchestrator.py +Use Cases: +- UC-DETONATE: Start compose targets, run exploits in isolated containers, teardown stacks. """ from __future__ import annotations @@ -27,10 +39,39 @@ _LOG = logging.getLogger(__name__) _DOCKER_INFO_TIMEOUT = 15.0 -_COMPOSE_UP_TIMEOUT = 120.0 +_COMPOSE_UP_TIMEOUT_DEFAULT = 120.0 _START_POLL_INTERVAL = 2.0 _START_WAIT_SECONDS = 120.0 + +def _env_float(name: str, default: float) -> float: + raw = os.environ.get(name) + if raw is None or not str(raw).strip(): + return default + try: + return float(raw) + except ValueError: + _LOG.warning("Ignoring invalid float for %s=%r; using default %s", name, raw, default) + return default + + +def compose_up_timeout_seconds() -> float: + """Return timeout for ``docker compose up`` / ``down`` subprocess calls. + + Override with ``SHIELDCLAW_COMPOSE_UP_TIMEOUT_SECONDS`` (e.g. slow CI runners). + """ + return _env_float("SHIELDCLAW_COMPOSE_UP_TIMEOUT_SECONDS", _COMPOSE_UP_TIMEOUT_DEFAULT) + + +def resolve_compose_start_wait_seconds(local_default: float) -> float: + """Return readiness polling budget after compose ``up``. + + Reads ``SHIELDCLAW_COMPOSE_START_TIMEOUT`` (same variable as ``_compose_start_timeout``). + When unset, falls back to ``local_default``. + """ + return _env_float("SHIELDCLAW_COMPOSE_START_TIMEOUT", local_default) + + # Default tag for the pre-built attacker image. Override via # SHIELDCLAW_ATTACKER_IMAGE to pin a different version or registry. _DETONATE_IMAGE_DEFAULT = "ghcr.io/blondres04/shieldclaw-attacker:0.1" @@ -108,7 +149,7 @@ def __init__( to ``120`` seconds when the variable is unset. start_poll_interval: Sleep interval between readiness probes. post_up_grace_seconds: Extra sleep after healthcheck gating completes. - Reduced from 10 s to 2 s now that readiness is healthcheck-gated. + Reduced from 10 s to 2 s now that readiness is healthcheck-gating. """ self._start_wait = ( _compose_start_timeout() if start_wait_seconds is None else start_wait_seconds @@ -141,7 +182,7 @@ def start_sandbox(self, compose_path: str, result_id: str) -> None: self._run_required( up_cmd, cwd=cwd, - timeout=_COMPOSE_UP_TIMEOUT, + timeout=compose_up_timeout_seconds(), error_cls=SandboxStartError, error_prefix="docker compose up failed", ) @@ -346,7 +387,7 @@ def teardown(self, compose_path: str, result_id: str) -> None: override = label_override_path(compose_file, result_id) down_cmd = self._compose_command_prefix(compose_file, override, project) + ["down", "-v"] try: - self._run_optional(down_cmd, cwd=cwd, timeout=_COMPOSE_UP_TIMEOUT) + self._run_optional(down_cmd, cwd=cwd, timeout=compose_up_timeout_seconds()) except Exception as exc: # noqa: BLE001 - best-effort teardown _LOG.warning("docker compose down failed: %s", exc) diff --git a/shield-claw/tests/test_aggregator.py b/shield-claw/tests/test_aggregator.py index d17369f..2409a93 100644 --- a/shield-claw/tests/test_aggregator.py +++ b/shield-claw/tests/test_aggregator.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os import subprocess import textwrap @@ -131,6 +132,10 @@ def test_git_diff_head_minus_one(tmp_path) -> None: root.mkdir() def run_git(args: list[str]) -> None: + # Strip GIT_* env vars so a parent git worktree's GIT_DIR does not + # bleed into the subprocess and cause "this operation must be run in + # a work tree" failures on Windows. + clean_env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} subprocess.run( args, cwd=root, @@ -138,6 +143,7 @@ def run_git(args: list[str]) -> None: capture_output=True, text=True, timeout=30.0, + env=clean_env, ) run_git(["git", "init", "-b", "main"]) diff --git a/shield-claw/tests/test_docker_orchestrator.py b/shield-claw/tests/test_docker_orchestrator.py index e94f76c..0f184dc 100644 --- a/shield-claw/tests/test_docker_orchestrator.py +++ b/shield-claw/tests/test_docker_orchestrator.py @@ -16,7 +16,9 @@ DockerOrchestrator, compose_default_network, compose_project_name, + compose_up_timeout_seconds, label_override_path, + resolve_compose_start_wait_seconds, ) @@ -269,3 +271,20 @@ def test_detonate_raises_on_docker_client_error(mocker: MockerFixture) -> None: orch = DockerOrchestrator() with pytest.raises(DetonationError): orch.detonate(payload, "net", "rid", timeout=5) + + +def test_resolve_compose_start_wait_seconds_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SHIELDCLAW_COMPOSE_START_TIMEOUT", "240") + assert resolve_compose_start_wait_seconds(120.0) == 240.0 + + +def test_resolve_compose_start_wait_seconds_invalid_env_uses_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SHIELDCLAW_COMPOSE_START_TIMEOUT", "bogus") + assert resolve_compose_start_wait_seconds(85.5) == 85.5 + + +def test_compose_up_timeout_seconds_env_override(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SHIELDCLAW_COMPOSE_UP_TIMEOUT_SECONDS", "999") + assert compose_up_timeout_seconds() == 999.0 diff --git a/shield-claw/tests/test_docker_orchestrator_concurrency.py b/shield-claw/tests/test_docker_orchestrator_concurrency.py index a830199..7f71a72 100644 --- a/shield-claw/tests/test_docker_orchestrator_concurrency.py +++ b/shield-claw/tests/test_docker_orchestrator_concurrency.py @@ -21,7 +21,10 @@ import pytest -from shieldclaw.sandbox.docker_orchestrator import DockerOrchestrator +from shieldclaw.sandbox.docker_orchestrator import ( + DockerOrchestrator, + resolve_compose_start_wait_seconds, +) _MINIMAL_COMPOSE = """\ services: @@ -68,11 +71,17 @@ def test_concurrent_runs_do_not_interfere(tmp_path: Path) -> None: result_id_a = str(uuid.uuid4()) result_id_b = str(uuid.uuid4()) + start_budget = resolve_compose_start_wait_seconds(120.0) + thread_wait = start_budget + 180.0 + join_budget = max(900.0, 3.0 * start_budget + 240.0) + orch_a = DockerOrchestrator( + start_wait_seconds=start_budget, start_poll_interval=1.0, post_up_grace_seconds=0.0, ) orch_b = DockerOrchestrator( + start_wait_seconds=start_budget, start_poll_interval=1.0, post_up_grace_seconds=0.0, ) @@ -86,8 +95,8 @@ def run_a() -> None: orch_a.start_sandbox(str(dir_a / "docker-compose.yml"), result_id_a) a_sandbox_ready.set() # Wait for B to perform its startup (which includes _cleanup_stale). - if not b_cleanup_done.wait(timeout=60): - errors.append(TimeoutError("Thread B did not signal within 60 s")) + if not b_cleanup_done.wait(timeout=thread_wait): + errors.append(TimeoutError("Thread B did not signal within expected time")) return # A's containers must still be alive after B's cleanup. containers = _containers_for_run(result_id_a) @@ -105,8 +114,10 @@ def run_a() -> None: def run_b() -> None: try: - if not a_sandbox_ready.wait(timeout=120): - errors.append(TimeoutError("Thread A did not start sandbox within 120 s")) + if not a_sandbox_ready.wait(timeout=thread_wait): + errors.append( + TimeoutError(f"Thread A did not start sandbox within {thread_wait} s") + ) return # start_sandbox calls _cleanup_stale(result_id_b) — must not touch A. orch_b.start_sandbox(str(dir_b / "docker-compose.yml"), result_id_b) @@ -130,8 +141,8 @@ def run_b() -> None: thread_a.start() thread_b.start() - thread_a.join(timeout=300) - thread_b.join(timeout=300) + thread_a.join(timeout=join_budget) + thread_b.join(timeout=join_budget) if errors: raise errors[0] diff --git a/shield-claw/tests/test_docker_orchestrator_integration.py b/shield-claw/tests/test_docker_orchestrator_integration.py index 476158e..ef9a99a 100644 --- a/shield-claw/tests/test_docker_orchestrator_integration.py +++ b/shield-claw/tests/test_docker_orchestrator_integration.py @@ -16,6 +16,7 @@ DockerOrchestrator, compose_default_network, compose_project_name, + resolve_compose_start_wait_seconds, ) _REPO_ROOT = Path(__file__).resolve().parents[2] @@ -89,6 +90,7 @@ def test_full_stack_detonate_and_teardown(integration_compose: Path) -> None: pytest.skip(f"docker compose build failed: {build.stderr}") orchestrator = DockerOrchestrator( + start_wait_seconds=resolve_compose_start_wait_seconds(120.0), start_poll_interval=2.0, post_up_grace_seconds=0.0, ) @@ -103,12 +105,12 @@ def test_full_stack_detonate_and_teardown(integration_compose: Path) -> None: execution_command="python -", language="python", ) - exit_code = orchestrator.detonate( + outcome = orchestrator.detonate( payload, network_name=network, result_id=result_id, timeout=60, ) - assert exit_code == 0 + assert outcome.exit_code == 0 finally: orchestrator.teardown(str(integration_compose), result_id) diff --git a/shield-claw/tests/test_sandbox_sealed_network.py b/shield-claw/tests/test_sandbox_sealed_network.py index 074cb59..a74264e 100644 --- a/shield-claw/tests/test_sandbox_sealed_network.py +++ b/shield-claw/tests/test_sandbox_sealed_network.py @@ -126,9 +126,10 @@ def test_pypi_unreachable_with_sealed_network() -> None: timeout=30.0, ) assert result.returncode == 0, f"Script failed unexpectedly:\n{result.stdout}\n{result.stderr}" - # Confirm the image works without internet — either PyPI was unreachable - # (sealed) or it was reachable but requests still worked from the image. - assert "requests" in result.stdout or "WARNING" in result.stdout + # Confirm the script reached one of the two expected branches: + # sealed network → "PyPI unreachable (expected): ..." + # open network → "WARNING: PyPI was reachable ..." + assert "unreachable" in result.stdout.lower() or "WARNING" in result.stdout @pytest.mark.integration