feat: isolate and attest OpenShell jobs

configs/config_openshell.yml

@@ -1,7 +1,5 @@
-# EXPERIMENTAL: AI-Q deep research over a local, single-operator OpenShell sandbox.
-# Run ./scripts/setup_openshell.sh first to provision the gateway and named sandbox.
-# Jobs attach to that shared sandbox; per-job directories prevent filename collisions
-# but are not a security boundary. Do not run mutually untrusted jobs concurrently.
+# EXPERIMENTAL: AI-Q deep research with one policy-bound OpenShell sandbox per job.
+# Run ./scripts/setup_openshell.sh first to provision the gateway, image, and policy.
 
 general:
   use_uvloop: true
@@ -141,13 +139,30 @@ functions:
   deep_research_sandbox:
     _type: deep_research_sandbox
     provider: openshell
-    sandbox_name: ${AIQ_OPENSHELL_SANDBOX_NAME:-aiq-openshell-demo}
+    openshell_image: ${AIQ_OPENSHELL_IMAGE:-aiq-openshell-demo:latest}
     policy: ${AIQ_OPENSHELL_POLICY_FILE:-configs/openshell/generated/aiq-openshell-policy.yaml}
     workdir: /sandbox
-    network: blocked
+    # The OpenShell policy is authoritative. This declaration is an upper bound:
+    # startup fails if the policy grants a hostname not listed here.
+    network: allowlist
+    network_allow:
+      - api.github.com
+      - github.com
+      - integrate.api.nvidia.com
+      - api.tavily.com
+      - google.serper.dev
+      - pypi.org
+      - files.pythonhosted.org
+      - huggingface.co
+      - cdn-lfs.huggingface.co
+      - export.arxiv.org
+      - api.semanticscholar.org
+      - registry.npmjs.org
     timeout: 1200
     idle_timeout: 1800
-    delete_on_exit: false
+    delete_on_exit: true
+    attest: true
+    require_hard_landlock: true
     artifact_capture:
       enabled: true
       max_file_bytes: 50000000

configs/openshell/aiq-research-policy.yaml

@@ -21,11 +21,12 @@ filesystem_policy:
     - /tmp
     - /dev/null
 
-# best_effort lets the sandbox start on hosts without Landlock (e.g. Docker Desktop on
-# macOS), but there filesystem confinement is silently dropped. Acceptable only for the
-# local single-operator demo; production must use `hard_requirement` (fail closed).
+# Fail closed when the host cannot enforce Landlock filesystem confinement. Local demo
+# environments that intentionally accept weaker isolation must generate a separate policy
+# with `scripts/setup_openshell.sh --landlock-compatibility best_effort` and configure
+# `require_hard_landlock: false` explicitly.
 landlock:
-  compatibility: best_effort
+  compatibility: hard_requirement
 
 process:
   run_as_user: sandbox

docs/source/architecture/agents/sandbox.md

@@ -6,9 +6,9 @@ SPDX-License-Identifier: Apache-2.0
 # Deep Research Sandbox Notes
 
 Deep research can optionally run DeepAgents `execute` calls through a sandbox provider
-(Modal, OpenShell, or any registered provider). Modal creates a sandbox per job. The
-experimental OpenShell path attaches to a pre-created named sandbox shared by its jobs;
-job-scoped directories prevent filename collisions but are not a security boundary.
+(Modal, OpenShell, or any registered provider). Modal and OpenShell create a fresh physical
+sandbox per job. OpenShell binds the configured policy at creation and attests readiness plus
+the loaded policy revision before exposing the execution backend.
 
 The sandbox is an internal execution detail. There are no sandbox-specific API
 endpoints, and job-level auth remains responsible for submit, stream, status,
@@ -21,12 +21,13 @@ runtime (`.../job/{job_id}/artifacts`), which is also auth-scoped to the job.
 
 ## Current Behavior
 
-- Modal uses one sandbox per deep research job. OpenShell currently attaches jobs to
-  the configured shared sandbox name and is intended for local, single-operator testing.
+- Modal and OpenShell use one physical sandbox per deep research job. OpenShell shared
+  attachment is available only through an explicit debug-only opt-in and is not job-isolated.
 - Synchronous sandbox-enabled runs use an internal per-agent runtime ID.
 - Providers are selected by config (`sandbox.provider` + `providers.<name>`); the
   provider is validated against the registry and gated by its declared capabilities.
-  OpenShell policy is provisioned externally and is not verified when AI-Q attaches.
+  OpenShell policy YAML is parsed strictly against the installed SDK schema, applied in the
+  job's creation spec, checked against the declared network upper bound, and attested before use.
 - Job IDs must satisfy each provider's object-name rules (Modal: 64 chars or fewer,
   alphanumeric plus dash/period/underscore).
 - `timeout` bounds individual execution. Other lifecycle controls are provider-dependent.
@@ -36,13 +37,13 @@ runtime (`.../job/{job_id}/artifacts`), which is also auth-scoped to the job.
 
 ## Operational Notes
 
-- High-concurrency Modal runs create one sandbox per job. OpenShell runs share the named
-  sandbox and must not be used concurrently for mutually untrusted jobs. Optional submit-path
+- High-concurrency Modal and OpenShell runs create one sandbox per job. Optional submit-path
   caps (`AIQ_MAX_SANDBOXES_PER_PRINCIPAL` / `AIQ_MAX_SANDBOXES_GLOBAL`, default-off) bound
-  concurrency/cost but do not provide filesystem isolation.
+  concurrency and cost.
 - Custom client-supplied job IDs must not be reused for a new job.
 - The runtime closes provider sessions on success, failure, cancellation, and timeout.
-  A named OpenShell sandbox persists when `delete_on_exit` is disabled.
+- Per-job OpenShell mode requires `delete_on_exit: true`. A persistent shared sandbox is
+  possible only through the explicit debug attachment settings.
 
 ## Current Safeguards
 
@@ -54,3 +55,4 @@ The following safeguards are in place:
   with MIME-from-bytes spoof rejection, SVG sanitization, and an inline-render allowlist.
 - Sandbox quota and concurrency controls, and artifact retention via job-expiry cleanup.
 - Structured lifecycle logging for sandbox create, reuse, failure, and cleanup.
+- Structured `sandbox.attestation` and `sandbox.cleanup` events.

frontends/aiq_api/src/aiq_api/jobs/runner.py

@@ -638,21 +638,31 @@ async def run_agent_job(
 
     finally:
         # Ensure terminal-path events are not left in the batch buffer.
-        if event_store is not None and hasattr(event_store, "flush"):
-            event_store.flush()
+        await _flush_event_store(event_store, job_id=job_id)
         if cancellation_monitor:
             cancellation_monitor.stop()
         # Release the sandbox off the event loop so the SDK session close never blocks the Dask
         # worker. The single artifact harvest already ran in agent.run() before this point, so
         # teardown only closes/terminates; interrupted jobs terminate() to preempt a live execute.
         await asyncio.to_thread(_teardown_sandbox, sandbox_runtime, job_id=job_id, interrupted=interrupted)
+        await _flush_event_store(event_store, job_id=job_id)
         # Clean up job-scoped auth token
         if _auth_token_reset is not None:
             from ._auth_context import job_auth_token
 
             job_auth_token.reset(_auth_token_reset)
 
 
+async def _flush_event_store(event_store: Any | None, *, job_id: str) -> None:
+    """Flush terminal events off-loop without replacing the job result."""
+    if event_store is None or not hasattr(event_store, "flush"):
+        return
+    try:
+        await asyncio.to_thread(event_store.flush)
+    except Exception as exc:  # noqa: BLE001 - terminal observability must not replace the job result
+        logger.warning("Event store flush failed for job %s (%s)", job_id, type(exc).__name__)
+
+
 def _teardown_sandbox(sandbox_runtime: Any | None, *, job_id: str, interrupted: bool) -> None:
     """Release sandbox resources on a terminal path (best-effort, never raises).
 
@@ -662,6 +672,14 @@ def _teardown_sandbox(sandbox_runtime: Any | None, *, job_id: str, interrupted:
     """
     if sandbox_runtime is None:
         return
+    finalize = getattr(sandbox_runtime, "finalize", None)
+    if finalize is not None:
+        try:
+            if not finalize(interrupted=interrupted):
+                logger.warning("Sandbox cleanup reported failure for job %s", job_id)
+        except Exception:  # noqa: BLE001 - cleanup must never replace the job result
+            logger.warning("Sandbox cleanup failed for job %s", job_id, exc_info=True)
+        return
     teardown = getattr(sandbox_runtime, "terminate", None) if interrupted else None
     if teardown is None:
         teardown = getattr(sandbox_runtime, "close", None)

scripts/README.md

@@ -51,18 +51,18 @@ Starts the agent in CLI mode with browser-based authentication.
 
 ### `setup_openshell.sh` - OpenShell Sandbox Setup
 
-Sets up the experimental, local single-operator NVIDIA OpenShell path for AI-Q. Run this once before using
+Sets up the experimental NVIDIA OpenShell path for AI-Q. Run this once before using
 `configs/config_openshell.yml` with `start_cli.sh` or `start_e2e.sh`. It installs
 the `openshell` SDK and the `langchain-nvidia-openshell` adapter, starts/verifies
-the local OpenShell gateway, builds the sandbox image, generates a network policy,
-and creates the named sandbox `aiq-openshell-demo`. Inference is unaffected (it
-stays host-side, routed to NVIDIA Build); only generated code runs in the
-network-blocked sandbox.
+the local OpenShell gateway, builds the reusable sandbox image, and generates a network
+policy. AI-Q then creates, attests, and deletes a policy-bound physical sandbox for every
+job. Inference is unaffected (it stays host-side, routed to NVIDIA Build); only generated
+code runs in the sandbox.
 
-The generated configuration attaches all jobs to one named sandbox. Per-job directories
-avoid filename collisions but do not isolate mutually untrusted jobs, and AI-Q does not
-verify the provisioned policy when attaching. Do not treat this setup as a multi-tenant
-security boundary.
+Production setup defaults to `landlock.compatibility: hard_requirement`. For a local demo
+host that cannot enforce Landlock, pass `--landlock-compatibility best_effort` and explicitly
+set `require_hard_landlock: false` in the AI-Q config. The legacy named shared sandbox is
+created only with `--create-shared-debug-sandbox` and remains a debug-only, non-isolated mode.
 
 ```bash
 ./scripts/setup_openshell.sh --policy offline
@@ -81,6 +81,33 @@ Useful version examples:
 
 In the interactive version prompt, pressing Enter selects `0.0.72`.
 
+Deterministic live acceptance (no LLM or external research API involved):
+
+```bash
+.venv/bin/python scripts/smoke_openshell_isolation.py --gateway aiq-local
+```
+
+The probe fails unless the gateway service itself reports OpenShell `0.0.72`. It creates two
+jobs concurrently, proves they have distinct physical sandbox IDs and positive loaded policy
+revisions, cancels one, proves the other still executes, and then proves both were deleted.
+Success is four explicit lines:
+
+```text
+PASS gateway version: 0.0.72
+PASS distinct attested sandboxes: A=<name>@r<revision>, B=<name>@r<revision>
+PASS cancellation isolation: A deleted; B remained usable
+PASS terminal cleanup: both probe sandboxes deleted
+```
+
+On a non-production local host that intentionally uses the generated `best_effort` policy:
+
+```bash
+.venv/bin/python scripts/smoke_openshell_isolation.py \
+  --gateway aiq-local \
+  --policy configs/openshell/generated/aiq-openshell-policy.yaml \
+  --allow-best-effort-landlock
+```
+
 The setup installs the `openshell` SDK plus the official `langchain-nvidia-openshell`
 adapter (`OpenShellSandbox`), published on PyPI. The script installs it from PyPI by
 default; set `LANGCHAIN_NVIDIA_REPO` or pass `--langchain-nvidia` to use another
@@ -187,7 +214,7 @@ Starts both backend and frontend for full WebSocket support and HITL workflows.
 | `configs/config_web_frag.yml` | Server/E2E mode with Foundational RAG |
 | `configs/config_web_default_llamaindex.yml` | Server/E2E mode with LlamaIndex |
 | `configs/config_skills.yml` | Deep research with DeepAgents skills + Modal sandbox |
-| `configs/config_openshell.yml` | Experimental local single-operator OpenShell sandbox + artifact capture (run `setup_openshell.sh` first) |
+| `configs/config_openshell.yml` | Experimental per-job OpenShell sandbox + artifact capture (run `setup_openshell.sh` first) |
 
 ## Development Workflow
 

scripts/setup_openshell.sh

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Set up NVIDIA OpenShell for AI-Q with a named, policy-backed sandbox.
+# Set up NVIDIA OpenShell for AI-Q per-job policy-backed sandboxes.
 
 set -euo pipefail
 
@@ -48,6 +48,7 @@ SANDBOX_LOG_LEVEL="${AIQ_OPENSHELL_SANDBOX_LOG_LEVEL:-warn}"
 POLICY_PRESET="${AIQ_OPENSHELL_POLICY:-}"
 POLICY_ALLOWLIST="${AIQ_OPENSHELL_POLICY_ALLOWLIST:-${AIQ_OPENSHELL_POLICY_SERVICES:-}}"
 POLICY_FILE="${AIQ_OPENSHELL_POLICY_FILE:-$REPO_ROOT/configs/openshell/generated/aiq-openshell-policy.yaml}"
+LANDLOCK_COMPATIBILITY="${AIQ_OPENSHELL_LANDLOCK_COMPATIBILITY:-hard_requirement}"
 GATEWAY_NAME="${AIQ_OPENSHELL_GATEWAY_NAME:-aiq-local}"
 GATEWAY_PORT="${AIQ_OPENSHELL_GATEWAY_PORT:-8080}"
 DOCKER_BIN="${DOCKER_BIN:-}"
@@ -57,7 +58,7 @@ GATEWAY_ENDPOINT=""
 GATEWAY_DISABLE_TLS=false
 RESTART_GATEWAY=true
 BUILD_IMAGE=true
-CREATE_SANDBOX=true
+CREATE_SANDBOX=false
 LIST_OPENSHELL_VERSIONS=false
 
 SUPPORTED_SERVICES="github,pypi,nvidia,tavily,serper,huggingface,arxiv,semantic-scholar,npm"
@@ -76,7 +77,7 @@ Sets up OpenShell for AI-Q:
   6. Generates an initial OpenShell policy.
   7. Starts/verifies the OpenShell gateway.
   8. Builds the AI-Q sandbox image.
-  9. Creates a named policy-backed sandbox.
+  9. Leaves per-job sandbox creation to the AI-Q runtime.
 
 Options:
   --openshell-version VERSION   Exact OpenShell version, or "latest".
@@ -89,7 +90,9 @@ Options:
                                 Services: $SUPPORTED_SERVICES
   --policy-file PATH            Output policy file.
                                 Default: configs/openshell/generated/aiq-openshell-policy.yaml
-  --sandbox-name NAME           OpenShell sandbox name (default: aiq-openshell-demo).
+  --landlock-compatibility MODE hard_requirement (default) or best_effort (local demo only).
+  --create-shared-debug-sandbox Create one named shared sandbox for explicit debug attachment.
+  --sandbox-name NAME           Debug sandbox name (default: aiq-openshell-demo).
   --image-name NAME             Docker image tag (default: aiq-openshell-demo:latest).
   --sandbox-log-level LEVEL     In-container OpenShell log verbosity baked into the
                                 image via RUST_LOG (default: warn). Use "debug" to
@@ -102,7 +105,7 @@ Options:
   --gateway-bin PATH            OpenShell gateway launcher path.
   --no-restart-gateway          Reuse the active gateway instead of starting one.
   --skip-build                  Do not build the sandbox image.
-  --skip-sandbox                Do not create the named sandbox.
+  --skip-sandbox                Deprecated no-op; per-job creation is already the default.
   --list-policies               Print supported policy choices.
   --list-services               Print supported services for --allow.
   --list-openshell-versions     Print released OpenShell versions >= 0.0.72.
@@ -149,6 +152,14 @@ while [[ $# -gt 0 ]]; do
             POLICY_FILE="$2"
             shift 2
             ;;
+        --landlock-compatibility)
+            LANDLOCK_COMPATIBILITY="$2"
+            shift 2
+            ;;
+        --create-shared-debug-sandbox)
+            CREATE_SANDBOX=true
+            shift
+            ;;
         --sandbox-name)
             SANDBOX_NAME="$2"
             shift 2
@@ -864,6 +875,13 @@ resolve_policy() {
     if [[ "$POLICY_ALLOWLIST" == *offline* && "$POLICY_ALLOWLIST" != "offline" ]]; then
         fail "Use either offline or a service allowlist, not both: $POLICY_ALLOWLIST"
     fi
+    case "$LANDLOCK_COMPATIBILITY" in
+        hard_requirement|best_effort)
+            ;;
+        *)
+            fail "Unsupported Landlock compatibility '$LANDLOCK_COMPATIBILITY'; use hard_requirement or best_effort"
+            ;;
+    esac
 }
 
 validate_service() {
@@ -877,7 +895,7 @@ validate_service() {
 }
 
 emit_policy_header() {
-    cat >"$POLICY_FILE" <<'EOF'
+    cat >"$POLICY_FILE" <<EOF
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -891,7 +909,6 @@ filesystem_policy:
     - /usr
     - /lib
     - /etc
-    - /app
     - /var/log
     - /proc/self
     - /dev/urandom
@@ -901,12 +918,10 @@ filesystem_policy:
     - /tmp
     - /dev/null
 
-# best_effort lets the sandbox start on hosts without Landlock (e.g. Docker Desktop on
-# macOS), but on those hosts filesystem confinement is silently dropped. This is acceptable
-# only for the local single-operator demo; production must use `hard_requirement` so a
-# missing LSM fails closed instead of running unconfined.
+# hard_requirement is the production default: a missing Landlock LSM fails closed. Set
+# best_effort only for an explicit local demo that accepts loss of filesystem confinement.
 landlock:
-  compatibility: best_effort
+  compatibility: $LANDLOCK_COMPATIBILITY
 
 process:
   run_as_user: sandbox
@@ -991,6 +1006,7 @@ generate_policy() {
     echo "Policy file: $POLICY_FILE"
     echo "Policy: $POLICY_PRESET"
     echo "Allowed services: $POLICY_ALLOWLIST"
+    echo "Landlock compatibility: $LANDLOCK_COMPATIBILITY"
 }
 
 build_image() {
@@ -1042,7 +1058,7 @@ OpenShell is ready for AI-Q.
 
 Use these exports in shells where you run AI-Q:
 
-  export AIQ_OPENSHELL_SANDBOX_NAME="$SANDBOX_NAME"
+  export AIQ_OPENSHELL_IMAGE="$IMAGE_NAME"
   export AIQ_OPENSHELL_POLICY_FILE="$POLICY_FILE"
 
 Start CLI mode:
@@ -1055,7 +1071,7 @@ Start E2E mode:
 
 Useful cleanup:
 
-  $OPENSHELL_BIN sandbox delete "$SANDBOX_NAME"
+  $OPENSHELL_BIN sandbox list
   pkill -f openshell-gateway
 
 EOF