fix(ci): stop leaking vaults and add cleanup safety net

.github/workflows/cleanup-ci-vaults.yml

@@ -0,0 +1,70 @@
+name: cleanup-ci-vaults
+
+# Manual only. Prefer deleting a single run prefix; optional orphan-defaults for backlog.
+on:
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "prefix = one workflow run; orphan-defaults = drain leaked name=default vaults"
+        required: true
+        default: "prefix"
+        type: choice
+        options:
+          - "prefix"
+          - "orphan-defaults"
+      prefix:
+        description: "Required for mode=prefix (e.g. ci-32862931541-execution)"
+        required: false
+        type: string
+      min_age_hours:
+        description: "For orphan-defaults: only delete default vaults older than this many hours"
+        required: false
+        default: "2"
+        type: string
+      dry_run:
+        description: "List vaults without deleting"
+        required: false
+        default: "false"
+        type: choice
+        options:
+          - "false"
+          - "true"
+
+jobs:
+  cleanup:
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    timeout-minutes: 30
+    env:
+      NOTTE_API_KEY: ${{ secrets.NOTTE_API_KEY }}
+      DISABLE_TELEMETRY: "true"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: ".python-version"
+
+      - name: Install notte-sdk
+        run: uv pip install --system notte-sdk
+
+      - name: Cleanup vaults
+        run: |
+          ARGS=()
+          if [ "${{ inputs.dry_run }}" = "true" ]; then
+            ARGS+=(--dry-run)
+          fi
+          if [ "${{ inputs.mode }}" = "orphan-defaults" ]; then
+            ARGS+=(--orphan-defaults --min-age-hours "${{ inputs.min_age_hours }}")
+          else
+            if [ -z "${{ inputs.prefix }}" ]; then
+              echo "prefix is required when mode=prefix"
+              exit 1
+            fi
+            ARGS+=(--prefix "${{ inputs.prefix }}")
+          fi
+          python scripts/cleanup_ci_vaults.py "${ARGS[@]}"

.github/workflows/docs-tests-cicd.yml

@@ -79,6 +79,8 @@ jobs:
       NOTTE_GITHUB_COM_EMAIL: ${{ secrets.NOTTE_GITHUB_COM_EMAIL }}
       NOTTE_GITHUB_COM_PASSWORD: ${{ secrets.NOTTE_GITHUB_COM_PASSWORD }}
       NOTTE_GITHUB_COM_MFA_SECRET: ${{ secrets.NOTTE_GITHUB_COM_MFA_SECRET }}
+      # Scope vault creates to this job only; cleanup deletes only this prefix.
+      NOTTE_CI_VAULT_PREFIX: ci-${{ github.run_id }}-${{ github.job }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -112,3 +114,9 @@ jobs:
 
       - name: Run execution tests
         run: cd docs/src && uv run pytest -v --tb=no
+
+      - name: Cleanup vaults created by this run
+        if: always()
+        run: |
+          source .venv/bin/activate
+          python scripts/cleanup_ci_vaults.py

.github/workflows/nightly-examples.yml

@@ -28,6 +28,7 @@ env:
   CREDIT_CARD_NUMBER: 4242424242424242
   CREDIT_CARD_CVV: 777
   CREDIT_CARD_EXPIRATION: "05/28"
+  NOTTE_CI_VAULT_PREFIX: ci-${{ github.run_id }}-${{ github.job }}
 
 jobs:
   example-tests:
@@ -83,6 +84,10 @@ jobs:
         with:
             run: bash tests/run_examples.sh
 
+      - name: Cleanup vaults created by this run
+        if: always()
+        run: uv run python scripts/cleanup_ci_vaults.py
+
       - name: Send Slack Notification
         uses: slackapi/slack-github-action@v1.24.0
         with:

.github/workflows/test-cicd.yml

@@ -38,6 +38,8 @@ env:
   # True for push, manual dispatch, and same-repo PRs; false for fork PRs. Steps that
   # need secrets are gated on this so fork PRs skip them cleanly instead of failing.
   IS_TRUSTED: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
+  # Scope vault creates to this workflow job; cleanup deletes only this prefix.
+  NOTTE_CI_VAULT_PREFIX: ci-${{ github.run_id }}-${{ github.job }}
 
 permissions:
   pull-requests: write
@@ -164,6 +166,10 @@ jobs:
           set -o pipefail
           uv run pytest -n logical tests --ignore=tests/examples/test_examples.py --ignore=tests/examples/test_readme.py --durations=10 --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=packages | tee pytest-coverage.txt
 
+      - name: Cleanup vaults created by this run
+        if: ${{ always() && env.IS_TRUSTED == 'true' }}
+        run: uv run python scripts/cleanup_ci_vaults.py
+
       - name: Pytest coverage comment
         if: ${{ always() && github.ref != 'refs/heads/main' && env.IS_TRUSTED == 'true' }}
         uses: MishaKav/pytest-coverage-comment@main

docs/src/conftest.py

@@ -0,0 +1,33 @@
+"""Docs snippet test hooks.
+
+Installs CI vault scoping when ``NOTTE_CI_VAULT_PREFIX`` is set so this workflow
+run names vaults under that prefix. Deletion is done by the workflow
+``Cleanup vaults created by this run`` step (not here) so pytest-xdist workers
+cannot delete vaults still in use by sibling workers.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+
+def _load_ci_vault_scope() -> Any | None:
+    if not os.environ.get("NOTTE_CI_VAULT_PREFIX"):
+        return None
+    path = Path(__file__).resolve().parents[2] / "scripts" / "ci_vault_scope.py"
+    spec = importlib.util.spec_from_file_location("ci_vault_scope", path)
+    if spec is None or spec.loader is None:
+        return None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["ci_vault_scope"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+_SCOPE = _load_ci_vault_scope()
+if _SCOPE is not None:
+    _SCOPE.install()

docs/src/snippets/getting-started/concept_vault.mdx

@@ -2,8 +2,8 @@
 {/* @sniptest testers/getting-started/concept_vault.py */}
 
 ```python concept_vault.py
-vault = client.Vault()
-vault.add_credentials(url="https://github.com", email="...", password="...")
-agent = client.Agent(session=session, vault=vault)
-agent.run(task="Login to GitHub")
+with client.Session() as session, client.Vault() as vault:
+    vault.add_credentials(url="https://github.com", email="...", password="...")
+    agent = client.Agent(session=session, vault=vault)
+    agent.run(task="Login to GitHub")
 ```

docs/src/testers/getting-started/concept_vault.py

@@ -1,10 +1,9 @@
 # @sniptest filename=concept_vault.py
-# @sniptest show=5-9
+# @sniptest show=4-7
 from notte_sdk import NotteClient
 
 client = NotteClient()
-with client.Session() as session:
-    vault = client.Vault()
+with client.Session() as session, client.Vault() as vault:
     vault.add_credentials(url="https://github.com", email="...", password="...")
     agent = client.Agent(session=session, vault=vault)
     agent.run(task="Login to GitHub")

docs/src/tests/test_snippets.py

@@ -172,12 +172,21 @@ def decorator(func: Callable[[EvalExample, str], Any]):
 
 
 @handle_file("vaults/index.py")
-def handle_vault(
+def handle_vault_index(
     eval_example: EvalExample,
     code: str,
 ) -> None:
-    code = code.replace("<your-mfa-secret>", "JBSWY3DPEHPK3PXP")
-    run_example(eval_example, code=code)
+    if FAST_MODE or TYPE_CHECK_MODE:
+        # Syntax/type check - don't create client
+        code = code.replace("<your-mfa-secret>", "JBSWY3DPEHPK3PXP").replace("my_vault_id", "placeholder-vault-id")
+        run_example(eval_example, code=code)
+    else:
+        # Full mode: create real vault and always delete it via context manager
+        _ = load_dotenv()
+        client = NotteClient()
+        with client.Vault() as vault:
+            code = code.replace("<your-mfa-secret>", "JBSWY3DPEHPK3PXP").replace("my_vault_id", vault.vault_id)
+            run_example(eval_example, code=code)
 
 
 @handle_file("agents/index.py")
@@ -236,22 +245,18 @@ def handle_workflow_fork(
     run_example(eval_example, code=code)
 
 
-@handle_file("vaults/index.py")
-def handle_vault_index(
+@handle_file("getting-started/concept_vault.py")
+def handle_concept_vault(
     eval_example: EvalExample,
     code: str,
 ) -> None:
-    if FAST_MODE or TYPE_CHECK_MODE:
-        # Syntax/type check - don't create client
-        code = code.replace("<your-mfa-secret>", "JBSWY3DPEHPK3PXP").replace("my_vault_id", "placeholder-vault-id")
-        run_example(eval_example, code=code)
-    else:
-        # Full mode: create real vault
-        _ = load_dotenv()
-        client = NotteClient()
-        with client.Vault() as vault:
-            code = code.replace("<your-mfa-secret>", "JBSWY3DPEHPK3PXP").replace("my_vault_id", vault.vault_id)
-            run_example(eval_example, code=code)
+    """Cap agent steps in execution mode; vault cleanup uses `with client.Vault()`."""
+    if not (FAST_MODE or TYPE_CHECK_MODE):
+        code = code.replace(
+            "agent = client.Agent(session=session, vault=vault)",
+            "agent = client.Agent(session=session, vault=vault, max_steps=1)",
+        )
+    run_example(eval_example, code=code)
 
 
 @handle_file("sessions/file_storage_basic.py")

examples/landing-examples/landing_examples.py

@@ -26,31 +26,35 @@
 def main():
     client = NotteClient(api_key=os.getenv("NOTTE_API_KEY"))
 
-    for task, url, vault in landing_examples[3:]:
+    for task, url, use_vault in landing_examples[3:]:
         with client.Session() as session:
-            if vault:
-                vault = client.Vault()
-                email = os.getenv("NOTTE_VAULT_TEST_EMAIL")
-                assert email is not None
-                pwd = os.getenv("NOTTE_VAULT_TEST_PASSWORD")
-                assert pwd is not None
-                _ = vault.add_credentials(
-                    url="https://google.com",
-                    email=email,
-                    password=pwd,
-                )
+            if use_vault:
+                with client.Vault() as vault:
+                    email = os.getenv("NOTTE_VAULT_TEST_EMAIL")
+                    assert email is not None
+                    pwd = os.getenv("NOTTE_VAULT_TEST_PASSWORD")
+                    assert pwd is not None
+                    _ = vault.add_credentials(
+                        url="https://google.com",
+                        email=email,
+                        password=pwd,
+                    )
+                    agent = client.Agent(
+                        session=session,
+                        reasoning_model="vertex_ai/gemini-2.0-flash",
+                        max_steps=15,
+                        vault=vault,
+                    )
+                    run_kwargs = {"task": task, **({"url": url} if url is not None else {})}
+                    response = agent.run(**run_kwargs)
             else:
-                vault = None
-
-            agent_kwargs = {
-                "session": session,
-                "reasoning_model": "vertex_ai/gemini-2.0-flash",
-                "max_steps": 15,
-                **({"vault": vault} if vault is not None else {}),
-            }
-            agent = client.Agent(**agent_kwargs)
-            run_kwargs = {"task": task, **({"url": url} if url is not None else {})}
-            response = agent.run(**run_kwargs)
+                agent = client.Agent(
+                    session=session,
+                    reasoning_model="vertex_ai/gemini-2.0-flash",
+                    max_steps=15,
+                )
+                run_kwargs = {"task": task, **({"url": url} if url is not None else {})}
+                response = agent.run(**run_kwargs)
 
             if not response.success:
                 exit(-1)