From 7d06067dc9e1c292023522f368f8999353d6016f Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Tue, 25 Aug 2026 16:56:14 +0200 Subject: [PATCH 1/5] fix(ci): stop leaking vaults and add cleanup safety net Docs/snippet tests were creating ephemeral vaults without reliable teardown, hitting the 1000 active-vault limit and failing vaults/create with HTTP 429. Co-authored-by: Cursor --- .github/workflows/cleanup-ci-vaults.yml | 55 +++++++++++ .github/workflows/docs-tests-cicd.yml | 13 +++ .github/workflows/nightly-examples.yml | 7 ++ .github/workflows/test-cicd.yml | 8 ++ .../getting-started/concept_vault.mdx | 8 +- docs/src/snippets/personas/index.mdx | 31 +++--- docs/src/snippets/vaults/manual.mdx | 36 +++---- .../testers/getting-started/concept_vault.py | 5 +- docs/src/testers/personas/index.py | 25 ++--- docs/src/testers/vaults/manual.py | 32 +++---- docs/src/tests/test_snippets.py | 37 ++++---- examples/landing-examples/landing_examples.py | 50 +++++----- scripts/cleanup_ci_vaults.py | 95 +++++++++++++++++++ tests/integration/sdk/test_vault.py | 20 ++-- 14 files changed, 304 insertions(+), 118 deletions(-) create mode 100644 .github/workflows/cleanup-ci-vaults.yml create mode 100644 scripts/cleanup_ci_vaults.py diff --git a/.github/workflows/cleanup-ci-vaults.yml b/.github/workflows/cleanup-ci-vaults.yml new file mode 100644 index 000000000..118ab4840 --- /dev/null +++ b/.github/workflows/cleanup-ci-vaults.yml @@ -0,0 +1,55 @@ +name: cleanup-ci-vaults + +on: + workflow_dispatch: + inputs: + include_persona: + description: "Also delete persona-owned vaults" + required: false + default: "false" + type: choice + options: + - "false" + - "true" + dry_run: + description: "List vaults without deleting" + required: false + default: "false" + type: choice + options: + - "false" + - "true" + +jobs: + cleanup: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + env: + NOTTE_API_KEY: ${{ secrets.NOTTE_API_KEY }} + # Match docs execution tests (default API host used by notte-sdk). + DISABLE_TELEMETRY: "true" + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: ".python-version" + + - name: Install notte-sdk + run: uv pip install --system notte-sdk + + - name: Cleanup leaked vaults + run: | + ARGS=() + if [ "${{ inputs.dry_run }}" = "true" ]; then + ARGS+=(--dry-run) + fi + if [ "${{ inputs.include_persona }}" = "true" ]; then + ARGS+=(--include-persona) + fi + python scripts/cleanup_ci_vaults.py "${ARGS[@]}" diff --git a/.github/workflows/docs-tests-cicd.yml b/.github/workflows/docs-tests-cicd.yml index 17b5c339b..e2a42dc71 100644 --- a/.github/workflows/docs-tests-cicd.yml +++ b/.github/workflows/docs-tests-cicd.yml @@ -110,5 +110,18 @@ jobs: fi echo "Environment variables are set" + - name: Cleanup leaked vaults (before) + run: | + # Docs execution creates ephemeral vaults; cancelled/failed runs leak them until + # the account hits the active-vault limit (HTTP 429 on vaults/create). + source .venv/bin/activate + python scripts/cleanup_ci_vaults.py + - name: Run execution tests run: cd docs/src && uv run pytest -v --tb=no + + - name: Cleanup leaked vaults (after) + if: always() + run: | + source .venv/bin/activate + python scripts/cleanup_ci_vaults.py diff --git a/.github/workflows/nightly-examples.yml b/.github/workflows/nightly-examples.yml index 3a832aed0..d2853b5a6 100644 --- a/.github/workflows/nightly-examples.yml +++ b/.github/workflows/nightly-examples.yml @@ -78,11 +78,18 @@ jobs: fi echo "Environment variables are set" + - name: Cleanup leaked vaults (before) + run: uv run python scripts/cleanup_ci_vaults.py + - name: Run example tests uses: coactions/setup-xvfb@v1 with: run: bash tests/run_examples.sh + - name: Cleanup leaked vaults (after) + if: always() + run: uv run python scripts/cleanup_ci_vaults.py + - name: Send Slack Notification uses: slackapi/slack-github-action@v1.24.0 with: diff --git a/.github/workflows/test-cicd.yml b/.github/workflows/test-cicd.yml index f38506938..82e9f030e 100644 --- a/.github/workflows/test-cicd.yml +++ b/.github/workflows/test-cicd.yml @@ -158,12 +158,20 @@ jobs: fi echo "Environment variables are set" + - name: Cleanup leaked vaults (before) + if: ${{ env.IS_TRUSTED == 'true' }} + run: uv run python scripts/cleanup_ci_vaults.py + - name: Run unit tests if: ${{ env.IS_TRUSTED == 'true' }} run: | set -o pipefail uv run pytest -n logical tests --ignore=tests/examples/test_examples.py --ignore=tests/examples/test_readme.py --durations=10 --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=packages | tee pytest-coverage.txt + - name: Cleanup leaked vaults (after) + if: ${{ always() && env.IS_TRUSTED == 'true' }} + run: uv run python scripts/cleanup_ci_vaults.py + - name: Pytest coverage comment if: ${{ always() && github.ref != 'refs/heads/main' && env.IS_TRUSTED == 'true' }} uses: MishaKav/pytest-coverage-comment@main diff --git a/docs/src/snippets/getting-started/concept_vault.mdx b/docs/src/snippets/getting-started/concept_vault.mdx index c64fce05c..892bfb9b3 100644 --- a/docs/src/snippets/getting-started/concept_vault.mdx +++ b/docs/src/snippets/getting-started/concept_vault.mdx @@ -2,8 +2,8 @@ {/* @sniptest testers/getting-started/concept_vault.py */} ```python concept_vault.py -vault = client.Vault() -vault.add_credentials(url="https://github.com", email="...", password="...") -agent = client.Agent(session=session, vault=vault) -agent.run(task="Login to GitHub") +with client.Session() as session, client.Vault() as vault: + vault.add_credentials(url="https://github.com", email="...", password="...") + agent = client.Agent(session=session, vault=vault) + agent.run(task="Login to GitHub") ``` diff --git a/docs/src/snippets/personas/index.mdx b/docs/src/snippets/personas/index.mdx index d7dabdf81..66b7fbe5f 100644 --- a/docs/src/snippets/personas/index.mdx +++ b/docs/src/snippets/personas/index.mdx @@ -9,19 +9,20 @@ from notte_sdk.client import NotteClient client = NotteClient() persona = client.Persona(create_vault=True) -print(f"Persona email: {persona.info.email}") - -# add a credential to the persona: password is generated automatically and email is the persona's email -persona.add_credentials(url="https://github.com/") - -# read recent emails -recent_emails = persona.emails(only_unread=True, limit=10, timedelta=dt.timedelta(minutes=5)) -print(f"Recent emails: {recent_emails}") - -# get your persona in subsequent scripts -same_persona = client.Persona(persona.info.persona_id) -assert same_persona.info == persona.info - -# delete the persona when you don't need it anymore -persona.delete() +try: + print(f"Persona email: {persona.info.email}") + + # add a credential to the persona: password is generated automatically and email is the persona's email + persona.add_credentials(url="https://github.com/") + + # read recent emails + recent_emails = persona.emails(only_unread=True, limit=10, timedelta=dt.timedelta(minutes=5)) + print(f"Recent emails: {recent_emails}") + + # get your persona in subsequent scripts + same_persona = client.Persona(persona.info.persona_id) + assert same_persona.info == persona.info +finally: + # delete the persona when you don't need it anymore (also deletes its vault) + persona.delete() ``` diff --git a/docs/src/snippets/vaults/manual.mdx b/docs/src/snippets/vaults/manual.mdx index e8f00d4f7..40c2389bb 100644 --- a/docs/src/snippets/vaults/manual.mdx +++ b/docs/src/snippets/vaults/manual.mdx @@ -8,24 +8,24 @@ client = NotteClient() # Creating a new vault vault = client.Vault() - -# Add your credentials securely -vault.add_credentials( - url="https://github.com/", - email="", - password="", - mfa_secret="", -) - -# remove a credential from the vault -vault.delete_credentials(url="https://github.com/") - -# list all credentials in the vault -credentials = vault.list_credentials() -print(credentials) - -# delete the vault when you don't need it anymore -vault.delete() +try: + # Add your credentials securely + vault.add_credentials( + url="https://github.com/", + email="", + password="", + mfa_secret="", + ) + + # remove a credential from the vault + vault.delete_credentials(url="https://github.com/") + + # list all credentials in the vault + credentials = vault.list_credentials() + print(credentials) +finally: + # delete the vault when you don't need it anymore + vault.delete() # you can also list your active vaults as follows: active_vaults = client.vaults.list() diff --git a/docs/src/testers/getting-started/concept_vault.py b/docs/src/testers/getting-started/concept_vault.py index 50a9c2982..e9074c5d6 100644 --- a/docs/src/testers/getting-started/concept_vault.py +++ b/docs/src/testers/getting-started/concept_vault.py @@ -1,10 +1,9 @@ # @sniptest filename=concept_vault.py -# @sniptest show=5-9 +# @sniptest show=4-7 from notte_sdk import NotteClient client = NotteClient() -with client.Session() as session: - vault = client.Vault() +with client.Session() as session, client.Vault() as vault: vault.add_credentials(url="https://github.com", email="...", password="...") agent = client.Agent(session=session, vault=vault) agent.run(task="Login to GitHub") diff --git a/docs/src/testers/personas/index.py b/docs/src/testers/personas/index.py index a30c9401d..2689ecf62 100644 --- a/docs/src/testers/personas/index.py +++ b/docs/src/testers/personas/index.py @@ -6,18 +6,19 @@ client = NotteClient() persona = client.Persona(create_vault=True) -print(f"Persona email: {persona.info.email}") +try: + print(f"Persona email: {persona.info.email}") -# add a credential to the persona: password is generated automatically and email is the persona's email -persona.add_credentials(url="https://github.com/") + # add a credential to the persona: password is generated automatically and email is the persona's email + persona.add_credentials(url="https://github.com/") -# read recent emails -recent_emails = persona.emails(only_unread=True, limit=10, timedelta=dt.timedelta(minutes=5)) -print(f"Recent emails: {recent_emails}") + # read recent emails + recent_emails = persona.emails(only_unread=True, limit=10, timedelta=dt.timedelta(minutes=5)) + print(f"Recent emails: {recent_emails}") -# get your persona in subsequent scripts -same_persona = client.Persona(persona.info.persona_id) -assert same_persona.info == persona.info - -# delete the persona when you don't need it anymore -persona.delete() + # get your persona in subsequent scripts + same_persona = client.Persona(persona.info.persona_id) + assert same_persona.info == persona.info +finally: + # delete the persona when you don't need it anymore (also deletes its vault) + persona.delete() diff --git a/docs/src/testers/vaults/manual.py b/docs/src/testers/vaults/manual.py index 5f3295077..81f2404e6 100644 --- a/docs/src/testers/vaults/manual.py +++ b/docs/src/testers/vaults/manual.py @@ -5,24 +5,24 @@ # Creating a new vault vault = client.Vault() +try: + # Add your credentials securely + vault.add_credentials( + url="https://github.com/", + email="", + password="", + mfa_secret="", + ) -# Add your credentials securely -vault.add_credentials( - url="https://github.com/", - email="", - password="", - mfa_secret="", -) + # remove a credential from the vault + vault.delete_credentials(url="https://github.com/") -# remove a credential from the vault -vault.delete_credentials(url="https://github.com/") - -# list all credentials in the vault -credentials = vault.list_credentials() -print(credentials) - -# delete the vault when you don't need it anymore -vault.delete() + # list all credentials in the vault + credentials = vault.list_credentials() + print(credentials) +finally: + # delete the vault when you don't need it anymore + vault.delete() # you can also list your active vaults as follows: active_vaults = client.vaults.list() diff --git a/docs/src/tests/test_snippets.py b/docs/src/tests/test_snippets.py index f5b14e2bc..704e656c2 100644 --- a/docs/src/tests/test_snippets.py +++ b/docs/src/tests/test_snippets.py @@ -172,12 +172,21 @@ def decorator(func: Callable[[EvalExample, str], Any]): @handle_file("vaults/index.py") -def handle_vault( +def handle_vault_index( eval_example: EvalExample, code: str, ) -> None: - code = code.replace("", "JBSWY3DPEHPK3PXP") - run_example(eval_example, code=code) + if FAST_MODE or TYPE_CHECK_MODE: + # Syntax/type check - don't create client + code = code.replace("", "JBSWY3DPEHPK3PXP").replace("my_vault_id", "placeholder-vault-id") + run_example(eval_example, code=code) + else: + # Full mode: create real vault and always delete it via context manager + _ = load_dotenv() + client = NotteClient() + with client.Vault() as vault: + code = code.replace("", "JBSWY3DPEHPK3PXP").replace("my_vault_id", vault.vault_id) + run_example(eval_example, code=code) @handle_file("agents/index.py") @@ -236,22 +245,18 @@ def handle_workflow_fork( run_example(eval_example, code=code) -@handle_file("vaults/index.py") -def handle_vault_index( +@handle_file("getting-started/concept_vault.py") +def handle_concept_vault( eval_example: EvalExample, code: str, ) -> None: - if FAST_MODE or TYPE_CHECK_MODE: - # Syntax/type check - don't create client - code = code.replace("", "JBSWY3DPEHPK3PXP").replace("my_vault_id", "placeholder-vault-id") - run_example(eval_example, code=code) - else: - # Full mode: create real vault - _ = load_dotenv() - client = NotteClient() - with client.Vault() as vault: - code = code.replace("", "JBSWY3DPEHPK3PXP").replace("my_vault_id", vault.vault_id) - run_example(eval_example, code=code) + """Cap agent steps in execution mode; vault cleanup uses `with client.Vault()`.""" + if not (FAST_MODE or TYPE_CHECK_MODE): + code = code.replace( + "agent = client.Agent(session=session, vault=vault)", + "agent = client.Agent(session=session, vault=vault, max_steps=1)", + ) + run_example(eval_example, code=code) @handle_file("sessions/file_storage_basic.py") diff --git a/examples/landing-examples/landing_examples.py b/examples/landing-examples/landing_examples.py index ca61cd375..77ff8687d 100644 --- a/examples/landing-examples/landing_examples.py +++ b/examples/landing-examples/landing_examples.py @@ -26,31 +26,35 @@ def main(): client = NotteClient(api_key=os.getenv("NOTTE_API_KEY")) - for task, url, vault in landing_examples[3:]: + for task, url, use_vault in landing_examples[3:]: with client.Session() as session: - if vault: - vault = client.Vault() - email = os.getenv("NOTTE_VAULT_TEST_EMAIL") - assert email is not None - pwd = os.getenv("NOTTE_VAULT_TEST_PASSWORD") - assert pwd is not None - _ = vault.add_credentials( - url="https://google.com", - email=email, - password=pwd, - ) + if use_vault: + with client.Vault() as vault: + email = os.getenv("NOTTE_VAULT_TEST_EMAIL") + assert email is not None + pwd = os.getenv("NOTTE_VAULT_TEST_PASSWORD") + assert pwd is not None + _ = vault.add_credentials( + url="https://google.com", + email=email, + password=pwd, + ) + agent = client.Agent( + session=session, + reasoning_model="vertex_ai/gemini-2.0-flash", + max_steps=15, + vault=vault, + ) + run_kwargs = {"task": task, **({"url": url} if url is not None else {})} + response = agent.run(**run_kwargs) else: - vault = None - - agent_kwargs = { - "session": session, - "reasoning_model": "vertex_ai/gemini-2.0-flash", - "max_steps": 15, - **({"vault": vault} if vault is not None else {}), - } - agent = client.Agent(**agent_kwargs) - run_kwargs = {"task": task, **({"url": url} if url is not None else {})} - response = agent.run(**run_kwargs) + agent = client.Agent( + session=session, + reasoning_model="vertex_ai/gemini-2.0-flash", + max_steps=15, + ) + run_kwargs = {"task": task, **({"url": url} if url is not None else {})} + response = agent.run(**run_kwargs) if not response.success: exit(-1) diff --git a/scripts/cleanup_ci_vaults.py b/scripts/cleanup_ci_vaults.py new file mode 100644 index 000000000..cb1c414b5 --- /dev/null +++ b/scripts/cleanup_ci_vaults.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Delete leaked CI vaults for the configured NOTTE_API_KEY account. + +Docs and integration tests create ephemeral vaults. If a job is cancelled or a +snippet exits without cleanup, vaults accumulate until the account hits the +active-vault limit (currently 1000) and CI fails with HTTP 429. + +This script is a safety net: list every active non-persona vault and delete it. +Persona vaults are owned by personas and excluded by default. + +Refuses to run outside CI unless --force is passed (protects personal vaults). +""" + +from __future__ import annotations + +import argparse +import os +import sys +from typing import Any + +from notte_sdk import NotteClient + + +def list_active_vaults(client: NotteClient, *, page_size: int = 100) -> list[Any]: + vaults: list[Any] = [] + page = 1 + while True: + batch = list(client.vaults.list(page=page, page_size=page_size, only_active=True)) + if not batch: + break + vaults.extend(batch) + if len(batch) < page_size: + break + page += 1 + return vaults + + +def cleanup_vaults(*, dry_run: bool, include_persona: bool, force: bool) -> int: + if not force and os.environ.get("CI", "").lower() not in {"1", "true", "yes"}: + print( + "Refusing to delete vaults outside CI. Re-run with --force if you really mean it.", + file=sys.stderr, + ) + return 2 + + client = NotteClient() + vaults = list_active_vaults(client) + targets = [vault for vault in vaults if include_persona or not bool(getattr(vault, "for_persona", False))] + + print(f"Found {len(vaults)} active vault(s); deleting {len(targets)}.") + if dry_run: + for vault in targets: + name = getattr(vault, "name", None) + for_persona = getattr(vault, "for_persona", False) + print(f" [dry-run] would delete {vault.vault_id} name={name!r} for_persona={for_persona}") + return 0 + + deleted = 0 + failed = 0 + for vault in targets: + try: + _ = client.vaults.delete(vault.vault_id) + deleted += 1 + print(f" deleted {vault.vault_id} name={getattr(vault, 'name', None)!r}") + except Exception as exc: # noqa: BLE001 - best-effort CI cleanup + failed += 1 + print(f" failed {vault.vault_id}: {exc}", file=sys.stderr) + + remaining = list_active_vaults(client) + remaining_targets = [ + vault for vault in remaining if include_persona or not bool(getattr(vault, "for_persona", False)) + ] + print(f"Done. deleted={deleted} failed={failed} remaining_non_persona={len(remaining_targets)}") + return 0 if failed == 0 else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + _ = parser.add_argument("--dry-run", action="store_true", help="List vaults without deleting") + _ = parser.add_argument( + "--include-persona", + action="store_true", + help="Also delete persona-owned vaults (dangerous; default skips them)", + ) + _ = parser.add_argument( + "--force", + action="store_true", + help="Allow deletion outside CI (required for local manual cleanup)", + ) + args = parser.parse_args() + return cleanup_vaults(dry_run=args.dry_run, include_persona=args.include_persona, force=args.force) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/sdk/test_vault.py b/tests/integration/sdk/test_vault.py index 91cfbb7bb..d26f513ef 100644 --- a/tests/integration/sdk/test_vault.py +++ b/tests/integration/sdk/test_vault.py @@ -34,17 +34,15 @@ async def fulfill_github_login(route): # pyright: ignore[reportUnknownParameter def test_vault_in_local_agent(): _ = load_dotenv() client = NotteClient(api_key=os.getenv("NOTTE_API_KEY")) - vault = client.Vault() - _ = vault.add_credentials( - url="https://github.com/", - email="xyz@notte.cc", - password="xyz", - ) - with notte.Session() as session: - agent = notte.Agent(session=session, vault=vault, max_steps=5) - _ = agent.run(task="Go to the github.com and try to login with the credentials") - - _ = client.vaults.delete(vault.vault_id) + with client.Vault() as vault: + _ = vault.add_credentials( + url="https://github.com/", + email="xyz@notte.cc", + password="xyz", + ) + with notte.Session() as session: + agent = notte.Agent(session=session, vault=vault, max_steps=5) + _ = agent.run(task="Go to the github.com and try to login with the credentials") @pytest.mark.asyncio From 9637c6963be73e5259ca24a2d629fa3ea1951a3f Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Tue, 25 Aug 2026 17:02:38 +0200 Subject: [PATCH 2/5] fix(ci): never wipe all vaults from parallel CI jobs Drop per-PR vault delete-all steps. Keep only age+name-filtered orphan cleanup via a manual/scheduled workflow so parallel CI sharing the API key cannot race. Co-authored-by: Cursor --- .github/workflows/cleanup-ci-vaults.yml | 29 +++--- .github/workflows/docs-tests-cicd.yml | 13 --- .github/workflows/nightly-examples.yml | 7 -- .github/workflows/test-cicd.yml | 8 -- scripts/cleanup_ci_vaults.py | 115 ++++++++++++++++++++---- 5 files changed, 113 insertions(+), 59 deletions(-) diff --git a/.github/workflows/cleanup-ci-vaults.yml b/.github/workflows/cleanup-ci-vaults.yml index 118ab4840..d2e9a9722 100644 --- a/.github/workflows/cleanup-ci-vaults.yml +++ b/.github/workflows/cleanup-ci-vaults.yml @@ -1,16 +1,10 @@ name: cleanup-ci-vaults +# Manual / scheduled orphan drain only. Do NOT run delete-all from every PR — +# dozens of parallel CI jobs share this API key and would race each other. on: workflow_dispatch: inputs: - include_persona: - description: "Also delete persona-owned vaults" - required: false - default: "false" - type: choice - options: - - "false" - - "true" dry_run: description: "List vaults without deleting" required: false @@ -19,6 +13,14 @@ on: options: - "false" - "true" + min_age_hours: + description: "Only delete ephemeral-named vaults at least this many hours old" + required: false + default: "2" + type: string + schedule: + # Daily drain of leaked default/pytest vaults older than 2h. + - cron: "0 5 * * *" jobs: cleanup: @@ -26,7 +28,6 @@ jobs: timeout-minutes: 30 env: NOTTE_API_KEY: ${{ secrets.NOTTE_API_KEY }} - # Match docs execution tests (default API host used by notte-sdk). DISABLE_TELEMETRY: "true" steps: - name: Checkout code @@ -43,13 +44,11 @@ jobs: - name: Install notte-sdk run: uv pip install --system notte-sdk - - name: Cleanup leaked vaults + - name: Cleanup orphaned CI vaults run: | - ARGS=() - if [ "${{ inputs.dry_run }}" = "true" ]; then + MIN_AGE="${{ github.event.inputs.min_age_hours || '2' }}" + ARGS=(--min-age-hours "$MIN_AGE") + if [ "${{ github.event.inputs.dry_run }}" = "true" ]; then ARGS+=(--dry-run) fi - if [ "${{ inputs.include_persona }}" = "true" ]; then - ARGS+=(--include-persona) - fi python scripts/cleanup_ci_vaults.py "${ARGS[@]}" diff --git a/.github/workflows/docs-tests-cicd.yml b/.github/workflows/docs-tests-cicd.yml index e2a42dc71..17b5c339b 100644 --- a/.github/workflows/docs-tests-cicd.yml +++ b/.github/workflows/docs-tests-cicd.yml @@ -110,18 +110,5 @@ jobs: fi echo "Environment variables are set" - - name: Cleanup leaked vaults (before) - run: | - # Docs execution creates ephemeral vaults; cancelled/failed runs leak them until - # the account hits the active-vault limit (HTTP 429 on vaults/create). - source .venv/bin/activate - python scripts/cleanup_ci_vaults.py - - name: Run execution tests run: cd docs/src && uv run pytest -v --tb=no - - - name: Cleanup leaked vaults (after) - if: always() - run: | - source .venv/bin/activate - python scripts/cleanup_ci_vaults.py diff --git a/.github/workflows/nightly-examples.yml b/.github/workflows/nightly-examples.yml index d2853b5a6..3a832aed0 100644 --- a/.github/workflows/nightly-examples.yml +++ b/.github/workflows/nightly-examples.yml @@ -78,18 +78,11 @@ jobs: fi echo "Environment variables are set" - - name: Cleanup leaked vaults (before) - run: uv run python scripts/cleanup_ci_vaults.py - - name: Run example tests uses: coactions/setup-xvfb@v1 with: run: bash tests/run_examples.sh - - name: Cleanup leaked vaults (after) - if: always() - run: uv run python scripts/cleanup_ci_vaults.py - - name: Send Slack Notification uses: slackapi/slack-github-action@v1.24.0 with: diff --git a/.github/workflows/test-cicd.yml b/.github/workflows/test-cicd.yml index 82e9f030e..f38506938 100644 --- a/.github/workflows/test-cicd.yml +++ b/.github/workflows/test-cicd.yml @@ -158,20 +158,12 @@ jobs: fi echo "Environment variables are set" - - name: Cleanup leaked vaults (before) - if: ${{ env.IS_TRUSTED == 'true' }} - run: uv run python scripts/cleanup_ci_vaults.py - - name: Run unit tests if: ${{ env.IS_TRUSTED == 'true' }} run: | set -o pipefail uv run pytest -n logical tests --ignore=tests/examples/test_examples.py --ignore=tests/examples/test_readme.py --durations=10 --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=packages | tee pytest-coverage.txt - - name: Cleanup leaked vaults (after) - if: ${{ always() && env.IS_TRUSTED == 'true' }} - run: uv run python scripts/cleanup_ci_vaults.py - - name: Pytest coverage comment if: ${{ always() && github.ref != 'refs/heads/main' && env.IS_TRUSTED == 'true' }} uses: MishaKav/pytest-coverage-comment@main diff --git a/scripts/cleanup_ci_vaults.py b/scripts/cleanup_ci_vaults.py index cb1c414b5..c7afb7d1e 100644 --- a/scripts/cleanup_ci_vaults.py +++ b/scripts/cleanup_ci_vaults.py @@ -1,25 +1,35 @@ #!/usr/bin/env python3 -"""Delete leaked CI vaults for the configured NOTTE_API_KEY account. +"""Delete *orphaned* CI vaults for the configured NOTTE_API_KEY account. -Docs and integration tests create ephemeral vaults. If a job is cancelled or a -snippet exits without cleanup, vaults accumulate until the account hits the -active-vault limit (currently 1000) and CI fails with HTTP 429. +Docs/integration tests create ephemeral vaults. Cancelled runs can leak them until +the account hits the active-vault limit (HTTP 429 on vaults/create). -This script is a safety net: list every active non-persona vault and delete it. -Persona vaults are owned by personas and excluded by default. +This script is intentionally conservative so parallel CI jobs are not disrupted: +- Never deletes persona-owned vaults (unless --include-persona). +- Only deletes vaults whose names look ephemeral (default / pytest- / test- / ...). +- Only deletes vaults older than --min-age-hours (default: 2h). -Refuses to run outside CI unless --force is passed (protects personal vaults). +Do not call this from every PR job with a short age window. Prefer proper +per-test teardown, and use this for manual/scheduled orphan drain. """ from __future__ import annotations import argparse +import datetime as dt import os +import re import sys from typing import Any from notte_sdk import NotteClient +# Names produced by CI/docs snippets (VaultCreateRequest default is "default"). +_EPHEMERAL_NAME_RE = re.compile( + r"^(default|pytest-.+|test-.+|test_vault.*|test-code-sample-.+)$", + re.IGNORECASE, +) + def list_active_vaults(client: NotteClient, *, page_size: int = 100) -> list[Any]: vaults: list[Any] = [] @@ -35,7 +45,35 @@ def list_active_vaults(client: NotteClient, *, page_size: int = 100) -> list[Any return vaults -def cleanup_vaults(*, dry_run: bool, include_persona: bool, force: bool) -> int: +def _vault_created_at(vault: Any) -> dt.datetime | None: + created_at = getattr(vault, "created_at", None) + if created_at is None: + return None + if created_at.tzinfo is None: + return created_at.replace(tzinfo=dt.timezone.utc) + return created_at + + +def is_ephemeral_ci_vault(vault: Any, *, min_age: dt.timedelta, now: dt.datetime) -> bool: + if bool(getattr(vault, "for_persona", False)): + return False + name = str(getattr(vault, "name", "") or "") + if not _EPHEMERAL_NAME_RE.match(name): + return False + created_at = _vault_created_at(vault) + if created_at is None: + return False + return (now - created_at) >= min_age + + +def cleanup_vaults( + *, + dry_run: bool, + include_persona: bool, + force: bool, + min_age_hours: float, + all_non_persona: bool, +) -> int: if not force and os.environ.get("CI", "").lower() not in {"1", "true", "yes"}: print( "Refusing to delete vaults outside CI. Re-run with --force if you really mean it.", @@ -43,16 +81,41 @@ def cleanup_vaults(*, dry_run: bool, include_persona: bool, force: bool) -> int: ) return 2 + if min_age_hours < 0: + print("--min-age-hours must be >= 0", file=sys.stderr) + return 2 + client = NotteClient() vaults = list_active_vaults(client) - targets = [vault for vault in vaults if include_persona or not bool(getattr(vault, "for_persona", False))] + now = dt.datetime.now(tz=dt.timezone.utc) + min_age = dt.timedelta(hours=min_age_hours) - print(f"Found {len(vaults)} active vault(s); deleting {len(targets)}.") + if all_non_persona: + targets = [vault for vault in vaults if include_persona or not bool(getattr(vault, "for_persona", False))] + print( + "WARNING: --all-non-persona deletes every matching vault regardless of name/age; unsafe with parallel CI." + ) + else: + targets = [ + vault + for vault in vaults + if (include_persona or not bool(getattr(vault, "for_persona", False))) + and is_ephemeral_ci_vault(vault, min_age=min_age, now=now) + ] + + print( + f"Found {len(vaults)} active vault(s); deleting {len(targets)} orphan(s) " + + f"(min_age_hours={min_age_hours}, all_non_persona={all_non_persona})." + ) if dry_run: for vault in targets: name = getattr(vault, "name", None) for_persona = getattr(vault, "for_persona", False) - print(f" [dry-run] would delete {vault.vault_id} name={name!r} for_persona={for_persona}") + created_at = getattr(vault, "created_at", None) + print( + f" [dry-run] would delete {vault.vault_id} " + + f"name={name!r} for_persona={for_persona} created_at={created_at}" + ) return 0 deleted = 0 @@ -67,10 +130,13 @@ def cleanup_vaults(*, dry_run: bool, include_persona: bool, force: bool) -> int: print(f" failed {vault.vault_id}: {exc}", file=sys.stderr) remaining = list_active_vaults(client) - remaining_targets = [ - vault for vault in remaining if include_persona or not bool(getattr(vault, "for_persona", False)) + remaining_orphans = [ + vault + for vault in remaining + if (include_persona or not bool(getattr(vault, "for_persona", False))) + and is_ephemeral_ci_vault(vault, min_age=min_age, now=dt.datetime.now(tz=dt.timezone.utc)) ] - print(f"Done. deleted={deleted} failed={failed} remaining_non_persona={len(remaining_targets)}") + print(f"Done. deleted={deleted} failed={failed} remaining_orphan_candidates={len(remaining_orphans)}") return 0 if failed == 0 else 1 @@ -80,15 +146,32 @@ def main() -> int: _ = parser.add_argument( "--include-persona", action="store_true", - help="Also delete persona-owned vaults (dangerous; default skips them)", + help="Also consider persona-owned vaults (dangerous; default skips them)", ) _ = parser.add_argument( "--force", action="store_true", help="Allow deletion outside CI (required for local manual cleanup)", ) + _ = parser.add_argument( + "--min-age-hours", + type=float, + default=2.0, + help="Only delete ephemeral-named vaults at least this old (default: 2)", + ) + _ = parser.add_argument( + "--all-non-persona", + action="store_true", + help="Delete all non-persona vaults (ignores name/age). Unsafe with parallel CI.", + ) args = parser.parse_args() - return cleanup_vaults(dry_run=args.dry_run, include_persona=args.include_persona, force=args.force) + return cleanup_vaults( + dry_run=args.dry_run, + include_persona=args.include_persona, + force=args.force, + min_age_hours=args.min_age_hours, + all_non_persona=args.all_non_persona, + ) if __name__ == "__main__": From ba240e46e473765cdfd29d388cbe03038086ab85 Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Tue, 25 Aug 2026 17:05:50 +0200 Subject: [PATCH 3/5] fix(ci): delete only vaults created by the current workflow run Tag unnamed vault creates with NOTTE_CI_VAULT_PREFIX and tear down just that prefix after the job, so parallel CI sharing the API key cannot wipe each other. Co-authored-by: Cursor --- .github/workflows/cleanup-ci-vaults.yml | 44 ++++-- .github/workflows/docs-tests-cicd.yml | 8 + .github/workflows/nightly-examples.yml | 5 + .github/workflows/test-cicd.yml | 6 + docs/src/conftest.py | 33 ++++ scripts/ci_vault_scope.py | 195 ++++++++++++++++++++++++ scripts/cleanup_ci_vaults.py | 183 +++++----------------- tests/conftest.py | 21 +++ 8 files changed, 333 insertions(+), 162 deletions(-) create mode 100644 docs/src/conftest.py create mode 100644 scripts/ci_vault_scope.py diff --git a/.github/workflows/cleanup-ci-vaults.yml b/.github/workflows/cleanup-ci-vaults.yml index d2e9a9722..c3d595cc0 100644 --- a/.github/workflows/cleanup-ci-vaults.yml +++ b/.github/workflows/cleanup-ci-vaults.yml @@ -1,10 +1,26 @@ name: cleanup-ci-vaults -# Manual / scheduled orphan drain only. Do NOT run delete-all from every PR — -# dozens of parallel CI jobs share this API key and would race each other. +# Manual only. Prefer deleting a single run prefix; optional orphan-defaults for backlog. on: workflow_dispatch: inputs: + mode: + description: "prefix = one workflow run; orphan-defaults = drain leaked name=default vaults" + required: true + default: "prefix" + type: choice + options: + - "prefix" + - "orphan-defaults" + prefix: + description: "Required for mode=prefix (e.g. ci-32862931541-execution)" + required: false + type: string + min_age_hours: + description: "For orphan-defaults: only delete default vaults older than this many hours" + required: false + default: "2" + type: string dry_run: description: "List vaults without deleting" required: false @@ -13,14 +29,6 @@ on: options: - "false" - "true" - min_age_hours: - description: "Only delete ephemeral-named vaults at least this many hours old" - required: false - default: "2" - type: string - schedule: - # Daily drain of leaked default/pytest vaults older than 2h. - - cron: "0 5 * * *" jobs: cleanup: @@ -44,11 +52,19 @@ jobs: - name: Install notte-sdk run: uv pip install --system notte-sdk - - name: Cleanup orphaned CI vaults + - name: Cleanup vaults run: | - MIN_AGE="${{ github.event.inputs.min_age_hours || '2' }}" - ARGS=(--min-age-hours "$MIN_AGE") - if [ "${{ github.event.inputs.dry_run }}" = "true" ]; then + ARGS=() + if [ "${{ inputs.dry_run }}" = "true" ]; then ARGS+=(--dry-run) fi + if [ "${{ inputs.mode }}" = "orphan-defaults" ]; then + ARGS+=(--orphan-defaults --min-age-hours "${{ inputs.min_age_hours }}") + else + if [ -z "${{ inputs.prefix }}" ]; then + echo "prefix is required when mode=prefix" + exit 1 + fi + ARGS+=(--prefix "${{ inputs.prefix }}") + fi python scripts/cleanup_ci_vaults.py "${ARGS[@]}" diff --git a/.github/workflows/docs-tests-cicd.yml b/.github/workflows/docs-tests-cicd.yml index 17b5c339b..8f2bb3ec5 100644 --- a/.github/workflows/docs-tests-cicd.yml +++ b/.github/workflows/docs-tests-cicd.yml @@ -79,6 +79,8 @@ jobs: NOTTE_GITHUB_COM_EMAIL: ${{ secrets.NOTTE_GITHUB_COM_EMAIL }} NOTTE_GITHUB_COM_PASSWORD: ${{ secrets.NOTTE_GITHUB_COM_PASSWORD }} NOTTE_GITHUB_COM_MFA_SECRET: ${{ secrets.NOTTE_GITHUB_COM_MFA_SECRET }} + # Scope vault creates to this job only; cleanup deletes only this prefix. + NOTTE_CI_VAULT_PREFIX: ci-${{ github.run_id }}-${{ github.job }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -112,3 +114,9 @@ jobs: - name: Run execution tests run: cd docs/src && uv run pytest -v --tb=no + + - name: Cleanup vaults created by this run + if: always() + run: | + source .venv/bin/activate + python scripts/cleanup_ci_vaults.py diff --git a/.github/workflows/nightly-examples.yml b/.github/workflows/nightly-examples.yml index 3a832aed0..c64c1301e 100644 --- a/.github/workflows/nightly-examples.yml +++ b/.github/workflows/nightly-examples.yml @@ -28,6 +28,7 @@ env: CREDIT_CARD_NUMBER: 4242424242424242 CREDIT_CARD_CVV: 777 CREDIT_CARD_EXPIRATION: "05/28" + NOTTE_CI_VAULT_PREFIX: ci-${{ github.run_id }}-${{ github.job }} jobs: example-tests: @@ -83,6 +84,10 @@ jobs: with: run: bash tests/run_examples.sh + - name: Cleanup vaults created by this run + if: always() + run: uv run python scripts/cleanup_ci_vaults.py + - name: Send Slack Notification uses: slackapi/slack-github-action@v1.24.0 with: diff --git a/.github/workflows/test-cicd.yml b/.github/workflows/test-cicd.yml index f38506938..a1927f2f1 100644 --- a/.github/workflows/test-cicd.yml +++ b/.github/workflows/test-cicd.yml @@ -38,6 +38,8 @@ env: # True for push, manual dispatch, and same-repo PRs; false for fork PRs. Steps that # need secrets are gated on this so fork PRs skip them cleanly instead of failing. IS_TRUSTED: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + # Scope vault creates to this workflow job; cleanup deletes only this prefix. + NOTTE_CI_VAULT_PREFIX: ci-${{ github.run_id }}-${{ github.job }} permissions: pull-requests: write @@ -164,6 +166,10 @@ jobs: set -o pipefail uv run pytest -n logical tests --ignore=tests/examples/test_examples.py --ignore=tests/examples/test_readme.py --durations=10 --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=packages | tee pytest-coverage.txt + - name: Cleanup vaults created by this run + if: ${{ always() && env.IS_TRUSTED == 'true' }} + run: uv run python scripts/cleanup_ci_vaults.py + - name: Pytest coverage comment if: ${{ always() && github.ref != 'refs/heads/main' && env.IS_TRUSTED == 'true' }} uses: MishaKav/pytest-coverage-comment@main diff --git a/docs/src/conftest.py b/docs/src/conftest.py new file mode 100644 index 000000000..88a18b009 --- /dev/null +++ b/docs/src/conftest.py @@ -0,0 +1,33 @@ +"""Docs snippet test hooks. + +Installs CI vault scoping when ``NOTTE_CI_VAULT_PREFIX`` is set so this workflow +run names vaults under that prefix. Deletion is done by the workflow +``Cleanup vaults created by this run`` step (not here) so pytest-xdist workers +cannot delete vaults still in use by sibling workers. +""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path +from typing import Any + + +def _load_ci_vault_scope() -> Any | None: + if not os.environ.get("NOTTE_CI_VAULT_PREFIX"): + return None + path = Path(__file__).resolve().parents[2] / "scripts" / "ci_vault_scope.py" + spec = importlib.util.spec_from_file_location("ci_vault_scope", path) + if spec is None or spec.loader is None: + return None + module = importlib.util.module_from_spec(spec) + sys.modules["ci_vault_scope"] = module + spec.loader.exec_module(module) + return module + + +_SCOPE = _load_ci_vault_scope() +if _SCOPE is not None: + _SCOPE.install() diff --git a/scripts/ci_vault_scope.py b/scripts/ci_vault_scope.py new file mode 100644 index 000000000..389aee855 --- /dev/null +++ b/scripts/ci_vault_scope.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Scope vault creates to the current CI workflow run and delete only those. + +When ``NOTTE_CI_VAULT_PREFIX`` is set (e.g. ``ci--``): +1. ``install()`` patches ``VaultsClient.create`` so unnamed/default vaults get a + unique name under that prefix and their IDs are recorded. +2. ``cleanup_this_run()`` deletes only vaults created under that prefix / recorded + for this run — never other parallel jobs' vaults. +""" + +from __future__ import annotations + +import os +import re +import sys +from pathlib import Path +from typing import Any +from uuid import uuid4 + +_VAULT_NAME_RE = re.compile(r"^[a-zA-Z0-9\s\-_]+$") +_installed = False + + +def sanitize_prefix(raw: str) -> str: + """Make a vault-name-safe prefix (API: 3-50 chars, [A-Za-z0-9 _-]).""" + cleaned = re.sub(r"[^a-zA-Z0-9_-]+", "-", raw.strip()).strip("-_") + # Leave room for "-xxxxxxxx" suffix (9 chars); API max name length is 50. + cleaned = cleaned[:41].rstrip("-_") + if len(cleaned) < 3: + cleaned = f"ci-{cleaned}" if cleaned else "ci-run" + return cleaned + + +def run_prefix() -> str | None: + raw = os.environ.get("NOTTE_CI_VAULT_PREFIX", "").strip() + if not raw: + return None + return sanitize_prefix(raw) + + +def _ids_file(prefix: str) -> Path: + base = Path(os.environ.get("RUNNER_TEMP") or os.environ.get("TMPDIR") or "/tmp") + safe = re.sub(r"[^a-zA-Z0-9_-]+", "-", prefix) + return base / f"notte-ci-vault-ids-{safe}.txt" + + +def record_vault_id(prefix: str, vault_id: str) -> None: + path = _ids_file(prefix) + path.parent.mkdir(parents=True, exist_ok=True) + # Append is atomic enough per line on POSIX for concurrent xdist workers. + with path.open("a", encoding="utf-8") as handle: + _ = handle.write(f"{vault_id}\n") + handle.flush() + os.fsync(handle.fileno()) + + +def recorded_vault_ids(prefix: str) -> list[str]: + path = _ids_file(prefix) + if not path.exists(): + return [] + return [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def install() -> None: + """Patch vault create so this CI run owns a unique name prefix.""" + global _installed + prefix = run_prefix() + if prefix is None or _installed: + return + + from notte_sdk.endpoints.vaults import VaultsClient + + original_create = VaultsClient.create + + def create(self: Any, **data: Any) -> Any: + name = data.get("name") + if name is None or name == "default": + candidate = f"{prefix}-{uuid4().hex[:8]}" + if not _VAULT_NAME_RE.match(candidate): + candidate = re.sub(r"[^a-zA-Z0-9_-]+", "-", candidate) + data = {**data, "name": candidate} + vault = original_create(self, **data) + vault_id = getattr(vault, "vault_id", None) + if isinstance(vault_id, str) and vault_id: + record_vault_id(prefix, vault_id) + return vault + + VaultsClient.create = create + _installed = True + print(f"[ci-vault-scope] installed create patch prefix={prefix!r}", file=sys.stderr) + + +def list_active_vaults(client: Any, *, page_size: int = 100) -> list[Any]: + vaults: list[Any] = [] + page = 1 + while True: + batch = list(client.vaults.list(page=page, page_size=page_size, only_active=True)) + if not batch: + break + vaults.extend(batch) + if len(batch) < page_size: + break + page += 1 + return vaults + + +def cleanup_this_run(*, dry_run: bool = False, prefix: str | None = None) -> int: + """Delete only vaults created by this workflow run (prefix + recorded IDs).""" + resolved = sanitize_prefix(prefix) if prefix else run_prefix() + if resolved is None: + print("No NOTTE_CI_VAULT_PREFIX / --prefix; nothing to clean.", file=sys.stderr) + return 0 + + from notte_sdk import NotteClient + + client = NotteClient() + recorded = set(recorded_vault_ids(resolved)) + listed = list_active_vaults(client) + by_prefix = { + vault.vault_id + for vault in listed + if str(getattr(vault, "name", "") or "").startswith(resolved) and not bool(getattr(vault, "for_persona", False)) + } + targets = sorted(recorded | by_prefix) + + print( + f"[ci-vault-scope] prefix={resolved!r} recorded={len(recorded)} " + + f"by_prefix={len(by_prefix)} deleting={len(targets)} dry_run={dry_run}" + ) + if dry_run: + for vault_id in targets: + print(f" [dry-run] would delete {vault_id}") + return 0 + + deleted = 0 + failed = 0 + for vault_id in targets: + try: + _ = client.vaults.delete(vault_id) + deleted += 1 + print(f" deleted {vault_id}") + except Exception as exc: # noqa: BLE001 - best-effort run teardown + failed += 1 + print(f" failed {vault_id}: {exc}", file=sys.stderr) + + print(f"[ci-vault-scope] done deleted={deleted} failed={failed}") + return 0 if failed == 0 else 1 + + +def cleanup_orphan_defaults(*, dry_run: bool, min_age_hours: float) -> int: + """One-shot drain of leaked name=default vaults older than min_age_hours.""" + import datetime as dt + + from notte_sdk import NotteClient + + if min_age_hours < 0: + print("--min-age-hours must be >= 0", file=sys.stderr) + return 2 + + client = NotteClient() + now = dt.datetime.now(tz=dt.timezone.utc) + min_age = dt.timedelta(hours=min_age_hours) + targets: list[Any] = [] + for vault in list_active_vaults(client): + if bool(getattr(vault, "for_persona", False)): + continue + if str(getattr(vault, "name", "") or "") != "default": + continue + created_at = getattr(vault, "created_at", None) + if created_at is None: + continue + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=dt.timezone.utc) + if (now - created_at) < min_age: + continue + targets.append(vault) + + print(f"[ci-vault-scope] orphan default vaults older than {min_age_hours}h: {len(targets)}") + if dry_run: + for vault in targets: + print(f" [dry-run] would delete {vault.vault_id} created_at={vault.created_at}") + return 0 + + deleted = 0 + failed = 0 + for vault in targets: + try: + _ = client.vaults.delete(vault.vault_id) + deleted += 1 + print(f" deleted {vault.vault_id}") + except Exception as exc: # noqa: BLE001 + failed += 1 + print(f" failed {vault.vault_id}: {exc}", file=sys.stderr) + print(f"[ci-vault-scope] orphan drain done deleted={deleted} failed={failed}") + return 0 if failed == 0 else 1 diff --git a/scripts/cleanup_ci_vaults.py b/scripts/cleanup_ci_vaults.py index c7afb7d1e..a0f44a3ad 100644 --- a/scripts/cleanup_ci_vaults.py +++ b/scripts/cleanup_ci_vaults.py @@ -1,177 +1,64 @@ #!/usr/bin/env python3 -"""Delete *orphaned* CI vaults for the configured NOTTE_API_KEY account. +"""CLI to delete vaults owned by a single CI workflow run. -Docs/integration tests create ephemeral vaults. Cancelled runs can leak them until -the account hits the active-vault limit (HTTP 429 on vaults/create). +Preferred usage (from a workflow ``if: always()`` step):: -This script is intentionally conservative so parallel CI jobs are not disrupted: -- Never deletes persona-owned vaults (unless --include-persona). -- Only deletes vaults whose names look ephemeral (default / pytest- / test- / ...). -- Only deletes vaults older than --min-age-hours (default: 2h). + NOTTE_CI_VAULT_PREFIX=ci-$GITHUB_RUN_ID-$GITHUB_JOB \\ + python scripts/cleanup_ci_vaults.py -Do not call this from every PR job with a short age window. Prefer proper -per-test teardown, and use this for manual/scheduled orphan drain. +Only vaults created under that prefix / recorded by ``ci_vault_scope.install()`` +are deleted — parallel jobs sharing the same API key are left alone. + +One-shot backlog drain (manual only):: + + python scripts/cleanup_ci_vaults.py --orphan-defaults --min-age-hours 2 """ from __future__ import annotations import argparse -import datetime as dt -import os -import re -import sys -from typing import Any - -from notte_sdk import NotteClient - -# Names produced by CI/docs snippets (VaultCreateRequest default is "default"). -_EPHEMERAL_NAME_RE = re.compile( - r"^(default|pytest-.+|test-.+|test_vault.*|test-code-sample-.+)$", - re.IGNORECASE, -) - - -def list_active_vaults(client: NotteClient, *, page_size: int = 100) -> list[Any]: - vaults: list[Any] = [] - page = 1 - while True: - batch = list(client.vaults.list(page=page, page_size=page_size, only_active=True)) - if not batch: - break - vaults.extend(batch) - if len(batch) < page_size: - break - page += 1 - return vaults - - -def _vault_created_at(vault: Any) -> dt.datetime | None: - created_at = getattr(vault, "created_at", None) - if created_at is None: - return None - if created_at.tzinfo is None: - return created_at.replace(tzinfo=dt.timezone.utc) - return created_at - - -def is_ephemeral_ci_vault(vault: Any, *, min_age: dt.timedelta, now: dt.datetime) -> bool: - if bool(getattr(vault, "for_persona", False)): - return False - name = str(getattr(vault, "name", "") or "") - if not _EPHEMERAL_NAME_RE.match(name): - return False - created_at = _vault_created_at(vault) - if created_at is None: - return False - return (now - created_at) >= min_age - - -def cleanup_vaults( - *, - dry_run: bool, - include_persona: bool, - force: bool, - min_age_hours: float, - all_non_persona: bool, -) -> int: - if not force and os.environ.get("CI", "").lower() not in {"1", "true", "yes"}: - print( - "Refusing to delete vaults outside CI. Re-run with --force if you really mean it.", - file=sys.stderr, - ) - return 2 - - if min_age_hours < 0: - print("--min-age-hours must be >= 0", file=sys.stderr) - return 2 +import importlib.util +from pathlib import Path - client = NotteClient() - vaults = list_active_vaults(client) - now = dt.datetime.now(tz=dt.timezone.utc) - min_age = dt.timedelta(hours=min_age_hours) - if all_non_persona: - targets = [vault for vault in vaults if include_persona or not bool(getattr(vault, "for_persona", False))] - print( - "WARNING: --all-non-persona deletes every matching vault regardless of name/age; unsafe with parallel CI." - ) - else: - targets = [ - vault - for vault in vaults - if (include_persona or not bool(getattr(vault, "for_persona", False))) - and is_ephemeral_ci_vault(vault, min_age=min_age, now=now) - ] - - print( - f"Found {len(vaults)} active vault(s); deleting {len(targets)} orphan(s) " - + f"(min_age_hours={min_age_hours}, all_non_persona={all_non_persona})." - ) - if dry_run: - for vault in targets: - name = getattr(vault, "name", None) - for_persona = getattr(vault, "for_persona", False) - created_at = getattr(vault, "created_at", None) - print( - f" [dry-run] would delete {vault.vault_id} " - + f"name={name!r} for_persona={for_persona} created_at={created_at}" - ) - return 0 - - deleted = 0 - failed = 0 - for vault in targets: - try: - _ = client.vaults.delete(vault.vault_id) - deleted += 1 - print(f" deleted {vault.vault_id} name={getattr(vault, 'name', None)!r}") - except Exception as exc: # noqa: BLE001 - best-effort CI cleanup - failed += 1 - print(f" failed {vault.vault_id}: {exc}", file=sys.stderr) - - remaining = list_active_vaults(client) - remaining_orphans = [ - vault - for vault in remaining - if (include_persona or not bool(getattr(vault, "for_persona", False))) - and is_ephemeral_ci_vault(vault, min_age=min_age, now=dt.datetime.now(tz=dt.timezone.utc)) - ] - print(f"Done. deleted={deleted} failed={failed} remaining_orphan_candidates={len(remaining_orphans)}") - return 0 if failed == 0 else 1 +def _load_scope() -> object: + path = Path(__file__).resolve().parent / "ci_vault_scope.py" + spec = importlib.util.spec_from_file_location("ci_vault_scope", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - _ = parser.add_argument("--dry-run", action="store_true", help="List vaults without deleting") _ = parser.add_argument( - "--include-persona", - action="store_true", - help="Also consider persona-owned vaults (dangerous; default skips them)", + "--prefix", + default=None, + help="Vault name prefix for this run (default: $NOTTE_CI_VAULT_PREFIX)", ) + _ = parser.add_argument("--dry-run", action="store_true", help="List vaults without deleting") _ = parser.add_argument( - "--force", + "--orphan-defaults", action="store_true", - help="Allow deletion outside CI (required for local manual cleanup)", + help="One-shot: delete name=default vaults older than --min-age-hours (backlog drain)", ) _ = parser.add_argument( "--min-age-hours", type=float, default=2.0, - help="Only delete ephemeral-named vaults at least this old (default: 2)", - ) - _ = parser.add_argument( - "--all-non-persona", - action="store_true", - help="Delete all non-persona vaults (ignores name/age). Unsafe with parallel CI.", + help="Age cutoff for --orphan-defaults (default: 2)", ) args = parser.parse_args() - return cleanup_vaults( - dry_run=args.dry_run, - include_persona=args.include_persona, - force=args.force, - min_age_hours=args.min_age_hours, - all_non_persona=args.all_non_persona, - ) + + scope = _load_scope() + if args.orphan_defaults: + cleanup_orphans = getattr(scope, "cleanup_orphan_defaults") + return int(cleanup_orphans(dry_run=args.dry_run, min_age_hours=args.min_age_hours)) + + cleanup = getattr(scope, "cleanup_this_run") + return int(cleanup(dry_run=args.dry_run, prefix=args.prefix)) if __name__ == "__main__": diff --git a/tests/conftest.py b/tests/conftest.py index 0a81382aa..908c3d251 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,8 @@ +import importlib.util import os +import sys from pathlib import Path +from typing import Any import notte_core @@ -14,6 +17,24 @@ os.environ["DISABLE_GPU"] = "true" +def _load_ci_vault_scope() -> Any | None: + if not os.environ.get("NOTTE_CI_VAULT_PREFIX"): + return None + path = Path(__file__).resolve().parents[1] / "scripts" / "ci_vault_scope.py" + spec = importlib.util.spec_from_file_location("ci_vault_scope", path) + if spec is None or spec.loader is None: + return None + module = importlib.util.module_from_spec(spec) + sys.modules["ci_vault_scope"] = module + spec.loader.exec_module(module) + return module + + +_CI_VAULT_SCOPE = _load_ci_vault_scope() +if _CI_VAULT_SCOPE is not None: + _CI_VAULT_SCOPE.install() + + # Flaky test configuration: # Tests marked with @pytest.mark.flaky(reruns=N, reruns_delay=S) will automatically # retry N times with S seconds delay between retries when they fail. From 692db3fd54bcbe0259f337545d2df84f14d3628a Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Tue, 25 Aug 2026 17:08:09 +0200 Subject: [PATCH 4/5] docs: keep vault/persona examples simple without try/finally CI vault-prefix teardown covers leaks if a snippet exits early; examples stay linear for readers. Co-authored-by: Cursor --- docs/src/snippets/personas/index.mdx | 31 ++++++++++++------------ docs/src/snippets/vaults/manual.mdx | 36 ++++++++++++++-------------- docs/src/testers/personas/index.py | 25 ++++++++++--------- docs/src/testers/vaults/manual.py | 32 ++++++++++++------------- 4 files changed, 61 insertions(+), 63 deletions(-) diff --git a/docs/src/snippets/personas/index.mdx b/docs/src/snippets/personas/index.mdx index 66b7fbe5f..d7dabdf81 100644 --- a/docs/src/snippets/personas/index.mdx +++ b/docs/src/snippets/personas/index.mdx @@ -9,20 +9,19 @@ from notte_sdk.client import NotteClient client = NotteClient() persona = client.Persona(create_vault=True) -try: - print(f"Persona email: {persona.info.email}") - - # add a credential to the persona: password is generated automatically and email is the persona's email - persona.add_credentials(url="https://github.com/") - - # read recent emails - recent_emails = persona.emails(only_unread=True, limit=10, timedelta=dt.timedelta(minutes=5)) - print(f"Recent emails: {recent_emails}") - - # get your persona in subsequent scripts - same_persona = client.Persona(persona.info.persona_id) - assert same_persona.info == persona.info -finally: - # delete the persona when you don't need it anymore (also deletes its vault) - persona.delete() +print(f"Persona email: {persona.info.email}") + +# add a credential to the persona: password is generated automatically and email is the persona's email +persona.add_credentials(url="https://github.com/") + +# read recent emails +recent_emails = persona.emails(only_unread=True, limit=10, timedelta=dt.timedelta(minutes=5)) +print(f"Recent emails: {recent_emails}") + +# get your persona in subsequent scripts +same_persona = client.Persona(persona.info.persona_id) +assert same_persona.info == persona.info + +# delete the persona when you don't need it anymore +persona.delete() ``` diff --git a/docs/src/snippets/vaults/manual.mdx b/docs/src/snippets/vaults/manual.mdx index 40c2389bb..e8f00d4f7 100644 --- a/docs/src/snippets/vaults/manual.mdx +++ b/docs/src/snippets/vaults/manual.mdx @@ -8,24 +8,24 @@ client = NotteClient() # Creating a new vault vault = client.Vault() -try: - # Add your credentials securely - vault.add_credentials( - url="https://github.com/", - email="", - password="", - mfa_secret="", - ) - - # remove a credential from the vault - vault.delete_credentials(url="https://github.com/") - - # list all credentials in the vault - credentials = vault.list_credentials() - print(credentials) -finally: - # delete the vault when you don't need it anymore - vault.delete() + +# Add your credentials securely +vault.add_credentials( + url="https://github.com/", + email="", + password="", + mfa_secret="", +) + +# remove a credential from the vault +vault.delete_credentials(url="https://github.com/") + +# list all credentials in the vault +credentials = vault.list_credentials() +print(credentials) + +# delete the vault when you don't need it anymore +vault.delete() # you can also list your active vaults as follows: active_vaults = client.vaults.list() diff --git a/docs/src/testers/personas/index.py b/docs/src/testers/personas/index.py index 2689ecf62..a30c9401d 100644 --- a/docs/src/testers/personas/index.py +++ b/docs/src/testers/personas/index.py @@ -6,19 +6,18 @@ client = NotteClient() persona = client.Persona(create_vault=True) -try: - print(f"Persona email: {persona.info.email}") +print(f"Persona email: {persona.info.email}") - # add a credential to the persona: password is generated automatically and email is the persona's email - persona.add_credentials(url="https://github.com/") +# add a credential to the persona: password is generated automatically and email is the persona's email +persona.add_credentials(url="https://github.com/") - # read recent emails - recent_emails = persona.emails(only_unread=True, limit=10, timedelta=dt.timedelta(minutes=5)) - print(f"Recent emails: {recent_emails}") +# read recent emails +recent_emails = persona.emails(only_unread=True, limit=10, timedelta=dt.timedelta(minutes=5)) +print(f"Recent emails: {recent_emails}") - # get your persona in subsequent scripts - same_persona = client.Persona(persona.info.persona_id) - assert same_persona.info == persona.info -finally: - # delete the persona when you don't need it anymore (also deletes its vault) - persona.delete() +# get your persona in subsequent scripts +same_persona = client.Persona(persona.info.persona_id) +assert same_persona.info == persona.info + +# delete the persona when you don't need it anymore +persona.delete() diff --git a/docs/src/testers/vaults/manual.py b/docs/src/testers/vaults/manual.py index 81f2404e6..5f3295077 100644 --- a/docs/src/testers/vaults/manual.py +++ b/docs/src/testers/vaults/manual.py @@ -5,24 +5,24 @@ # Creating a new vault vault = client.Vault() -try: - # Add your credentials securely - vault.add_credentials( - url="https://github.com/", - email="", - password="", - mfa_secret="", - ) - # remove a credential from the vault - vault.delete_credentials(url="https://github.com/") +# Add your credentials securely +vault.add_credentials( + url="https://github.com/", + email="", + password="", + mfa_secret="", +) - # list all credentials in the vault - credentials = vault.list_credentials() - print(credentials) -finally: - # delete the vault when you don't need it anymore - vault.delete() +# remove a credential from the vault +vault.delete_credentials(url="https://github.com/") + +# list all credentials in the vault +credentials = vault.list_credentials() +print(credentials) + +# delete the vault when you don't need it anymore +vault.delete() # you can also list your active vaults as follows: active_vaults = client.vaults.list() From 616cf339add838e011dcaf3504ddc492f534f0bf Mon Sep 17 00:00:00 2001 From: Lucas Giordano Date: Tue, 25 Aug 2026 17:15:58 +0200 Subject: [PATCH 5/5] fix(ci): treat already-deleted vaults as success in run cleanup Tests that tear down correctly leave recorded IDs that return 400 not active; cleanup must be idempotent. Co-authored-by: Cursor --- scripts/ci_vault_scope.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/ci_vault_scope.py b/scripts/ci_vault_scope.py index 389aee855..c233cdcec 100644 --- a/scripts/ci_vault_scope.py +++ b/scripts/ci_vault_scope.py @@ -134,19 +134,29 @@ def cleanup_this_run(*, dry_run: bool = False, prefix: str | None = None) -> int deleted = 0 failed = 0 + already_gone = 0 for vault_id in targets: try: _ = client.vaults.delete(vault_id) deleted += 1 print(f" deleted {vault_id}") except Exception as exc: # noqa: BLE001 - best-effort run teardown + if _is_already_deleted_error(exc): + already_gone += 1 + print(f" already gone {vault_id}") + continue failed += 1 print(f" failed {vault_id}: {exc}", file=sys.stderr) - print(f"[ci-vault-scope] done deleted={deleted} failed={failed}") + print(f"[ci-vault-scope] done deleted={deleted} already_gone={already_gone} failed={failed}") return 0 if failed == 0 else 1 +def _is_already_deleted_error(exc: BaseException) -> bool: + text = str(exc).lower() + return "not active" in text or "not found" in text or ("already" in text and "deleted" in text) + + def cleanup_orphan_defaults(*, dry_run: bool, min_age_hours: float) -> int: """One-shot drain of leaked name=default vaults older than min_age_hours.""" import datetime as dt @@ -183,13 +193,18 @@ def cleanup_orphan_defaults(*, dry_run: bool, min_age_hours: float) -> int: deleted = 0 failed = 0 + already_gone = 0 for vault in targets: try: _ = client.vaults.delete(vault.vault_id) deleted += 1 print(f" deleted {vault.vault_id}") except Exception as exc: # noqa: BLE001 + if _is_already_deleted_error(exc): + already_gone += 1 + print(f" already gone {vault.vault_id}") + continue failed += 1 print(f" failed {vault.vault_id}: {exc}", file=sys.stderr) - print(f"[ci-vault-scope] orphan drain done deleted={deleted} failed={failed}") + print(f"[ci-vault-scope] orphan drain done deleted={deleted} already_gone={already_gone} failed={failed}") return 0 if failed == 0 else 1