diff --git a/.github/workflows/cleanup-ci-vaults.yml b/.github/workflows/cleanup-ci-vaults.yml new file mode 100644 index 000000000..c3d595cc0 --- /dev/null +++ b/.github/workflows/cleanup-ci-vaults.yml @@ -0,0 +1,70 @@ +name: cleanup-ci-vaults + +# Manual only. Prefer deleting a single run prefix; optional orphan-defaults for backlog. +on: + workflow_dispatch: + inputs: + mode: + description: "prefix = one workflow run; orphan-defaults = drain leaked name=default vaults" + required: true + default: "prefix" + type: choice + options: + - "prefix" + - "orphan-defaults" + prefix: + description: "Required for mode=prefix (e.g. ci-32862931541-execution)" + required: false + type: string + min_age_hours: + description: "For orphan-defaults: only delete default vaults older than this many hours" + required: false + default: "2" + type: string + dry_run: + description: "List vaults without deleting" + required: false + default: "false" + type: choice + options: + - "false" + - "true" + +jobs: + cleanup: + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 30 + env: + NOTTE_API_KEY: ${{ secrets.NOTTE_API_KEY }} + DISABLE_TELEMETRY: "true" + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: ".python-version" + + - name: Install notte-sdk + run: uv pip install --system notte-sdk + + - name: Cleanup vaults + run: | + ARGS=() + if [ "${{ inputs.dry_run }}" = "true" ]; then + ARGS+=(--dry-run) + fi + if [ "${{ inputs.mode }}" = "orphan-defaults" ]; then + ARGS+=(--orphan-defaults --min-age-hours "${{ inputs.min_age_hours }}") + else + if [ -z "${{ inputs.prefix }}" ]; then + echo "prefix is required when mode=prefix" + exit 1 + fi + ARGS+=(--prefix "${{ inputs.prefix }}") + fi + python scripts/cleanup_ci_vaults.py "${ARGS[@]}" diff --git a/.github/workflows/docs-tests-cicd.yml b/.github/workflows/docs-tests-cicd.yml index 17b5c339b..8f2bb3ec5 100644 --- a/.github/workflows/docs-tests-cicd.yml +++ b/.github/workflows/docs-tests-cicd.yml @@ -79,6 +79,8 @@ jobs: NOTTE_GITHUB_COM_EMAIL: ${{ secrets.NOTTE_GITHUB_COM_EMAIL }} NOTTE_GITHUB_COM_PASSWORD: ${{ secrets.NOTTE_GITHUB_COM_PASSWORD }} NOTTE_GITHUB_COM_MFA_SECRET: ${{ secrets.NOTTE_GITHUB_COM_MFA_SECRET }} + # Scope vault creates to this job only; cleanup deletes only this prefix. + NOTTE_CI_VAULT_PREFIX: ci-${{ github.run_id }}-${{ github.job }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -112,3 +114,9 @@ jobs: - name: Run execution tests run: cd docs/src && uv run pytest -v --tb=no + + - name: Cleanup vaults created by this run + if: always() + run: | + source .venv/bin/activate + python scripts/cleanup_ci_vaults.py diff --git a/.github/workflows/nightly-examples.yml b/.github/workflows/nightly-examples.yml index 3a832aed0..c64c1301e 100644 --- a/.github/workflows/nightly-examples.yml +++ b/.github/workflows/nightly-examples.yml @@ -28,6 +28,7 @@ env: CREDIT_CARD_NUMBER: 4242424242424242 CREDIT_CARD_CVV: 777 CREDIT_CARD_EXPIRATION: "05/28" + NOTTE_CI_VAULT_PREFIX: ci-${{ github.run_id }}-${{ github.job }} jobs: example-tests: @@ -83,6 +84,10 @@ jobs: with: run: bash tests/run_examples.sh + - name: Cleanup vaults created by this run + if: always() + run: uv run python scripts/cleanup_ci_vaults.py + - name: Send Slack Notification uses: slackapi/slack-github-action@v1.24.0 with: diff --git a/.github/workflows/test-cicd.yml b/.github/workflows/test-cicd.yml index f38506938..a1927f2f1 100644 --- a/.github/workflows/test-cicd.yml +++ b/.github/workflows/test-cicd.yml @@ -38,6 +38,8 @@ env: # True for push, manual dispatch, and same-repo PRs; false for fork PRs. Steps that # need secrets are gated on this so fork PRs skip them cleanly instead of failing. IS_TRUSTED: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + # Scope vault creates to this workflow job; cleanup deletes only this prefix. + NOTTE_CI_VAULT_PREFIX: ci-${{ github.run_id }}-${{ github.job }} permissions: pull-requests: write @@ -164,6 +166,10 @@ jobs: set -o pipefail uv run pytest -n logical tests --ignore=tests/examples/test_examples.py --ignore=tests/examples/test_readme.py --durations=10 --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=packages | tee pytest-coverage.txt + - name: Cleanup vaults created by this run + if: ${{ always() && env.IS_TRUSTED == 'true' }} + run: uv run python scripts/cleanup_ci_vaults.py + - name: Pytest coverage comment if: ${{ always() && github.ref != 'refs/heads/main' && env.IS_TRUSTED == 'true' }} uses: MishaKav/pytest-coverage-comment@main diff --git a/docs/src/conftest.py b/docs/src/conftest.py new file mode 100644 index 000000000..88a18b009 --- /dev/null +++ b/docs/src/conftest.py @@ -0,0 +1,33 @@ +"""Docs snippet test hooks. + +Installs CI vault scoping when ``NOTTE_CI_VAULT_PREFIX`` is set so this workflow +run names vaults under that prefix. Deletion is done by the workflow +``Cleanup vaults created by this run`` step (not here) so pytest-xdist workers +cannot delete vaults still in use by sibling workers. +""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from pathlib import Path +from typing import Any + + +def _load_ci_vault_scope() -> Any | None: + if not os.environ.get("NOTTE_CI_VAULT_PREFIX"): + return None + path = Path(__file__).resolve().parents[2] / "scripts" / "ci_vault_scope.py" + spec = importlib.util.spec_from_file_location("ci_vault_scope", path) + if spec is None or spec.loader is None: + return None + module = importlib.util.module_from_spec(spec) + sys.modules["ci_vault_scope"] = module + spec.loader.exec_module(module) + return module + + +_SCOPE = _load_ci_vault_scope() +if _SCOPE is not None: + _SCOPE.install() diff --git a/docs/src/snippets/getting-started/concept_vault.mdx b/docs/src/snippets/getting-started/concept_vault.mdx index c64fce05c..892bfb9b3 100644 --- a/docs/src/snippets/getting-started/concept_vault.mdx +++ b/docs/src/snippets/getting-started/concept_vault.mdx @@ -2,8 +2,8 @@ {/* @sniptest testers/getting-started/concept_vault.py */} ```python concept_vault.py -vault = client.Vault() -vault.add_credentials(url="https://github.com", email="...", password="...") -agent = client.Agent(session=session, vault=vault) -agent.run(task="Login to GitHub") +with client.Session() as session, client.Vault() as vault: + vault.add_credentials(url="https://github.com", email="...", password="...") + agent = client.Agent(session=session, vault=vault) + agent.run(task="Login to GitHub") ``` diff --git a/docs/src/testers/getting-started/concept_vault.py b/docs/src/testers/getting-started/concept_vault.py index 50a9c2982..e9074c5d6 100644 --- a/docs/src/testers/getting-started/concept_vault.py +++ b/docs/src/testers/getting-started/concept_vault.py @@ -1,10 +1,9 @@ # @sniptest filename=concept_vault.py -# @sniptest show=5-9 +# @sniptest show=4-7 from notte_sdk import NotteClient client = NotteClient() -with client.Session() as session: - vault = client.Vault() +with client.Session() as session, client.Vault() as vault: vault.add_credentials(url="https://github.com", email="...", password="...") agent = client.Agent(session=session, vault=vault) agent.run(task="Login to GitHub") diff --git a/docs/src/tests/test_snippets.py b/docs/src/tests/test_snippets.py index f5b14e2bc..704e656c2 100644 --- a/docs/src/tests/test_snippets.py +++ b/docs/src/tests/test_snippets.py @@ -172,12 +172,21 @@ def decorator(func: Callable[[EvalExample, str], Any]): @handle_file("vaults/index.py") -def handle_vault( +def handle_vault_index( eval_example: EvalExample, code: str, ) -> None: - code = code.replace("", "JBSWY3DPEHPK3PXP") - run_example(eval_example, code=code) + if FAST_MODE or TYPE_CHECK_MODE: + # Syntax/type check - don't create client + code = code.replace("", "JBSWY3DPEHPK3PXP").replace("my_vault_id", "placeholder-vault-id") + run_example(eval_example, code=code) + else: + # Full mode: create real vault and always delete it via context manager + _ = load_dotenv() + client = NotteClient() + with client.Vault() as vault: + code = code.replace("", "JBSWY3DPEHPK3PXP").replace("my_vault_id", vault.vault_id) + run_example(eval_example, code=code) @handle_file("agents/index.py") @@ -236,22 +245,18 @@ def handle_workflow_fork( run_example(eval_example, code=code) -@handle_file("vaults/index.py") -def handle_vault_index( +@handle_file("getting-started/concept_vault.py") +def handle_concept_vault( eval_example: EvalExample, code: str, ) -> None: - if FAST_MODE or TYPE_CHECK_MODE: - # Syntax/type check - don't create client - code = code.replace("", "JBSWY3DPEHPK3PXP").replace("my_vault_id", "placeholder-vault-id") - run_example(eval_example, code=code) - else: - # Full mode: create real vault - _ = load_dotenv() - client = NotteClient() - with client.Vault() as vault: - code = code.replace("", "JBSWY3DPEHPK3PXP").replace("my_vault_id", vault.vault_id) - run_example(eval_example, code=code) + """Cap agent steps in execution mode; vault cleanup uses `with client.Vault()`.""" + if not (FAST_MODE or TYPE_CHECK_MODE): + code = code.replace( + "agent = client.Agent(session=session, vault=vault)", + "agent = client.Agent(session=session, vault=vault, max_steps=1)", + ) + run_example(eval_example, code=code) @handle_file("sessions/file_storage_basic.py") diff --git a/examples/landing-examples/landing_examples.py b/examples/landing-examples/landing_examples.py index ca61cd375..77ff8687d 100644 --- a/examples/landing-examples/landing_examples.py +++ b/examples/landing-examples/landing_examples.py @@ -26,31 +26,35 @@ def main(): client = NotteClient(api_key=os.getenv("NOTTE_API_KEY")) - for task, url, vault in landing_examples[3:]: + for task, url, use_vault in landing_examples[3:]: with client.Session() as session: - if vault: - vault = client.Vault() - email = os.getenv("NOTTE_VAULT_TEST_EMAIL") - assert email is not None - pwd = os.getenv("NOTTE_VAULT_TEST_PASSWORD") - assert pwd is not None - _ = vault.add_credentials( - url="https://google.com", - email=email, - password=pwd, - ) + if use_vault: + with client.Vault() as vault: + email = os.getenv("NOTTE_VAULT_TEST_EMAIL") + assert email is not None + pwd = os.getenv("NOTTE_VAULT_TEST_PASSWORD") + assert pwd is not None + _ = vault.add_credentials( + url="https://google.com", + email=email, + password=pwd, + ) + agent = client.Agent( + session=session, + reasoning_model="vertex_ai/gemini-2.0-flash", + max_steps=15, + vault=vault, + ) + run_kwargs = {"task": task, **({"url": url} if url is not None else {})} + response = agent.run(**run_kwargs) else: - vault = None - - agent_kwargs = { - "session": session, - "reasoning_model": "vertex_ai/gemini-2.0-flash", - "max_steps": 15, - **({"vault": vault} if vault is not None else {}), - } - agent = client.Agent(**agent_kwargs) - run_kwargs = {"task": task, **({"url": url} if url is not None else {})} - response = agent.run(**run_kwargs) + agent = client.Agent( + session=session, + reasoning_model="vertex_ai/gemini-2.0-flash", + max_steps=15, + ) + run_kwargs = {"task": task, **({"url": url} if url is not None else {})} + response = agent.run(**run_kwargs) if not response.success: exit(-1) diff --git a/scripts/ci_vault_scope.py b/scripts/ci_vault_scope.py new file mode 100644 index 000000000..c233cdcec --- /dev/null +++ b/scripts/ci_vault_scope.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Scope vault creates to the current CI workflow run and delete only those. + +When ``NOTTE_CI_VAULT_PREFIX`` is set (e.g. ``ci--``): +1. ``install()`` patches ``VaultsClient.create`` so unnamed/default vaults get a + unique name under that prefix and their IDs are recorded. +2. ``cleanup_this_run()`` deletes only vaults created under that prefix / recorded + for this run — never other parallel jobs' vaults. +""" + +from __future__ import annotations + +import os +import re +import sys +from pathlib import Path +from typing import Any +from uuid import uuid4 + +_VAULT_NAME_RE = re.compile(r"^[a-zA-Z0-9\s\-_]+$") +_installed = False + + +def sanitize_prefix(raw: str) -> str: + """Make a vault-name-safe prefix (API: 3-50 chars, [A-Za-z0-9 _-]).""" + cleaned = re.sub(r"[^a-zA-Z0-9_-]+", "-", raw.strip()).strip("-_") + # Leave room for "-xxxxxxxx" suffix (9 chars); API max name length is 50. + cleaned = cleaned[:41].rstrip("-_") + if len(cleaned) < 3: + cleaned = f"ci-{cleaned}" if cleaned else "ci-run" + return cleaned + + +def run_prefix() -> str | None: + raw = os.environ.get("NOTTE_CI_VAULT_PREFIX", "").strip() + if not raw: + return None + return sanitize_prefix(raw) + + +def _ids_file(prefix: str) -> Path: + base = Path(os.environ.get("RUNNER_TEMP") or os.environ.get("TMPDIR") or "/tmp") + safe = re.sub(r"[^a-zA-Z0-9_-]+", "-", prefix) + return base / f"notte-ci-vault-ids-{safe}.txt" + + +def record_vault_id(prefix: str, vault_id: str) -> None: + path = _ids_file(prefix) + path.parent.mkdir(parents=True, exist_ok=True) + # Append is atomic enough per line on POSIX for concurrent xdist workers. + with path.open("a", encoding="utf-8") as handle: + _ = handle.write(f"{vault_id}\n") + handle.flush() + os.fsync(handle.fileno()) + + +def recorded_vault_ids(prefix: str) -> list[str]: + path = _ids_file(prefix) + if not path.exists(): + return [] + return [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def install() -> None: + """Patch vault create so this CI run owns a unique name prefix.""" + global _installed + prefix = run_prefix() + if prefix is None or _installed: + return + + from notte_sdk.endpoints.vaults import VaultsClient + + original_create = VaultsClient.create + + def create(self: Any, **data: Any) -> Any: + name = data.get("name") + if name is None or name == "default": + candidate = f"{prefix}-{uuid4().hex[:8]}" + if not _VAULT_NAME_RE.match(candidate): + candidate = re.sub(r"[^a-zA-Z0-9_-]+", "-", candidate) + data = {**data, "name": candidate} + vault = original_create(self, **data) + vault_id = getattr(vault, "vault_id", None) + if isinstance(vault_id, str) and vault_id: + record_vault_id(prefix, vault_id) + return vault + + VaultsClient.create = create + _installed = True + print(f"[ci-vault-scope] installed create patch prefix={prefix!r}", file=sys.stderr) + + +def list_active_vaults(client: Any, *, page_size: int = 100) -> list[Any]: + vaults: list[Any] = [] + page = 1 + while True: + batch = list(client.vaults.list(page=page, page_size=page_size, only_active=True)) + if not batch: + break + vaults.extend(batch) + if len(batch) < page_size: + break + page += 1 + return vaults + + +def cleanup_this_run(*, dry_run: bool = False, prefix: str | None = None) -> int: + """Delete only vaults created by this workflow run (prefix + recorded IDs).""" + resolved = sanitize_prefix(prefix) if prefix else run_prefix() + if resolved is None: + print("No NOTTE_CI_VAULT_PREFIX / --prefix; nothing to clean.", file=sys.stderr) + return 0 + + from notte_sdk import NotteClient + + client = NotteClient() + recorded = set(recorded_vault_ids(resolved)) + listed = list_active_vaults(client) + by_prefix = { + vault.vault_id + for vault in listed + if str(getattr(vault, "name", "") or "").startswith(resolved) and not bool(getattr(vault, "for_persona", False)) + } + targets = sorted(recorded | by_prefix) + + print( + f"[ci-vault-scope] prefix={resolved!r} recorded={len(recorded)} " + + f"by_prefix={len(by_prefix)} deleting={len(targets)} dry_run={dry_run}" + ) + if dry_run: + for vault_id in targets: + print(f" [dry-run] would delete {vault_id}") + return 0 + + deleted = 0 + failed = 0 + already_gone = 0 + for vault_id in targets: + try: + _ = client.vaults.delete(vault_id) + deleted += 1 + print(f" deleted {vault_id}") + except Exception as exc: # noqa: BLE001 - best-effort run teardown + if _is_already_deleted_error(exc): + already_gone += 1 + print(f" already gone {vault_id}") + continue + failed += 1 + print(f" failed {vault_id}: {exc}", file=sys.stderr) + + print(f"[ci-vault-scope] done deleted={deleted} already_gone={already_gone} failed={failed}") + return 0 if failed == 0 else 1 + + +def _is_already_deleted_error(exc: BaseException) -> bool: + text = str(exc).lower() + return "not active" in text or "not found" in text or ("already" in text and "deleted" in text) + + +def cleanup_orphan_defaults(*, dry_run: bool, min_age_hours: float) -> int: + """One-shot drain of leaked name=default vaults older than min_age_hours.""" + import datetime as dt + + from notte_sdk import NotteClient + + if min_age_hours < 0: + print("--min-age-hours must be >= 0", file=sys.stderr) + return 2 + + client = NotteClient() + now = dt.datetime.now(tz=dt.timezone.utc) + min_age = dt.timedelta(hours=min_age_hours) + targets: list[Any] = [] + for vault in list_active_vaults(client): + if bool(getattr(vault, "for_persona", False)): + continue + if str(getattr(vault, "name", "") or "") != "default": + continue + created_at = getattr(vault, "created_at", None) + if created_at is None: + continue + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=dt.timezone.utc) + if (now - created_at) < min_age: + continue + targets.append(vault) + + print(f"[ci-vault-scope] orphan default vaults older than {min_age_hours}h: {len(targets)}") + if dry_run: + for vault in targets: + print(f" [dry-run] would delete {vault.vault_id} created_at={vault.created_at}") + return 0 + + deleted = 0 + failed = 0 + already_gone = 0 + for vault in targets: + try: + _ = client.vaults.delete(vault.vault_id) + deleted += 1 + print(f" deleted {vault.vault_id}") + except Exception as exc: # noqa: BLE001 + if _is_already_deleted_error(exc): + already_gone += 1 + print(f" already gone {vault.vault_id}") + continue + failed += 1 + print(f" failed {vault.vault_id}: {exc}", file=sys.stderr) + print(f"[ci-vault-scope] orphan drain done deleted={deleted} already_gone={already_gone} failed={failed}") + return 0 if failed == 0 else 1 diff --git a/scripts/cleanup_ci_vaults.py b/scripts/cleanup_ci_vaults.py new file mode 100644 index 000000000..a0f44a3ad --- /dev/null +++ b/scripts/cleanup_ci_vaults.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""CLI to delete vaults owned by a single CI workflow run. + +Preferred usage (from a workflow ``if: always()`` step):: + + NOTTE_CI_VAULT_PREFIX=ci-$GITHUB_RUN_ID-$GITHUB_JOB \\ + python scripts/cleanup_ci_vaults.py + +Only vaults created under that prefix / recorded by ``ci_vault_scope.install()`` +are deleted — parallel jobs sharing the same API key are left alone. + +One-shot backlog drain (manual only):: + + python scripts/cleanup_ci_vaults.py --orphan-defaults --min-age-hours 2 +""" + +from __future__ import annotations + +import argparse +import importlib.util +from pathlib import Path + + +def _load_scope() -> object: + path = Path(__file__).resolve().parent / "ci_vault_scope.py" + spec = importlib.util.spec_from_file_location("ci_vault_scope", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"Unable to load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + _ = parser.add_argument( + "--prefix", + default=None, + help="Vault name prefix for this run (default: $NOTTE_CI_VAULT_PREFIX)", + ) + _ = parser.add_argument("--dry-run", action="store_true", help="List vaults without deleting") + _ = parser.add_argument( + "--orphan-defaults", + action="store_true", + help="One-shot: delete name=default vaults older than --min-age-hours (backlog drain)", + ) + _ = parser.add_argument( + "--min-age-hours", + type=float, + default=2.0, + help="Age cutoff for --orphan-defaults (default: 2)", + ) + args = parser.parse_args() + + scope = _load_scope() + if args.orphan_defaults: + cleanup_orphans = getattr(scope, "cleanup_orphan_defaults") + return int(cleanup_orphans(dry_run=args.dry_run, min_age_hours=args.min_age_hours)) + + cleanup = getattr(scope, "cleanup_this_run") + return int(cleanup(dry_run=args.dry_run, prefix=args.prefix)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/conftest.py b/tests/conftest.py index 0a81382aa..908c3d251 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,8 @@ +import importlib.util import os +import sys from pathlib import Path +from typing import Any import notte_core @@ -14,6 +17,24 @@ os.environ["DISABLE_GPU"] = "true" +def _load_ci_vault_scope() -> Any | None: + if not os.environ.get("NOTTE_CI_VAULT_PREFIX"): + return None + path = Path(__file__).resolve().parents[1] / "scripts" / "ci_vault_scope.py" + spec = importlib.util.spec_from_file_location("ci_vault_scope", path) + if spec is None or spec.loader is None: + return None + module = importlib.util.module_from_spec(spec) + sys.modules["ci_vault_scope"] = module + spec.loader.exec_module(module) + return module + + +_CI_VAULT_SCOPE = _load_ci_vault_scope() +if _CI_VAULT_SCOPE is not None: + _CI_VAULT_SCOPE.install() + + # Flaky test configuration: # Tests marked with @pytest.mark.flaky(reruns=N, reruns_delay=S) will automatically # retry N times with S seconds delay between retries when they fail. diff --git a/tests/integration/sdk/test_vault.py b/tests/integration/sdk/test_vault.py index 91cfbb7bb..d26f513ef 100644 --- a/tests/integration/sdk/test_vault.py +++ b/tests/integration/sdk/test_vault.py @@ -34,17 +34,15 @@ async def fulfill_github_login(route): # pyright: ignore[reportUnknownParameter def test_vault_in_local_agent(): _ = load_dotenv() client = NotteClient(api_key=os.getenv("NOTTE_API_KEY")) - vault = client.Vault() - _ = vault.add_credentials( - url="https://github.com/", - email="xyz@notte.cc", - password="xyz", - ) - with notte.Session() as session: - agent = notte.Agent(session=session, vault=vault, max_steps=5) - _ = agent.run(task="Go to the github.com and try to login with the credentials") - - _ = client.vaults.delete(vault.vault_id) + with client.Vault() as vault: + _ = vault.add_credentials( + url="https://github.com/", + email="xyz@notte.cc", + password="xyz", + ) + with notte.Session() as session: + agent = notte.Agent(session=session, vault=vault, max_steps=5) + _ = agent.run(task="Go to the github.com and try to login with the credentials") @pytest.mark.asyncio