diff --git a/backend/app/app_jobs.py b/backend/app/app_jobs.py index e6465c4a3..8b6df319f 100644 --- a/backend/app/app_jobs.py +++ b/backend/app/app_jobs.py @@ -26,14 +26,29 @@ def runner_script() -> Path: return Path(__file__).resolve().parent.parent / "scripts" / "app-job-runner.py" -def runner_command(app_id: int, job_path: Path) -> list[str]: - return [sys.executable, str(runner_script()), str(app_id), str(job_path)] +def runner_command( + app_id: int, job_path: Path, *, wait_for_ready: bool = False, +) -> list[str]: + """Build the common supervisor command for one app job. + Bootstrap installs happen inside FastAPI's lifespan, before the server can + answer the capability calls the supervisor makes. Only that launch path + needs to wait for the already-defined readiness contract; ordinary cron and + manual jobs run against an already-serving backend. + """ + command = [sys.executable, str(runner_script())] + if wait_for_ready: + command.append("--wait-for-ready") + command.extend((str(app_id), str(job_path))) + return command -def launch_app_job(app_id: int, job_path: Path, source_dir: Path): + +def launch_app_job( + app_id: int, job_path: Path, source_dir: Path, *, wait_for_ready: bool = False, +): """Launch the common wrapper detached from the API worker's pipes.""" return subprocess.Popen( - runner_command(app_id, job_path), + runner_command(app_id, job_path, wait_for_ready=wait_for_ready), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=str(source_dir), diff --git a/backend/app/install.py b/backend/app/install.py index 4e252448a..10afc0931 100644 --- a/backend/app/install.py +++ b/backend/app/install.py @@ -3348,9 +3348,19 @@ async def install_from_manifest( ): try: from app.app_jobs import launch_app_job - source = Path(app.source_dir) - launch_app_job(app.id, source / job_name, source) - warnings.append("initialization started") + source_dir = Path(app.source_dir) + # Bootstrap runs inside FastAPI lifespan, before this backend can answer + # the supervisor's scoped capability calls. Keep that ordering detail in + # the generic runner: it waits for the existing readiness signal before + # starting. Interactive installs already happen against a live server. + wait_for_ready = source == "bootstrap" + launch_app_job( + app.id, source_dir / job_name, source_dir, wait_for_ready=wait_for_ready, + ) + warnings.append( + "initialization waiting for startup readiness" + if wait_for_ready else "initialization started" + ) except Exception as exc: log.exception("install: initialization job failed to start") warnings.append(f"initialization failed to start — {exc!r}") diff --git a/backend/scripts/app-job-runner.py b/backend/scripts/app-job-runner.py index bdfa6c623..4d632e1da 100755 --- a/backend/scripts/app-job-runner.py +++ b/backend/scripts/app-job-runner.py @@ -10,6 +10,7 @@ import subprocess import sys import tempfile +import time import urllib.request import uuid from pathlib import Path @@ -29,6 +30,7 @@ # cron never restarts the container for us. SUPERVISOR_LOG = DATA_DIR / "cron-logs" / "app-jobs.log" SUPERVISOR_LOG_CAP = 2 * 1024 * 1024 +READY_WAIT_SECONDS = 90 def _log(app_id: object, message: str) -> None: @@ -52,6 +54,29 @@ def _start_ticks(pid: int) -> int: return int(tail[19]) +def _wait_for_ready(timeout_seconds: int = READY_WAIT_SECONDS) -> bool: + """Wait only for the platform startup dependency bootstrap jobs require. + + A bootstrap install runs during FastAPI lifespan, while the app-job runner + needs the backend to mint a scoped token and return job context. `/api/ready` + is the platform's existing readiness contract; polling it here avoids a + startup ordering race without adding a second scheduler or retry system. + """ + deadline = time.monotonic() + max(0, timeout_seconds) + while True: + try: + request = urllib.request.Request(f"{API_BASE_URL}/api/ready") + with urllib.request.urlopen(request, timeout=2) as response: + if response.status == 200: + return True + except Exception: + pass + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + time.sleep(min(1, remaining)) + + def _atomic_json(path: Path, value: dict) -> None: path.parent.mkdir(parents=True, exist_ok=True) fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=".lease-", suffix=".tmp") @@ -243,11 +268,15 @@ def _sandboxed_command( def run() -> int: - if len(sys.argv) != 3 or not re.fullmatch(r"[0-9]+", sys.argv[1]): - _log(sys.argv[1] if len(sys.argv) > 1 else "?", "rejected: bad argv") + argv = sys.argv[1:] + wait_for_ready = argv[:1] == ["--wait-for-ready"] + if wait_for_ready: + argv = argv[1:] + if len(argv) != 2 or not re.fullmatch(r"[0-9]+", argv[0]): + _log(argv[0] if argv else "?", "rejected: bad argv") return 2 - app_id = int(sys.argv[1]) - job = Path(sys.argv[2]) + app_id = int(argv[0]) + job = Path(argv[1]) if job.is_symlink(): _log(app_id, f"rejected: symlinked job {job}") return 2 @@ -283,6 +312,9 @@ def run() -> int: "job": str(resolved), }) try: + if wait_for_ready and not _wait_for_ready(): + _log(app_id, "failed: timed out waiting for platform readiness") + return 4 app_token = _mint_app_token(app_id) if not app_token: _log(app_id, "failed: could not mint app token (backend down or bad service token)") diff --git a/backend/tests/test_app_capabilities.py b/backend/tests/test_app_capabilities.py index 95abf032b..e292e8a2a 100644 --- a/backend/tests/test_app_capabilities.py +++ b/backend/tests/test_app_capabilities.py @@ -1,6 +1,8 @@ """Owner-reviewable app capability contracts and install binding.""" import json + +import pytest from pathlib import Path from unittest.mock import patch @@ -207,6 +209,46 @@ def stream(self, method, url, **kwargs): assert db.query(models.App).count() == 0 + + +@pytest.mark.asyncio +async def test_bootstrap_initialization_waits_for_backend_readiness( + db, bypass_url_validation, +): + """Bootstrap and interactive installs share launch ownership but not timing.""" + from app.install import install_from_manifest + + base = "https://capability.test/bootstrap-memory/" + manifest = _manifest(id="bootstrap-memory", name="Bootstrap Memory") + _contract, digest = contract_and_digest(manifest) + responses = { + base + "mobius.json": (200, json.dumps(manifest).encode()), + base + "index.jsx": (200, JSX.encode()), + base + "memory-core.md": (200, b"Retrieve memory only on demand."), + base + "memory-job.sh": (200, b"#!/bin/sh\nexit 0\n"), + } + with patch( + "app.install.httpx.AsyncClient", + side_effect=_fake_async_client(responses), + ), patch("app.app_jobs.launch_app_job") as launch: + app, mode, warnings, *_rest = await install_from_manifest( + db, + base + "mobius.json", + None, + None, + source="bootstrap", + reviewed_capability_digest=digest, + ) + + assert mode == "install" + source_dir = Path(app.source_dir) + launch.assert_called_once_with( + app.id, source_dir / "memory-job.sh", source_dir, + wait_for_ready=True, + ) + assert "initialization waiting for startup readiness" in warnings + + def test_matching_digest_is_persisted_with_explicit_system_identity( client, auth, db, bypass_url_validation, ): @@ -237,5 +279,8 @@ def test_matching_digest_is_persisted_with_explicit_system_identity( assert app.system_app is True assert app.capability_contract == contract assert response.json()["capability_contract"] == contract - launch.assert_called_once() + launch.assert_called_once_with( + app.id, Path(app.source_dir) / "memory-job.sh", Path(app.source_dir), + wait_for_ready=False, + ) assert "initialization started" in response.json()["warnings"] diff --git a/backend/tests/test_app_jobs.py b/backend/tests/test_app_jobs.py index 523778e62..7c1e7cb73 100644 --- a/backend/tests/test_app_jobs.py +++ b/backend/tests/test_app_jobs.py @@ -29,6 +29,15 @@ def test_cron_parser_resolves_supervised_command_to_real_job(): ) +def test_only_bootstrap_commands_request_a_readiness_wait(): + job = Path("/data/apps/memory/memory-job.sh") + + assert app_jobs.runner_command(57, job, wait_for_ready=True)[-3:] == [ + "--wait-for-ready", "57", str(job), + ] + assert app_jobs.runner_command(57, job)[-2:] == ["57", str(job)] + + def test_terminate_verifies_start_ticks_before_signalling(monkeypatch): data_dir = Path(get_settings().data_dir) leases = data_dir / "run" / "app-jobs" / "57" @@ -109,6 +118,61 @@ def urlopen(request, timeout): } +def test_bootstrap_waits_for_ready_before_minting_a_job_token( + tmp_path, monkeypatch, +): + runner = _load_runner() + data_dir = tmp_path / "data" + source = data_dir / "apps" / "memory" + source.mkdir(parents=True) + job = source / "memory-job.sh" + job.write_text("#!/bin/sh\nexit 0\n") + monkeypatch.setattr(runner, "DATA_DIR", data_dir) + monkeypatch.setattr(runner.os, "getsid", lambda _pid: os.getpid()) + events = [] + monkeypatch.setattr( + runner, "_wait_for_ready", lambda: events.append("ready") or True, + ) + monkeypatch.setattr( + runner, "_mint_app_token", lambda _app_id: events.append("mint") or "token", + ) + monkeypatch.setattr(runner, "_app_is_live", lambda *_args: True) + monkeypatch.setattr( + runner, "_job_context", lambda *_args: {"source_dir": str(source)}, + ) + monkeypatch.setattr( + runner.subprocess, "Popen", lambda *_args, **_kwargs: types.SimpleNamespace(wait=lambda: 0), + ) + monkeypatch.setattr(runner.sys, "argv", [ + "app-job-runner.py", "--wait-for-ready", "57", str(job), + ]) + + assert runner.run() == 0 + assert events == ["ready", "mint"] + + +def test_bootstrap_readiness_timeout_never_mints_a_job_token( + tmp_path, monkeypatch, +): + runner = _load_runner() + data_dir = tmp_path / "data" + source = data_dir / "apps" / "memory" + source.mkdir(parents=True) + job = source / "memory-job.sh" + job.write_text("#!/bin/sh\nexit 0\n") + monkeypatch.setattr(runner, "DATA_DIR", data_dir) + monkeypatch.setattr(runner.os, "getsid", lambda _pid: os.getpid()) + monkeypatch.setattr(runner, "_wait_for_ready", lambda: False) + minted = [] + monkeypatch.setattr(runner, "_mint_app_token", lambda _app_id: minted.append(True)) + monkeypatch.setattr(runner.sys, "argv", [ + "app-job-runner.py", "--wait-for-ready", "57", str(job), + ]) + + assert runner.run() == 4 + assert minted == [] + + def test_wrapper_publishes_lease_before_live_check_and_cleans_it( tmp_path, monkeypatch, ):