diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 938a58c27..ccdd364b7 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -11,6 +11,11 @@ on: - 'delphi/**/*.py' - 'delphi/requirements*.txt' - 'delphi/Dockerfile' + - 'scripts/*install.sh' + - 'scripts/application_stop.sh' + - 'scripts/validate_service.sh' + - 'scripts/test-deploy-hooks.sh' + - 'docker-compose*.yml' - '.github/workflows/python-ci.yml' # server/src so a new server-side wildcard runs the projection-gate sweep - 'server/src/**' @@ -105,13 +110,8 @@ jobs: echo "Copying script to be tested into container..." docker compose -f docker-compose.test.yml cp delphi/polismath/run_math_pipeline.py delphi:/app/run_math_pipeline.py docker compose -f docker-compose.test.yml cp delphi/umap_narrative delphi:/app/umap_narrative - echo "Copying the compose files into container..." - # tests/test_compose_math_env.py asserts that every stack's delphi service - # receives the MATH_ENV its math service writes under. /app/tests has no - # checkout above it, so put the two files it parses beside it; without - # them the test skips instead of guarding the report pipeline's math_env. - docker compose -f docker-compose.test.yml cp docker-compose.yml delphi:/app/docker-compose.yml - docker compose -f docker-compose.test.yml cp docker-compose.test.yml delphi:/app/docker-compose.test.yml + # The same container-layout preparation and hook tests run on mm5. + bash scripts/test-deploy-hooks.sh # Wire a checkout-shaped root so delphi/tests/scripts collects AND RUNS the # pure-Python inventory sweep + the DB-free gate unit tests (the conftest @@ -127,9 +127,6 @@ jobs: # its first item (observed: only server/src copied). # Recordings tests also load the packer and its digest helper from ci/. # battery_coverage.py is already included in the declared delphi/scripts. - # test_compose_math_env.py reads the deploy hook's per-role compose lines. - # test_before_install_hook.py runs scripts/before_install.sh against a fake docker. - # test_compose_math_env.py also reads the stop hook's per-role stop lines. # Representative payload tests import the box planner/gate and their # helpers from this same checkout root; keep their relative paths intact. # Light-shadow triage tests also import the compare/triage modules. @@ -141,8 +138,7 @@ jobs: ci/private_cert/image_admission.py ci/probe_box/receipt.py \ ci/probe_box/contracts.py ci/probe_box/light_shadow.py ci/probe_box/light_shadow_queries.py \ ci/private_cert/images/light_shadow_compare.py ci/private_cert/images/light_shadow_triage.py \ - ci/private_cert/images/recipe.py \ - scripts/after_install.sh scripts/before_install.sh scripts/application_stop.sh; do + ci/private_cert/images/recipe.py; do docker compose -f docker-compose.test.yml exec -T delphi mkdir -p "/app/projgate/$(dirname "$rel")" docker compose -f docker-compose.test.yml cp "$rel" "delphi:/app/projgate/$rel" \ || { echo "failed to copy scan input: $rel"; exit 1; } diff --git a/appspec.yml b/appspec.yml index bee1617e7..332da07e0 100644 --- a/appspec.yml +++ b/appspec.yml @@ -23,4 +23,8 @@ hooks: ApplicationStop: - location: scripts/application_stop.sh timeout: 300 - runas: root \ No newline at end of file + runas: root + ValidateService: + - location: scripts/validate_service.sh + timeout: 1200 + runas: root diff --git a/delphi/tests/test_after_install_hook.py b/delphi/tests/test_after_install_hook.py index cceaa0a13..c9f86a350 100644 --- a/delphi/tests/test_after_install_hook.py +++ b/delphi/tests/test_after_install_hook.py @@ -1,4 +1,4 @@ -"""scripts/after_install.sh selects one branch per service type, and a queue +"""scripts/after_install.sh migrates before replacing any service, and a queue worker box (delphi-large, delphi-worker) starts only its polis-jobs.service. Runs the real deploy hook (and the stop hook) with every external command @@ -75,6 +75,7 @@ def _find(rel): ' *) echo \'{"username":"u","password":"p","dbname":"d"}\';;\n' 'esac\n'), "docker": RECORD + ( + 'if [ "$1" = "$FAKE_MIGRATION_FAILURE" ]; then exit 17; fi\n' 'if [ "$1" = ps ]; then\n' ' filter=""\n' ' while [ $# -gt 0 ]; do [ "$1" = --filter ] && filter="${2#name=}"; shift; done\n' @@ -124,9 +125,10 @@ def _run(tmp_path, script_path, rules, root, env): return proc, log -def _deploy(tmp_path, service_type, **kw): +def _deploy(tmp_path, service_type, migration_failure="", **kw): root, env = _box(tmp_path, service_type, **kw) - rules = [("/usr/local/bin/docker-compose", "docker-compose", 4), + env["FAKE_MIGRATION_FAILURE"] = migration_failure + rules = [("/usr/local/bin/docker-compose", "docker-compose", 3), ("/etc/app-info/", f"{root}/etc/app-info/", 2), ("/etc/systemd/system/", f"{root}/etc/systemd/system/", 0), ("/opt/polis", f"{root}/opt/polis", 2)] @@ -145,10 +147,10 @@ def _removed(log): ALL_IDS = {i for i, _ in CONTAINERS} UNCHANGED = { - "server": ["up -d server nginx-proxy client-participation-alpha --build --force-recreate"], - "delphi": ["up -d delphi math-python --build --force-recreate"], + "server": ["up -d server nginx-proxy client-participation-alpha --no-build --force-recreate"], + "delphi": ["up -d delphi math-python --no-build --force-recreate"], "math": [], - "ollama": ["up -d --build --force-recreate"], # unknown type: the start-everything catch-all + } @@ -158,9 +160,12 @@ def test_the_other_service_types_behave_as_before(tmp_path, service_type): assert proc.returncode == 0, proc.stdout + proc.stderr compose = _calls(log, "docker-compose") assert [c for c in compose if c.startswith("up")] == UNCHANGED[service_type] - assert not [c for c in compose if c.startswith("build")] + builds = [c for c in compose if c.startswith("build")] + assert builds == {"server": ["build server nginx-proxy client-participation-alpha"], "delphi": ["build delphi math-python"], "math": []}[service_type] + if builds: + assert compose.index(builds[0]) < compose.index("down") < compose.index(UNCHANGED[service_type][0]) assert _calls(log, "systemctl") == [] - assert _removed(log) == ALL_IDS + assert not [c for c in _calls(log, "docker") if c.startswith(("rm ", "system prune"))] @pytest.mark.parametrize("service_type,worker_class", [("delphi-large", "large"), @@ -176,7 +181,7 @@ def test_a_worker_box_restarts_only_its_daemon_unit(tmp_path, service_type, work assert log.index("docker-compose build delphi") < log.index( "systemctl restart --no-block polis-jobs.service") # The cleanup spares the running daemon; the restart drains it. - assert _removed(log) == ALL_IDS - {JOBS_ID} + assert not [c for c in _calls(log, "docker") if c.startswith(("rm ", "system prune"))] assert f"worker class '{worker_class}'" in proc.stdout @@ -206,17 +211,18 @@ def test_a_worker_box_that_does_not_match_fails_the_deploy(tmp_path, service_typ assert _calls(log, "systemctl") == [] -@pytest.mark.parametrize("service_type,stops", [("delphi", ["stop delphi"]), - ("delphi-large", []), ("delphi-worker", [])]) -def test_the_stop_hook_stops_no_worker_service(tmp_path, service_type, stops): +@pytest.mark.parametrize("service_type", ["server", "delphi", "math", "delphi-large", "delphi-worker", "unknown"]) +def test_the_stop_hook_leaves_every_service_running(tmp_path, service_type): root, env = _box(tmp_path, service_type) - (root / "opt/polis/polis").mkdir(parents=True) - # command -v needs the compose path to exist; point it at the fake. - rules = [("/usr/local/bin/docker-compose", str(tmp_path / "bin/docker-compose"), 2), - ("/etc/app-info/", f"{root}/etc/app-info/", 1), - ("/opt/polis/polis", f"{root}/opt/polis/polis", 1)] - proc, log = _run(tmp_path, STOP_PATH, rules, root, env) + # No path rewrite: the revised hook has no external calls or absolute paths. + proc, log = _run(tmp_path, STOP_PATH, [], root, env) assert proc.returncode == 0, proc.stdout + proc.stderr - assert [c for c in _calls(log, "docker-compose") if c.startswith("stop")] == stops - assert _calls(log, "systemctl") == [] - assert "Unknown service type" not in proc.stdout + proc.stderr + assert log == [] + assert "AfterInstall" in proc.stdout + + +def test_unknown_role_refuses_before_migration_or_replacement(tmp_path): + proc, log = _deploy(tmp_path, "unknown") + assert proc.returncode != 0 + assert not _calls(log, "docker") + assert not _calls(log, "docker-compose") diff --git a/delphi/tests/test_before_install_hook.py b/delphi/tests/test_before_install_hook.py index a9dbbd3b2..71a36998e 100644 --- a/delphi/tests/test_before_install_hook.py +++ b/delphi/tests/test_before_install_hook.py @@ -1,20 +1,10 @@ -"""scripts/before_install.sh stops only the exact containers it names. +"""BeforeInstall leaves every healthy service running until AfterInstall migrates. -Docker's ``name`` filter is an unanchored regex, so ``--filter name=polis-math`` -also matched the Delphi box's ``polis-math-python-1``; the hook then ran -``docker stop polis-math-1``, which exists only on the math box, and the -production BeforeInstall failed with ``No such container: polis-math-1``. - -Runs the real hook against a fake ``docker`` on PATH that applies the filter -the way the daemon does (regex search against the name, with and without its -leading ``/``) and fails ``stop`` for a container that is not running. The hook -is located like ``test_compose_math_env.py`` locates after_install.sh: -$POLIS_CHECKOUT_DIR, else an ancestor of this file (CI copies it into the -checkout-shaped root). +Run the exact hook with forbidden external commands on PATH. A reintroduced +stop, removal, daemon restart or migration here must fail these controls. """ import os -import re import shutil import subprocess from pathlib import Path @@ -45,117 +35,23 @@ def _find_before_install(): pytest.mark.skipif(shutil.which("bash") is None, reason="bash not available"), ] -FAKE_DOCKER = r"""#!/bin/bash -# Fake docker: running containers come from $FAKE_DOCKER_RUNNING. -echo "$*" >> "$FAKE_DOCKER_LOG" -case "$1" in - ps) - filter="" - while [ $# -gt 0 ]; do - if [ "$1" = "--filter" ]; then filter="$2"; shift; fi - shift - done - regex="${filter#name=}" - for n in $FAKE_DOCKER_RUNNING; do - if printf '%s\n' "$n" | grep -Eq -- "$regex" || printf '%s\n' "/$n" | grep -Eq -- "$regex"; then - echo "id-$n" - fi - done - ;; - stop) - for n in $FAKE_DOCKER_RUNNING; do - if [ "$n" = "$2" ]; then echo "$2"; exit 0; fi - done - echo "Error response from daemon: No such container: $2" >&2 - exit 1 - ;; - *) - echo "fake docker: unexpected command: $*" >&2 - exit 2 - ;; -esac -""" - - -@pytest.fixture -def fake_docker(tmp_path): +@pytest.mark.parametrize("running", [ + "polis-math-python-1", "polis-delphi-1 polis-math-python-1", + "polis-math-python-large-1 polis-jobs", "polis-math-1", + "polis-server-1 polis-server-helper-1", "", +]) +def test_before_install_never_touches_running_services(tmp_path, running): bin_dir = tmp_path / "bin" bin_dir.mkdir() - docker = bin_dir / "docker" - docker.write_text(FAKE_DOCKER) - docker.chmod(0o755) - log = tmp_path / "docker.log" - log.write_text("") - - def run(*running, script=None): - env = dict(os.environ) - env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" - env["FAKE_DOCKER_RUNNING"] = " ".join(running) - env["FAKE_DOCKER_LOG"] = str(log) - log.write_text("") - if script is None: - args = ["bash", str(BEFORE_INSTALL_PATH)] - else: - args = ["bash", "-c", script] - result = subprocess.run(args, env=env, capture_output=True, text=True, timeout=30) - stops = [line.split()[1] for line in log.read_text().splitlines() if line.startswith("stop ")] - return result, stops - - return run - - -def test_fake_docker_filters_by_substring_like_the_daemon(fake_docker): - # Guards the fake: an unanchored filter must reproduce the production failure. - result, _ = fake_docker( - "polis-math-python-1", - script="docker ps -q --filter name=polis-math", - ) - assert result.stdout.strip() == "id-polis-math-python-1" - - -def test_delphi_box_with_math_python_stops_nothing_math(fake_docker): - result, stops = fake_docker("polis-math-python-1") - assert result.returncode == 0, result.stderr - assert stops == [] - - -def test_delphi_box_stops_delphi_but_not_math_python(fake_docker): - result, stops = fake_docker("polis-delphi-1", "polis-math-python-1") - assert result.returncode == 0, result.stderr - assert stops == ["polis-delphi-1"] - - -def test_large_box_stops_nothing(fake_docker): - # A container no anchored filter names (here the former large poller's - # name, which no compose file defines any more) is left alone: the - # large box's queue worker is not this hook's to stop. - result, stops = fake_docker("polis-math-python-large-1") - assert result.returncode == 0, result.stderr - assert stops == [] - - -def test_math_box_stops_math(fake_docker): - result, stops = fake_docker("polis-math-1") - assert result.returncode == 0, result.stderr - assert stops == ["polis-math-1"] - - -def test_server_box_stops_server(fake_docker): - result, stops = fake_docker("polis-server-1", "polis-server-helper-1") - assert result.returncode == 0, result.stderr - assert stops == ["polis-server-1"] - - -def test_no_containers_is_a_clean_exit(fake_docker): - result, stops = fake_docker() - assert result.returncode == 0, result.stderr - assert stops == [] - - -def test_every_name_filter_is_anchored_to_the_container_it_stops(): - text = BEFORE_INSTALL_PATH.read_text() - filters = re.findall(r'--filter "name=([^"]*)"', text) - stopped = re.findall(r"docker stop (\S+)", text) - assert filters, "before_install.sh no longer filters by name" - assert "--filter name=" not in text and "--filter 'name=" not in text - assert [f"^/?{name}$" for name in stopped] == filters + log = tmp_path / "calls" + for name in ("docker", "docker-compose", "systemctl", "sudo", "polis-migrate"): + stub = bin_dir / name + stub.write_text('#!/bin/sh\necho "$0 $*" >> "$FAKE_LOG"\nexit 91\n') + stub.chmod(0o755) + env = {"PATH": f"{bin_dir}:/usr/bin:/bin", "FAKE_LOG": str(log), + "FAKE_DOCKER_RUNNING": running} + result = subprocess.run(["bash", str(BEFORE_INSTALL_PATH)], env=env, + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + assert not log.exists(), "BeforeInstall must not invoke service or migration commands" + assert "AfterInstall" in result.stdout diff --git a/delphi/tests/test_compose_math_env.py b/delphi/tests/test_compose_math_env.py index 951058e45..ca137fef2 100644 --- a/delphi/tests/test_compose_math_env.py +++ b/delphi/tests/test_compose_math_env.py @@ -16,7 +16,8 @@ ``docker compose config`` would print. The checkout is located by walking up from this file (``POLIS_CHECKOUT_DIR`` overrides), because the CI job copies ``delphi/tests`` into the delphi image at ``/app/tests``, where the repo root is -not an ancestor — ``python-ci.yml`` copies the two compose files in beside it. +not an ancestor. ``scripts/test-deploy-hooks.sh`` copies the compose files, +hooks and Dockerfile into the explicit checkout root in both CI and mm5. """ import os @@ -279,7 +280,13 @@ def test_delphi_role_starts_delphi_and_the_shadow_poller(): assert len(roles.get("delphi", [])) == 1, "the delphi role must have exactly one compose up line" (line,) = roles["delphi"] assert _services_named(line) == {"delphi", "math-python"} - assert {"-d", "--build", "--force-recreate"} <= set(line) + assert {"-d", "--no-build", "--force-recreate"} <= set(line) + assert "--build" not in line + # Release A builds while the old services still serve, then replaces them. + body = _role_bodies()["delphi"] + build = "sudo /usr/local/bin/docker-compose build delphi math-python" + down = "sudo /usr/local/bin/docker-compose down" + assert body.index(build) < body.index(down) < body.index(" ".join(line)) @requires_after_install @@ -474,9 +481,10 @@ def test_delphi_llm_selection_defaults_when_unset(): # Datadog agent listening the tracer only logs failed sends, so the default is # off; the ddtrace package stays installed so tracing can be switched back on. -DOCKERFILE = Path(__file__).resolve().parent.parent / "Dockerfile" +DOCKERFILE = (CHECKOUT / "delphi/Dockerfile" if CHECKOUT is not None + else Path(__file__).resolve().parent.parent / "Dockerfile") requires_dockerfile = pytest.mark.skipif( - not DOCKERFILE.is_file(), reason=f"{DOCKERFILE} not found (CI copies only tests/)" + not DOCKERFILE.is_file(), reason=f"{DOCKERFILE} not found" ) @@ -640,7 +648,6 @@ def test_there_is_no_large_poller_service_and_no_manifest(): APPLICATION_STOP = Path("scripts") / "application_stop.sh" _ROLE_BLOCK = re.compile(r'^(?:if|elif) \[ "\$SERVICE_FROM_FILE" == "(?P[a-z-]+)" \]; then$') -_STOP_BRANCH = re.compile(r'^ (?:if|elif) \[ "\$SERVICE_TYPE" == "(?P[a-z-]+)" \]; then$') def _role_bodies() -> dict: @@ -710,26 +717,11 @@ def _find_stop_hook(): requires_stop_hook = pytest.mark.skipif(STOP_HOOK is None, reason=f"{APPLICATION_STOP} not found") -def _stop_lines() -> dict: - roles, current = {}, None - for line in STOP_HOOK.read_text().splitlines(): - match = _STOP_BRANCH.match(line) - if match: - current = match["role"] - roles[current] = [] - continue - if line.startswith((" else", " fi", "else", "fi")): - current = None - continue - code = line.split("#", 1)[0].split() - if current and code[:1] and code[0].endswith("docker-compose") and "stop" in code: - roles[current].append(code) - return roles - - @requires_stop_hook -def test_the_stop_hook_stops_nothing_on_the_large_box_and_no_large_poller_anywhere(): - roles = _stop_lines() - assert roles.get("delphi-large", []) == [] - assert not any("math-python-large" in l for lines in roles.values() for l in lines) - assert "delphi-large" in STOP_HOOK.read_text() +def test_the_stop_hook_defers_shutdown_for_every_role(): + # ApplicationStop is now role-independent; the behavioral hook suite also + # executes it for every role and rejects any external command invocation. + text = STOP_HOOK.read_text() + code = "\n".join(line.split("#", 1)[0] for line in text.splitlines()) + assert not re.search(r"\b(?:docker(?:-compose)?|systemctl)\b", code) + assert "AfterInstall" in text diff --git a/delphi/tests/test_validate_service_hook.py b/delphi/tests/test_validate_service_hook.py new file mode 100644 index 000000000..2602f7d74 --- /dev/null +++ b/delphi/tests/test_validate_service_hook.py @@ -0,0 +1,63 @@ +"""Actual validation shell control flow, deterministic command-level failure probes.""" +import os +import subprocess +from pathlib import Path +import pytest + +def _validate_service_path(): + # The container flattens delphi/tests to /app/tests. Use the same explicit + # checkout root as the other hook tests; never silently skip a missing hook. + override = os.environ.get("POLIS_CHECKOUT_DIR") + candidates = [Path(override)] if override else Path(__file__).resolve().parents + for root in candidates: + path = root / "scripts/validate_service.sh" + if path.is_file(): + return path + raise FileNotFoundError( + "scripts/validate_service.sh not found; run scripts/test-deploy-hooks.sh " + "or set POLIS_CHECKOUT_DIR to the checkout root" + ) + + +VALIDATE_SERVICE_PATH = _validate_service_path() + +def probe(tmp_path, role='server', state='running false none', http=0, missing=False, drain=0): + bin_dir=tmp_path/'bin';bin_dir.mkdir() + (tmp_path/'service_type.txt').write_text(role) + source=VALIDATE_SERVICE_PATH.read_text() + source=source.replace('/opt/polis/polis',str(tmp_path)).replace('/etc/app-info/service_type.txt',str(tmp_path/'service_type.txt')).replace('/usr/local/bin/docker-compose','docker-compose') + script=tmp_path/'validate.sh';script.write_text(source) + fakes={ + 'sudo':'exec "$@"', + 'sleep':'exit 0', + 'docker-compose':'''case "$1" in + ps) [ "$MISSING" = 1 ] || echo owned ;; + exec) echo http >> "$CALLS"; exit "$HTTP_STATUS" ;; + *) exit 90 ;; esac''', + 'docker':'''case "$*" in *Health*) echo "$STATE";; *) echo 'running false';; esac''', + 'systemctl':'''n=0; [ ! -f "$COUNT" ] || n=$(cat "$COUNT"); n=$((n+1)); echo "$n" > "$COUNT"; [ "$n" -gt "$DRAIN" ]''', + } + for name,body in fakes.items(): + f=bin_dir/name;f.write_text('#!/bin/bash\n'+body+'\n');f.chmod(0o755) + env=dict(os.environ,PATH=str(bin_dir)+':/usr/bin:/bin',STATE=state,HTTP_STATUS=str(http),MISSING=str(int(missing)),DRAIN=str(drain),COUNT=str(tmp_path/'count'),CALLS=str(tmp_path/'calls')) + return subprocess.run(['bash',str(script)],env=env,text=True,capture_output=True,timeout=30) + +def test_healthy_server_requires_three_http_successes(tmp_path): + p=probe(tmp_path);assert p.returncode==0,p.stderr + assert (tmp_path/'calls').read_text().splitlines()==['http']*3 + +@pytest.mark.parametrize('kw',[{'http':1},{'missing':True},{'state':'exited false none'},{'state':'running true none'},{'state':'running false unhealthy'}]) +def test_unhealthy_server_fails_even_when_other_checks_pass(tmp_path,kw): + p=probe(tmp_path,**kw);assert p.returncode!=0 + assert 'ValidateService failed' in p.stderr + +@pytest.mark.parametrize('role',['delphi-large','delphi-worker']) +def test_worker_drain_can_outlast_server_validation_window(tmp_path,role): + p=probe(tmp_path,role=role,drain=80);assert p.returncode==0,p.stderr + assert int((tmp_path/'count').read_text())==83 + +def test_unknown_role_fails(tmp_path): + assert probe(tmp_path,role='unknown').returncode!=0 + +def test_retired_math_role_needs_no_containers(tmp_path): + assert probe(tmp_path,role='math',missing=True).returncode==0 diff --git a/docs/upgrading.md b/docs/upgrading.md index 976f65e0f..787e64538 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -1,5 +1,17 @@ # Upgrade Guide +## Safe deployment hooks (release A) + +This release changes deployment hooks only; it does not introduce the migration +runner or an API migration startup check. Install it successfully on every box +before enabling the runner in release B. CodeDeploy uses the PREVIOUS successful +revision's ApplicationStop, so this first transition can still interrupt service. +New hooks defer replacement until AfterInstall and validate the server's HTTP and +database routes. A failure in ValidateService fails the deployment; it does not +automatically restore containers or undo schema changes. Delphi/worker validation +checks process/container liveness, not real-time math correctness. + + ## Configuration Changes (Q1 2023) `polis.config.template.js` and `polis.config.js` files are removed and no longer used. diff --git a/scripts/after_install.sh b/scripts/after_install.sh index c259259ff..13db753c0 100644 --- a/scripts/after_install.sh +++ b/scripts/after_install.sh @@ -82,22 +82,13 @@ printf "\nDATABASE_URL=%s\n" "$DATABASE_URL" | sudo tee -a .env > /dev/null SERVICE_FROM_FILE=$(cat /etc/app-info/service_type.txt) echo "DEBUG: Service type read from /etc/app-info/service_type.txt: [$SERVICE_FROM_FILE]" -# Original Docker cleanup/start logic -echo "Stopping and removing existing Docker containers..." -sudo /usr/local/bin/docker-compose down || true -if [ "$SERVICE_FROM_FILE" == "delphi-large" ] || [ "$SERVICE_FROM_FILE" == "delphi-worker" ]; then - # A queue worker box keeps its polis-jobs container (started by the unit - # polis-jobs.service, not by compose): removing it would kill a job - # mid-attempt. worker_daemon below restarts the unit, which drains it. - sudo docker rm -f $(docker ps -aq | grep -vxF -e "$(docker ps -aq --filter 'name=^/?polis-jobs$')") || true -else - sudo docker rm -f $(docker ps -aq) || true -fi -echo "Docker containers stopped and removed." - -yes | sudo docker system prune -a --filter "until=72h" -echo "Docker cache cleared" +case "$SERVICE_FROM_FILE" in + server|delphi|math|delphi-large|delphi-worker) ;; + *) echo "Error: Unknown service type: [$SERVICE_FROM_FILE]"; exit 1 ;; +esac +# Validate configuration before replacing any running container. Compose replaces +# only this Compose project below; never remove unrelated containers or prune here. sudo /usr/local/bin/docker-compose config --quiet if [ -f "/etc/app-info/log_group_name.txt" ]; then @@ -152,18 +143,22 @@ worker_daemon() { fi echo "Building the Delphi image (it carries polis-jobs) and restarting polis-jobs.service for worker class '$want_class'" sudo /usr/local/bin/docker-compose build delphi + sudo /usr/local/bin/docker-compose down sudo systemctl restart --no-block polis-jobs.service } if [ "$SERVICE_FROM_FILE" == "server" ]; then echo "Starting docker-compose up for 'server', 'nginx-proxy', and 'client-participation-alpha' services" - sudo /usr/local/bin/docker-compose up -d server nginx-proxy client-participation-alpha --build --force-recreate + sudo /usr/local/bin/docker-compose build server nginx-proxy client-participation-alpha + sudo /usr/local/bin/docker-compose down + sudo /usr/local/bin/docker-compose up -d server nginx-proxy client-participation-alpha --no-build --force-recreate elif [ "$SERVICE_FROM_FILE" == "math" ]; then # The Clojure math engine is retired; the Python engine (math-python) runs on # the Delphi role. A box still tagged `math` (the math-worker ASG until it is # scaled to zero) starts nothing. This branch must stay until no such box is # left: without it the box would fall through to the catch-all below and # start every service, including a second Delphi job poller. + sudo /usr/local/bin/docker-compose down echo "Service type 'math' is retired (the Clojure engine was removed); starting no services on this box" elif [ "$SERVICE_FROM_FILE" == "delphi" ]; then echo "Starting docker-compose up for 'delphi' and 'math-python' (math engine) services" @@ -268,7 +263,7 @@ elif [ "$SERVICE_FROM_FILE" == "delphi" ]; then # ASG max of 1 for the Delphi small group is a later belt-and-braces CDK # change, not needed for correctness. # Durable stop: remove `math-python` from this line and redeploy (the hook - # removes every container before starting the named ones). + # removes this Compose project's containers before starting the named ones). # Removing only the holder's container is a FAILOVER, not a stop: a waiting # poller on another Delphi box takes the lock. Fleet-wide emergency stop: # 1. Pause anything that runs this hook: no deploy, and suspend Launch on @@ -286,7 +281,9 @@ elif [ "$SERVICE_FROM_FILE" == "delphi" ]; then # 5. Make it durable (remove `math-python` here and redeploy) before # resuming deploys or ASG launches. poller_identity - sudo /usr/local/bin/docker-compose up -d delphi math-python --build --force-recreate + sudo /usr/local/bin/docker-compose build delphi math-python + sudo /usr/local/bin/docker-compose down + sudo /usr/local/bin/docker-compose up -d delphi math-python --no-build --force-recreate elif [ "$SERVICE_FROM_FILE" == "delphi-large" ]; then # The large memory class box (P-073 r2). Its worker is the polis-jobs # daemon run as worker class `large`, which starts `math_poller.py --job` @@ -305,6 +302,6 @@ elif [ "$SERVICE_FROM_FILE" == "delphi-worker" ]; then echo "Service type 'delphi-worker': no compose service to start (the Delphi stages run as queue jobs under the polis-jobs daemon)" worker_daemon delphi else - echo "Error: Unknown service type: [$SERVICE_FROM_FILE]. Starting all services (default docker-compose up -d)" - sudo /usr/local/bin/docker-compose up -d --build --force-recreate -fi \ No newline at end of file + echo "Error: Unknown service type: [$SERVICE_FROM_FILE]" + exit 1 +fi diff --git a/scripts/application_stop.sh b/scripts/application_stop.sh index a16c56e71..a704ca08c 100644 --- a/scripts/application_stop.sh +++ b/scripts/application_stop.sh @@ -1,83 +1,6 @@ #!/bin/bash -# This script runs during the ApplicationStop lifecycle event in CodeDeploy. -# It stops the relevant Docker containers based on the instance's role. - -set -e # Exit immediately if a command exits with a non-zero status. -set -x # Print commands and their arguments as they are executed. - -echo "Executing ApplicationStop hook..." - -# --- Configuration --- -# Directory where the docker-compose.yml file for the *current* deployment resides -# Adjust this path if your deployment process places files elsewhere -DEPLOY_DIR="/opt/polis/polis" -# File indicating the role of this instance (created by UserData/AfterInstall) -SERVICE_TYPE_FILE="/etc/app-info/service_type.txt" - -# --- Determine Service Type --- -if [ -f "$SERVICE_TYPE_FILE" ]; then - SERVICE_TYPE=$(cat "$SERVICE_TYPE_FILE") - echo "Detected service type: $SERVICE_TYPE" -else - echo "Warning: Service type file not found at $SERVICE_TYPE_FILE. Assuming nothing specific needs to be stopped by this script." - # Exit cleanly as we don't know what to stop, or maybe the instance role changed. - # CodeDeploy will likely proceed, and the AfterInstall script handles cleanup anyway. - exit 0 -fi - -# --- Stop Services based on Type --- - -# Check if the deployment directory exists (where docker-compose.yml should be) -if [ -d "$DEPLOY_DIR" ]; then - cd "$DEPLOY_DIR" - echo "Changed directory to $DEPLOY_DIR" - - # Check if docker-compose command exists - if ! command -v /usr/local/bin/docker-compose &> /dev/null; then - echo "Error: docker-compose command not found at /usr/local/bin/docker-compose. Cannot stop services." - # Exit with error because compose is expected if the directory exists and type isn't ollama - if [ "$SERVICE_TYPE" != "ollama" ]; then - exit 1 - fi - fi - - if [ "$SERVICE_TYPE" == "server" ]; then - echo "Stopping server-related services (server, nginx-proxy, file-server, client-participation-alpha)..." - # Stop services related to the 'server' type instance (as started in AfterInstall) - /usr/local/bin/docker-compose stop server nginx-proxy file-server client-participation-alpha || echo "Warning: Failed to stop server component(s), might already be stopped." - # Optional: Use 'down' if you want to remove networks etc. during stop, but 'stop' is usually sufficient here. - # /usr/local/bin/docker-compose down --remove-orphans server nginx-proxy file-server || echo "Warning..." - - elif [ "$SERVICE_TYPE" == "math" ]; then - # The Clojure math service is retired; AfterInstall starts nothing on a - # `math` box. Stop a container left by an earlier revision, if any. - echo "Service type 'math' is retired; stopping any leftover math container..." - docker stop polis-math-1 2>/dev/null || echo "No math container running." - - elif [ "$SERVICE_TYPE" == "delphi" ]; then - echo "Stopping delphi service..." - /usr/local/bin/docker-compose stop delphi || echo "Warning: Failed to stop delphi service, might already be stopped." - - elif [ "$SERVICE_TYPE" == "delphi-large" ]; then - # A queue worker box runs no compose service: its worker is the polis-jobs - # daemon (polis-jobs.service). Nothing is stopped here: stopping drains the - # running job for up to 900 s, longer than this hook may take, so - # AfterInstall restarts the unit instead (queued; the drain runs in systemd). - echo "Service type 'delphi-large': no compose service to stop (polis-jobs.service is restarted by AfterInstall)" - - elif [ "$SERVICE_TYPE" == "delphi-worker" ]; then - # The same for a Delphi queue worker box. - echo "Service type 'delphi-worker': no compose service to stop (polis-jobs.service is restarted by AfterInstall)" - - else - echo "Warning: Unknown service type '$SERVICE_TYPE' found in $SERVICE_TYPE_FILE. No specific services stopped." - # Avoid running a generic 'down' as it might affect unrelated containers if any exist - fi - -else - echo "Warning: Deployment directory $DEPLOY_DIR not found. Assuming no services need stopping." - # Exit cleanly if the directory isn't there, as nothing from this app could be running - exit 0 -fi - -echo "ApplicationStop hook finished successfully for service type: $SERVICE_TYPE." \ No newline at end of file +set -eu +# Keep the healthy revision running until AfterInstall has migrated successfully. +# CodeDeploy executes ApplicationStop from the PREVIOUS successful revision; see +# docs/upgrading.md for the one-time transition from the old stopping hook. +echo "Service shutdown is deferred until migrations succeed in AfterInstall." diff --git a/scripts/before_install.sh b/scripts/before_install.sh index 35e506acc..a704ca08c 100644 --- a/scripts/before_install.sh +++ b/scripts/before_install.sh @@ -1,19 +1,6 @@ #!/bin/bash -set -e -set -x - -# Stop any existing Docker containers (if needed) -# Docker's `name` filter is an unanchored regex (a substring match), so -# `name=polis-math` also matches the Delphi box's `polis-math-python-1` and the -# hook then tried to stop a `polis-math-1` that only exists on the math box. -# Anchor every filter to the exact container name; the optional leading `/` -# covers daemons that match against the stored `/name` form. -if docker ps -q --filter "name=^/?polis-server-1$" | grep -q .; then - docker stop polis-server-1 -fi -if docker ps -q --filter "name=^/?polis-math-1$" | grep -q .; then - docker stop polis-math-1 -fi -if docker ps -q --filter "name=^/?polis-delphi-1$" | grep -q .; then - docker stop polis-delphi-1 -fi +set -eu +# Keep the healthy revision running until AfterInstall has migrated successfully. +# CodeDeploy executes ApplicationStop from the PREVIOUS successful revision; see +# docs/upgrading.md for the one-time transition from the old stopping hook. +echo "Service shutdown is deferred until migrations succeed in AfterInstall." diff --git a/scripts/test-deploy-hooks.sh b/scripts/test-deploy-hooks.sh new file mode 100644 index 000000000..00dcb8f13 --- /dev/null +++ b/scripts/test-deploy-hooks.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Shared mm5/CI entry point. Requires an already-running Delphi test container. +# Usage: bash scripts/test-deploy-hooks.sh [-f compose.yml --env-file .env ...] +# Compose global options are passed through; this never starts/stops services. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +if [ "$#" -eq 0 ]; then + set -- -f docker-compose.test.yml --env-file .env +fi +compose=(docker compose "$@") +inputs=( + scripts/after_install.sh + scripts/before_install.sh + scripts/application_stop.sh + scripts/validate_service.sh + docker-compose.yml + docker-compose.test.yml + delphi/Dockerfile +) +tests=( + test_after_install_hook.py + test_before_install_hook.py + test_validate_service_hook.py + test_compose_math_env.py +) +# Fail before copying anything if the source checkout is incomplete. Missing +# inputs must not turn these deployment controls into a green skipped suite. +for rel in "${inputs[@]}" "${tests[@]/#/delphi/tests/}"; do + if [ ! -f "$rel" ]; then + echo "Missing deploy-hook test input: $rel" >&2 + exit 1 + fi +done +for rel in "${inputs[@]}"; do + "${compose[@]}" exec -T delphi mkdir -p "/app/projgate/$(dirname "$rel")" + "${compose[@]}" cp "$rel" "delphi:/app/projgate/$rel" +done +"${compose[@]}" exec -T delphi mkdir -p /app/tests +for test in "${tests[@]}"; do + "${compose[@]}" cp "delphi/tests/$test" "delphi:/app/tests/$test" +done +# This suite needs neither database fixtures nor the unrelated Delphi session +# conftest. Keep its configuration identical here and in hosted CI. The full +# Delphi suite still runs afterward in CI with its normal conftest and coverage. +"${compose[@]}" exec -T -e POLIS_CHECKOUT_DIR=/app/projgate delphi \ + python -m pytest --noconftest -o addopts= -q "${tests[@]/#//app/tests/}" diff --git a/scripts/validate_service.sh b/scripts/validate_service.sh new file mode 100644 index 000000000..ba452cd1b --- /dev/null +++ b/scripts/validate_service.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -eu +cd /opt/polis/polis +role=$(cat /etc/app-info/service_type.txt) +compose() { sudo /usr/local/bin/docker-compose "$@"; } +containers_ready() { + local service ids id state + for service in "$@"; do + ids=$(compose ps -q "$service") || return 1 + [ -n "$ids" ] || return 1 + for id in $ids; do + state=$(sudo docker inspect --format '{{.State.Status}} {{.State.Restarting}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id") || return 1 + case "$state" in 'running false healthy'|'running false none') ;; *) return 1 ;; esac + done + done +} +server_ready() { + containers_ready server nginx-proxy client-participation-alpha || return 1 + # A running container alone is insufficient: exercise the actual HTTP server + # and its database route. Abort hung requests as well as rejecting non-200s. + compose exec -T server node -e ' + Promise.all(["/api/v3/testConnection", "/api/v3/testDatabase"].map(async path => { + const port = process.env.API_SERVER_PORT || process.env.PORT || "5000"; + const r = await fetch("http://127.0.0.1:" + port + path, {signal: AbortSignal.timeout(4000)}); + if (r.status !== 200 || (await r.json()).status !== "ok") throw Error("health check failed"); + })).then(() => process.exit(0)).catch(() => process.exit(1)); + ' +} +ready() { + case "$role" in + server) server_ready ;; + delphi) containers_ready delphi math-python ;; + delphi-large|delphi-worker) + sudo systemctl is-active --quiet polis-jobs.service && + [ "$(sudo docker inspect --format '{{.State.Status}} {{.State.Restarting}}' polis-jobs)" = 'running false' ] ;; + math) return 0 ;; # Explicitly retired: no service is expected. + *) echo "Unknown service type: $role" >&2; return 1 ;; + esac +} +# Require consecutive successes so a restarting process cannot pass on one poll. +successes=0 +attempts=60 +# Worker restarts can legitimately drain an in-flight job for 900 seconds. +case "$role" in delphi-large|delphi-worker) attempts=500 ;; esac +for attempt in $(seq 1 "$attempts"); do + if ready; then + successes=$((successes + 1)) + if [ "$successes" -ge 3 ]; then + echo "ValidateService passed for $role" + exit 0 + fi + else + successes=0 + fi + sleep 2 +done +echo "ValidateService failed for $role" >&2 +exit 1