From b126a85cf26c8eaf6ac25e1b27a495bc55a902d4 Mon Sep 17 00:00:00 2001 From: miljanm <1547789+miljanm@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:16:57 +0000 Subject: [PATCH 1/2] Make app schedules timezone-durable via IANA identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Möbius Agent --- backend/app/cron_tz.py | 128 +++++++++++++++++++++++++++++++ backend/app/install.py | 42 ++++++++++- backend/app/main.py | 46 ++++++++++++ backend/app/routes/apps.py | 78 ++++++++++++++++++- backend/app/schemas.py | 21 +++++- backend/tests/test_apps.py | 121 +++++++++++++++++++++++++++++ backend/tests/test_cron_tz.py | 138 ++++++++++++++++++++++++++++++++++ 7 files changed, 567 insertions(+), 7 deletions(-) create mode 100644 backend/app/cron_tz.py create mode 100644 backend/tests/test_cron_tz.py diff --git a/backend/app/cron_tz.py b/backend/app/cron_tz.py new file mode 100644 index 000000000..a6039fa83 --- /dev/null +++ b/backend/app/cron_tz.py @@ -0,0 +1,128 @@ +"""Zone-aware cron schedules — durable IANA identity, materialized entries. + +Cron itself evaluates wall-clock time in the server's local timezone, which +is not a durable way to express "5:00 AM in Europe/Belgrade": the server's +offset to that zone changes at daylight-saving transitions, in either zone. + +The durable schedule identity is therefore an IANA timezone name plus a +zone-local daily cron expression, declared in the app's ``init-cron.sh`` +(``SCHEDULE_TZ`` / ``SCHEDULE_SOURCE``). The crontab line cron actually runs +is a *materialization* of that identity into the server's current clock, +recomputed at boot and periodically at runtime so an offset change in either +zone reschedules the entry instead of silently drifting the wall time. + +Only simple daily expressions (numeric minute + hour, ``* * *`` date fields) +may carry a timezone: shifting an expression across an offset can cross a day +boundary, which is well-defined for a daily job and ambiguous for date-pinned +ones. +""" + +from __future__ import annotations + +import os +import re +from datetime import datetime +from pathlib import Path +from zoneinfo import ZoneInfo + +_DAILY_CRON_RE = re.compile( + r"^\s*(\d{1,2})\s+(\d{1,2})\s+\*\s+\*\s+\*\s*$" +) +# Written by init-cron-scaffold.sh; parsed (never executed) from init-cron.sh. +_DECL_TZ_RE = re.compile(r'^SCHEDULE_TZ="([A-Za-z0-9_+/-]+)"\s*$', re.M) +_DECL_SOURCE_RE = re.compile(r'^SCHEDULE_SOURCE="([^"\n]+)"\s*$', re.M) + + +def valid_timezone(name: str) -> bool: + """True when ``name`` is a resolvable IANA timezone identifier.""" + if not isinstance(name, str) or not name or len(name) > 64: + return False + if not re.fullmatch(r"[A-Za-z0-9_+/-]+", name): + return False + try: + ZoneInfo(name) + except Exception: + return False + return True + + +def parse_daily_cron(expr: str) -> tuple[int, int] | None: + """Returns (minute, hour) for a plain daily cron, else None.""" + m = _DAILY_CRON_RE.match(expr or "") + if not m: + return None + minute, hour = int(m.group(1)), int(m.group(2)) + if minute > 59 or hour > 23: + return None + return minute, hour + + +def materialize_zone_cron( + zone_cron: str, + tz_name: str, + *, + now: datetime | None = None, + server_tz=None, +) -> str: + """Expresses a zone-local daily cron in the server's current clock. + + ``now`` and ``server_tz`` exist for tests; production uses the real clock + and the process-local timezone. The conversion is exact for the current + offset pair — the caller re-materializes when offsets change, which is the + whole durability contract. + + Raises ValueError for a non-daily expression or unknown timezone. + """ + parsed = parse_daily_cron(zone_cron) + if parsed is None: + raise ValueError( + "A timezone-owned schedule must be a plain daily cron " + f"('m h * * *'), got: {zone_cron!r}" + ) + minute, hour = parsed + if not valid_timezone(tz_name): + raise ValueError(f"Unknown IANA timezone: {tz_name!r}") + tz = ZoneInfo(tz_name) + moment = (now or datetime.now(tz)).astimezone(tz) + local = moment.replace(hour=hour, minute=minute, second=0, microsecond=0) + server = local.astimezone(server_tz) + return f"{server.minute} {server.hour} * * *" + + +def parse_zone_declaration(init_cron_text: str) -> tuple[str, str] | None: + """Extracts (timezone, zone_cron) from init-cron.sh text, if declared. + + Returns None when either variable is absent or invalid — an app whose + declaration was hand-edited into an inconsistent state falls back to plain + server-local scheduling rather than guessing. + """ + tz_match = _DECL_TZ_RE.search(init_cron_text or "") + source_match = _DECL_SOURCE_RE.search(init_cron_text or "") + if not tz_match or not source_match: + return None + tz_name = tz_match.group(1) + zone_cron = source_match.group(1).strip() + if not valid_timezone(tz_name) or parse_daily_cron(zone_cron) is None: + return None + return tz_name, zone_cron + + +def server_timezone_name() -> str: + """The server clock's IANA identity: TZ env, /etc/localtime, else UTC. + + Containers conventionally run UTC with neither signal present; "UTC" is a + valid IANA identifier, so the fallback stays truthful for them. + """ + tz_env = os.environ.get("TZ", "").strip() + if tz_env and valid_timezone(tz_env): + return tz_env + try: + target = Path("/etc/localtime").resolve() + parts = target.parts + if "zoneinfo" in parts: + name = "/".join(parts[parts.index("zoneinfo") + 1:]) + if valid_timezone(name): + return name + except OSError: + pass + return "UTC" diff --git a/backend/app/install.py b/backend/app/install.py index d5ea7d04c..a820e6c3e 100644 --- a/backend/app/install.py +++ b/backend/app/install.py @@ -1127,9 +1127,18 @@ def _process_icon(raw: bytes) -> bytes: def _register_cron(slug: str, schedule_expr: str, job_path: Path, - app_id: int | None = None) -> None: + app_id: int | None = None, + timezone: str | None = None, + zone_cron: str | None = None) -> None: """Runs init-cron-scaffold.sh to install the crontab entry. + ``timezone``/``zone_cron`` (given together) declare the schedule's durable + zone-aware identity: ``schedule_expr`` is then its server-local + materialization, and the identity is appended to ``init-cron.sh`` after the + scaffold rewrites it, so reconciliation can re-materialize the entry when + either zone's UTC offset changes. The scaffold's own contract stays + unchanged; the platform owns the declaration lines (see app.cron_tz). + The scaffold writes both the durable ``init-cron.sh`` declaration and the live crontab entry. On restart, lifespan parses that declaration and rewrites it through the supervised runner; app-owned shell is never executed merely @@ -1180,6 +1189,37 @@ def _register_cron(slug: str, schedule_expr: str, job_path: Path, 500, f"Cron registration failed: {result.stderr.strip()[:400]}", ) + if (timezone is None) != (zone_cron is None): + raise HTTPException( + 500, "Cron registration bug: timezone and zone_cron must be paired.", + ) + if timezone is not None: + from app import cron_tz + + if not cron_tz.valid_timezone(timezone): + raise HTTPException(500, f"Unknown IANA timezone: {timezone!r}") + if cron_tz.parse_daily_cron(zone_cron) is None: + raise HTTPException( + 500, f"Zone-owned schedule must be a plain daily cron: {zone_cron!r}", + ) + init_path = job_path.parent / "init-cron.sh" + try: + # The scaffold just rewrote init-cron.sh from its template, so a plain + # append is deterministic and repeat-registration cannot duplicate. + with init_path.open("a", encoding="utf-8") as fh: + fh.write( + "\n# --- Zone-aware schedule identity " + "(platform-managed; parsed, never executed).\n" + "# ENTRY above is the server-local materialization of this durable\n" + "# identity, recomputed at boot and periodically so DST transitions\n" + "# reschedule the entry instead of drifting its wall time.\n" + f'SCHEDULE_TZ="{timezone}"\n' + f'SCHEDULE_SOURCE="{zone_cron}"\n' + ) + except OSError as exc: + raise HTTPException( + 500, f"Could not persist schedule timezone: {exc}", + ) from exc def _reconcile_cron_after_install_rollback() -> None: diff --git a/backend/app/main.py b/backend/app/main.py index 30d417a58..91397dfb3 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -442,6 +442,50 @@ async def lifespan(app): _cron_ready.write_text(f"{_BOOT_ID}\n", encoding="utf-8") except Exception as exc: _log.error("app cron supervision wiring failed: %s", exc, exc_info=True) + # Periodic re-materialization for timezone-owned schedules (app.cron_tz): + # a schedule declared as "wall time in an IANA zone" must be rescheduled + # when either that zone's or the server's UTC offset changes (DST). Boot + # reconciliation covers restarts; this hourly pass covers long-running + # containers crossing a transition. It reuses the same idempotent + # reconciler, and only runs while any zone declaration actually exists. + _cron_tz_task = None + try: + from app.routes.apps import ( + _app_zone_declaration as _zone_decl, + reconcile_app_cron_supervision as _cron_reconcile, + ) + from app.database import SessionLocal as _CronTzSession + + async def _cron_tz_loop(): + while True: + await _asyncio.sleep(3600) + try: + def _pass(): + from app import models as _tz_models + _db = _CronTzSession() + try: + _live = _db.query(_tz_models.App).filter( + _tz_models.App.deleted_at.is_(None) + ).all() + if not any(_zone_decl(_a) for _a in _live): + return None + return _cron_reconcile(_db) + finally: + _db.close() + _result = await _asyncio.to_thread(_pass) + if _result is not None: + for _warning in _result[1]: + _log.warning("cron tz re-materialization: %s", _warning) + except _asyncio.CancelledError: + raise + except Exception as _exc: + _log.error( + "cron tz re-materialization failed: %s", _exc, exc_info=True, + ) + + _cron_tz_task = _asyncio.create_task(_cron_tz_loop()) + except Exception as exc: + _log.error("cron tz loop wiring failed: %s", exc, exc_info=True) record_memory_checkpoint("startup_metadata_reconciled") # Route provider model-registry and memory diagnostics to the durable # rotating chat.log handler. These loggers otherwise land only on @@ -665,6 +709,8 @@ async def _browser_profile_loop(): _reset_park_task.cancel() if _browser_profile_task is not None: _browser_profile_task.cancel() + if _cron_tz_task is not None: + _cron_tz_task.cancel() if _writer_supervisor_task is not None: _writer_supervisor_task.cancel() # Drain + join the chat-writer actor so any in-flight persistence diff --git a/backend/app/routes/apps.py b/backend/app/routes/apps.py index c5be4d8e0..0ff085708 100644 --- a/backend/app/routes/apps.py +++ b/backend/app/routes/apps.py @@ -590,6 +590,26 @@ def _app_schedule(app: models.App, live_crontab: str) -> tuple[str, str] | None: return _manifest_schedule(source_dir) +def _app_zone_declaration(app: models.App) -> tuple[str, str] | None: + """The schedule's durable (timezone, zone_cron) identity, if declared. + + Read from the init-cron.sh declaration lines the platform appends when a + schedule is owned in an IANA zone (see app.cron_tz / _register_cron). + """ + from app import cron_tz + + if not app.source_dir: + return None + source_dir = Path(app.source_dir) + for replay_dir in _cron_replay_dirs_for_app(app, source_dir): + declaration = cron_tz.parse_zone_declaration( + _read_init_cron_text(replay_dir), + ) + if declaration is not None: + return declaration + return None + + def reconcile_app_cron_supervision(db: Session) -> tuple[int, list[str]]: """Converge every live managed schedule through the common job runner. @@ -600,7 +620,13 @@ def reconcile_app_cron_supervision(db: Session) -> tuple[int, list[str]]: crontab entry and its durable declaration via the current scaffold. Tombstoned apps are excluded and source trees must be ordinary, non-symlink direct children of ``/data/apps``. + + A schedule owned in an IANA timezone (see ``app.cron_tz``) is + re-materialized here from its durable (timezone, zone_cron) identity + rather than preserved verbatim — this pass, run at boot and periodically + at runtime, is what reschedules the entry across DST transitions. """ + from app import cron_tz from app.install import _register_cron settings = get_settings() @@ -632,9 +658,19 @@ def reconcile_app_cron_supervision(db: Session) -> tuple[int, list[str]]: try: if job_path.is_symlink() or not job_path.is_file(): raise ValueError(f"job is missing or a symlink: {job_name}") - _register_cron( - resolved_source.name, cron, job_path, app.id, - ) + declaration = _app_zone_declaration(app) + if declaration is not None: + timezone, zone_cron = declaration + _register_cron( + resolved_source.name, + cron_tz.materialize_zone_cron(zone_cron, timezone), + job_path, app.id, + timezone=timezone, zone_cron=zone_cron, + ) + else: + _register_cron( + resolved_source.name, cron, job_path, app.id, + ) except Exception as exc: warnings.append(f"app {app.id}: {exc}") continue @@ -723,7 +759,10 @@ def list_app_schedules( _: models.Owner = Depends(get_current_owner_or_app), ): """Returns read-only recurring app schedules visible to owners and apps.""" + from app import cron_tz + live_crontab = _read_live_crontab() + server_timezone = cron_tz.server_timezone_name() rows = [] apps = ( db.query(models.App) @@ -736,12 +775,16 @@ def list_app_schedules( if schedule is None: continue cron, job = schedule + declaration = _app_zone_declaration(app) rows.append(schemas.AppScheduleOut( id=app.id, name=app.name, slug=app.slug, cron=cron, job=job, + timezone=declaration[0] if declaration else None, + zone_cron=declaration[1] if declaration else None, + server_timezone=server_timezone, )) return rows @@ -2863,11 +2906,27 @@ def update_app_schedule( raise HTTPException( status_code=400, detail="App has no source_dir; cannot locate job.", ) + from app import cron_tz from app.install import _register_cron try: validate_cron_expr(body.cron) except ManifestContractError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc + timezone = (body.timezone or "").strip() or None + if timezone is not None: + # A zone-owned schedule is durable data: body.cron is the daily wall + # time in that zone, and the crontab entry is its server-local + # materialization, re-materialized when either zone's offset changes. + if not cron_tz.valid_timezone(timezone): + raise HTTPException( + status_code=400, detail=f"Unknown IANA timezone: {timezone!r}", + ) + if cron_tz.parse_daily_cron(body.cron) is None: + raise HTTPException( + status_code=400, + detail="A timezone-owned schedule must be a plain daily cron " + "('m h * * *').", + ) source_dir = Path(app.source_dir) job_name = body.job or "fetch.sh" if "/" in job_name or "\\" in job_name or not job_name.strip(): @@ -2876,8 +2935,19 @@ def update_app_schedule( if not job_path.is_file(): raise HTTPException(status_code=400, detail="Job script not found.") slug = app.slug or _slugify_for_source_dir(app.name) + if timezone is not None: + materialized = cron_tz.materialize_zone_cron(body.cron, timezone) + _register_cron( + slug, materialized, job_path, app_id, + timezone=timezone, zone_cron=body.cron, + ) + return { + "cron": materialized, "job": job_name, + "timezone": timezone, "zone_cron": body.cron, + } _register_cron(slug, body.cron, job_path, app_id) - return {"cron": body.cron, "job": job_name} + return {"cron": body.cron, "job": job_name, "timezone": None, + "zone_cron": None} def _etag_for_app(app: models.App) -> str | None: diff --git a/backend/app/schemas.py b/backend/app/schemas.py index e26e170be..a4039249f 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -258,14 +258,28 @@ class AppInstallOut(AppOut): class AppScheduleUpdate(BaseModel): - """Body for updating one installed app's cron schedule.""" + """Body for updating one installed app's cron schedule. + + When ``timezone`` (an IANA identifier) is set, ``cron`` is a plain daily + expression owned in that zone; the platform stores that identity durably + and materializes/re-materializes the server-local crontab entry itself. + Without it, ``cron`` is interpreted in the server's local timezone as + before. + """ cron: str job: str | None = None + timezone: str | None = None class AppScheduleOut(BaseModel): - """Read-only metadata for an installed app's recurring cron job.""" + """Read-only metadata for an installed app's recurring cron job. + + ``cron`` is always the materialized server-local entry. When the schedule + is owned in an IANA zone, ``timezone``/``zone_cron`` carry that durable + identity (the platform re-materializes ``cron`` when offsets change). + ``server_timezone`` is the server clock's own IANA identity. + """ id: int name: str @@ -273,6 +287,9 @@ class AppScheduleOut(BaseModel): cron: str job: str next_run: datetime | None = None + timezone: str | None = None + zone_cron: str | None = None + server_timezone: str = "UTC" class ConflictFile(BaseModel): diff --git a/backend/tests/test_apps.py b/backend/tests/test_apps.py index 8849e1633..ec1daaeb9 100644 --- a/backend/tests/test_apps.py +++ b/backend/tests/test_apps.py @@ -300,6 +300,123 @@ def fake_register(slug, schedule_expr, job_path, app_id=None): assert r.status_code == 403 +def test_schedule_update_with_timezone_materializes_and_declares( + client, auth, monkeypatch, +): + """A timezone-owned schedule registers its MATERIALIZED server-local cron + plus the durable (timezone, zone_cron) identity; invalid zones and + non-daily expressions are rejected before any registration.""" + calls = [] + + def fake_register(slug, schedule_expr, job_path, app_id=None, + timezone=None, zone_cron=None): + calls.append((slug, schedule_expr, job_path.name, app_id, + timezone, zone_cron)) + + monkeypatch.setattr("app.install._register_cron", fake_register) + monkeypatch.setattr( + "app.cron_tz.materialize_zone_cron", + lambda zone_cron, tz_name, **kw: "0 3 * * *", + ) + source_dir = Path(get_settings().data_dir) / "apps" / "memory" + source_dir.mkdir(parents=True, exist_ok=True) + (source_dir / "fetch.sh").write_text("#!/bin/sh\n", encoding="utf-8") + app_id = create_local_app( + client, auth, name="Memory", description="test", source_dir=source_dir, + )["id"] + + r = client.post( + f"/api/apps/{app_id}/schedule", + json={"cron": "0 5 * * *", "job": "fetch.sh", + "timezone": "Europe/Belgrade"}, + headers=auth, + ) + assert r.status_code == 200, r.text + assert r.json() == { + "cron": "0 3 * * *", "job": "fetch.sh", + "timezone": "Europe/Belgrade", "zone_cron": "0 5 * * *", + } + assert calls == [ + ("memory", "0 3 * * *", "fetch.sh", app_id, + "Europe/Belgrade", "0 5 * * *"), + ] + + r = client.post( + f"/api/apps/{app_id}/schedule", + json={"cron": "0 5 * * *", "timezone": "Not/AZone"}, + headers=auth, + ) + assert r.status_code == 400 + r = client.post( + f"/api/apps/{app_id}/schedule", + json={"cron": "0 5 * * 1", "timezone": "Europe/Belgrade"}, + headers=auth, + ) + assert r.status_code == 400 + assert len(calls) == 1 + + +def test_reconcile_rematerializes_zone_owned_schedule(client, auth, db): + """Reconciliation recomputes a zone-owned entry from its durable identity — + the mechanism that reschedules it across a DST transition.""" + source_dir = Path(get_settings().data_dir) / "apps" / "memory" + source_dir.mkdir(parents=True) + (source_dir / "fetch.sh").write_text("#!/bin/sh\n", encoding="utf-8") + (source_dir / "init-cron.sh").write_text( + f'ENTRY="0 4 * * * {source_dir}/fetch.sh 56"\n' + 'SCHEDULE_TZ="Europe/Belgrade"\n' + 'SCHEDULE_SOURCE="0 5 * * *"\n', + encoding="utf-8", + ) + app_id = create_local_app( + client, _service_auth(), name="Memory", description="test", + source_dir=source_dir, + )["id"] + + from app.routes import apps as apps_module + calls = [] + + def fake_register(slug, schedule_expr, job_path, app_id=None, + timezone=None, zone_cron=None): + calls.append((slug, schedule_expr, timezone, zone_cron)) + + # The offset changed since the entry was written (0 4 → materializes 0 3 + # now): reconciliation must register the RECOMPUTED entry, not preserve + # the stale one. + with patch("app.install._register_cron", fake_register), \ + patch("app.cron_tz.materialize_zone_cron", + lambda zone_cron, tz_name, **kw: "0 3 * * *"): + count, warnings = apps_module.reconcile_app_cron_supervision(db) + + assert warnings == [] + assert count == 1 + assert calls == [("memory", "0 3 * * *", "Europe/Belgrade", "0 5 * * *")] + + +def test_app_schedules_expose_zone_declaration(client, auth): + """A schedule owned in an IANA zone surfaces its durable identity.""" + source_dir = Path(get_settings().data_dir) / "apps" / "memory" + source_dir.mkdir(parents=True) + (source_dir / "fetch.sh").write_text("#!/bin/sh\n", encoding="utf-8") + (source_dir / "init-cron.sh").write_text( + f'ENTRY="0 3 * * * {source_dir}/fetch.sh 56"\n' + "# --- Zone-aware schedule identity (platform-managed).\n" + 'SCHEDULE_TZ="Europe/Belgrade"\n' + 'SCHEDULE_SOURCE="0 5 * * *"\n', + encoding="utf-8", + ) + create_local_app( + client, _service_auth(), name="Memory", description="test", + source_dir=source_dir, + ) + r = client.get("/api/apps/schedules", headers=auth) + assert r.status_code == 200, r.text + rows = r.json() + assert [(j["cron"], j["timezone"], j["zone_cron"]) for j in rows] == [ + ("0 3 * * *", "Europe/Belgrade", "0 5 * * *"), + ] + + def test_platform_source_patch_rejected_and_store_identity_preserved(client, auth, db): from app import models data_dir = Path(get_settings().data_dir) @@ -363,6 +480,7 @@ def test_app_schedules_are_readable_by_app_tokens(client, auth): headers={"Authorization": f"Bearer {token}"}, ) assert r.status_code == 200, r.text + from app import cron_tz assert r.json() == [{ "id": 1, "name": "News", @@ -370,6 +488,9 @@ def test_app_schedules_are_readable_by_app_tokens(client, auth): "cron": "0 10 * * *", "job": "fetch.sh", "next_run": None, + "timezone": None, + "zone_cron": None, + "server_timezone": cron_tz.server_timezone_name(), }] diff --git a/backend/tests/test_cron_tz.py b/backend/tests/test_cron_tz.py new file mode 100644 index 000000000..800afe52d --- /dev/null +++ b/backend/tests/test_cron_tz.py @@ -0,0 +1,138 @@ +"""Zone-aware schedule materialization — including DST boundaries. + +The durable schedule identity is (IANA timezone, zone-local daily cron); +the crontab entry is a server-local materialization recomputed when either +zone's UTC offset changes. These tests pin the conversion on both sides of +the 2026 European transitions (spring forward 2026-03-29, fall back +2026-10-25) and in both directions of ownership. +""" + +from datetime import datetime +from zoneinfo import ZoneInfo + +import pytest + +from app import cron_tz + +UTC = ZoneInfo("UTC") +BELGRADE = ZoneInfo("Europe/Belgrade") + + +def test_parse_daily_cron_accepts_plain_daily(): + assert cron_tz.parse_daily_cron("0 5 * * *") == (0, 5) + assert cron_tz.parse_daily_cron("30 23 * * *") == (30, 23) + + +@pytest.mark.parametrize("expr", [ + "0 5 * * 1", # weekday-pinned + "0 5 1 * *", # date-pinned + "*/5 5 * * *", # stepped minute + "0 24 * * *", # invalid hour + "60 5 * * *", # invalid minute + "0 5 * *", # too few fields + "", +]) +def test_parse_daily_cron_rejects_non_daily(expr): + assert cron_tz.parse_daily_cron(expr) is None + + +def test_materialize_summer_offset_on_utc_server(): + # 2026-07-01: Belgrade is UTC+2 → 5:00 local = 3:00 server. + now = datetime(2026, 7, 1, 12, 0, tzinfo=UTC) + assert cron_tz.materialize_zone_cron( + "0 5 * * *", "Europe/Belgrade", now=now, server_tz=UTC, + ) == "0 3 * * *" + + +def test_materialize_winter_offset_on_utc_server(): + # 2026-12-01: Belgrade is UTC+1 → 5:00 local = 4:00 server. + now = datetime(2026, 12, 1, 12, 0, tzinfo=UTC) + assert cron_tz.materialize_zone_cron( + "0 5 * * *", "Europe/Belgrade", now=now, server_tz=UTC, + ) == "0 4 * * *" + + +def test_materialize_across_spring_forward_boundary(): + # Europe's 2026 spring transition: 2026-03-29 02:00 CET → 03:00 CEST. + before = datetime(2026, 3, 28, 12, 0, tzinfo=UTC) + after = datetime(2026, 3, 29, 12, 0, tzinfo=UTC) + materialize = lambda now: cron_tz.materialize_zone_cron( + "0 5 * * *", "Europe/Belgrade", now=now, server_tz=UTC, + ) + assert materialize(before) == "0 4 * * *" # +1 before the switch + assert materialize(after) == "0 3 * * *" # +2 after — entry rescheduled + + +def test_materialize_across_fall_back_boundary(): + # Europe's 2026 fall transition: 2026-10-25 03:00 CEST → 02:00 CET. + before = datetime(2026, 10, 24, 12, 0, tzinfo=UTC) + after = datetime(2026, 10, 25, 12, 0, tzinfo=UTC) + materialize = lambda now: cron_tz.materialize_zone_cron( + "0 5 * * *", "Europe/Belgrade", now=now, server_tz=UTC, + ) + assert materialize(before) == "0 3 * * *" + assert materialize(after) == "0 4 * * *" + + +def test_materialize_when_server_itself_observes_dst(): + # Server clock in Berlin, schedule owned in UTC: the SERVER side of the + # conversion moves at ITS transition, the schedule's identity does not. + berlin = ZoneInfo("Europe/Berlin") + materialize = lambda now: cron_tz.materialize_zone_cron( + "0 5 * * *", "UTC", now=now, server_tz=berlin, + ) + assert materialize(datetime(2026, 7, 1, 12, 0, tzinfo=UTC)) == "0 7 * * *" + assert materialize(datetime(2026, 12, 1, 12, 0, tzinfo=UTC)) == "0 6 * * *" + + +def test_materialize_wraps_across_the_day_boundary(): + # 00:30 in Kathmandu (UTC+5:45) is 18:45 the previous day in UTC; a + # daily job simply wraps — date fields are '*' by contract. + now = datetime(2026, 7, 1, 12, 0, tzinfo=UTC) + assert cron_tz.materialize_zone_cron( + "30 0 * * *", "Asia/Kathmandu", now=now, server_tz=UTC, + ) == "45 18 * * *" + + +def test_materialize_rejects_non_daily_and_unknown_zone(): + now = datetime(2026, 7, 1, 12, 0, tzinfo=UTC) + with pytest.raises(ValueError): + cron_tz.materialize_zone_cron("0 5 * * 1", "UTC", now=now, server_tz=UTC) + with pytest.raises(ValueError): + cron_tz.materialize_zone_cron( + "0 5 * * *", "Not/AZone", now=now, server_tz=UTC, + ) + + +def test_parse_zone_declaration_round_trip(): + text = ( + "#!/bin/sh\n" + 'ENTRY="0 3 * * * python3 /app/scripts/app-job-runner.py 4 ' + '/data/apps/memory/fetch.sh"\n' + "# --- Zone-aware schedule identity (platform-managed).\n" + 'SCHEDULE_TZ="Europe/Belgrade"\n' + 'SCHEDULE_SOURCE="0 5 * * *"\n' + ) + assert cron_tz.parse_zone_declaration(text) == ( + "Europe/Belgrade", "0 5 * * *", + ) + + +@pytest.mark.parametrize("text", [ + "", + 'SCHEDULE_TZ="Europe/Belgrade"\n', # half a declaration + 'SCHEDULE_SOURCE="0 5 * * *"\n', # half a declaration + 'SCHEDULE_TZ="Nope"\nSCHEDULE_SOURCE="0 5 * * *"\n', # unknown zone + 'SCHEDULE_TZ="UTC"\nSCHEDULE_SOURCE="0 5 * * 1"\n', # non-daily source +]) +def test_parse_zone_declaration_rejects_incomplete(text): + assert cron_tz.parse_zone_declaration(text) is None + + +def test_server_timezone_name_prefers_valid_tz_env(monkeypatch): + monkeypatch.setenv("TZ", "Europe/Belgrade") + assert cron_tz.server_timezone_name() == "Europe/Belgrade" + monkeypatch.setenv("TZ", "Total/Nonsense") + # Invalid TZ falls through to /etc/localtime or the UTC default — + # either way a valid IANA identifier comes back. + assert cron_tz.valid_timezone(cron_tz.server_timezone_name()) From f31100d58f17f1e80e0346c7671c7ee67cfda72b Mon Sep 17 00:00:00 2001 From: hamzamerzic <10846014+hamzamerzic@users.noreply.github.com> Date: Tue, 28 Jul 2026 17:40:22 +0000 Subject: [PATCH 2/2] Make IANA schedules execute at truthful wall times MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scheduler now owns gap/fold policy and commits durable declarations before live cron updates. Co-authored-by: Möbius Agent --- backend/app/cron_tz.py | 123 ++++++++++++---- backend/app/install.py | 66 ++++----- backend/app/main.py | 46 ------ backend/app/routes/apps.py | 20 +-- backend/app/schemas.py | 14 +- backend/scripts/app-job-runner.py | 59 ++++++++ backend/scripts/init-cron-scaffold.sh | 74 ++++++++-- backend/tests/test_app_jobs.py | 66 +++++++++ backend/tests/test_apps.py | 58 +++++--- backend/tests/test_apps_install.py | 50 +++++++ backend/tests/test_cron_scaffold_script.py | 159 +++++++++++++++++++++ backend/tests/test_cron_tz.py | 139 +++++++++--------- 12 files changed, 652 insertions(+), 222 deletions(-) diff --git a/backend/app/cron_tz.py b/backend/app/cron_tz.py index a6039fa83..55b7b157a 100644 --- a/backend/app/cron_tz.py +++ b/backend/app/cron_tz.py @@ -1,4 +1,4 @@ -"""Zone-aware cron schedules — durable IANA identity, materialized entries. +"""Zone-aware cron schedules — durable IANA identity, truthful wall clocks. Cron itself evaluates wall-clock time in the server's local timezone, which is not a durable way to express "5:00 AM in Europe/Belgrade": the server's @@ -6,22 +6,27 @@ The durable schedule identity is therefore an IANA timezone name plus a zone-local daily cron expression, declared in the app's ``init-cron.sh`` -(``SCHEDULE_TZ`` / ``SCHEDULE_SOURCE``). The crontab line cron actually runs -is a *materialization* of that identity into the server's current clock, -recomputed at boot and periodically at runtime so an offset change in either -zone reschedules the entry instead of silently drifting the wall time. +(``SCHEDULE_TZ`` / ``SCHEDULE_SOURCE``). Its live crontab materialization runs +the supervised job gate every minute; the gate compares real instants with the +declared zone clock and claims at most one run per local date. Only simple daily expressions (numeric minute + hour, ``* * *`` date fields) -may carry a timezone: shifting an expression across an offset can cross a day -boundary, which is well-defined for a daily job and ambiguous for date-pinned -ones. +may carry a timezone. DST edge behavior is explicit: + +* an ambiguous wall time runs once, at its first occurrence (``fold=0``); +* a nonexistent wall time runs at the first valid minute after the gap; +* a civil date with no valid minute at or after the requested time is skipped. + +That policy gives a daily schedule one deterministic launch on ordinary and +DST-transition dates without pretending a static server-local cron expression +can preserve an IANA wall clock. """ from __future__ import annotations import os import re -from datetime import datetime +from datetime import date, datetime, timedelta, timezone from pathlib import Path from zoneinfo import ZoneInfo @@ -31,6 +36,7 @@ # Written by init-cron-scaffold.sh; parsed (never executed) from init-cron.sh. _DECL_TZ_RE = re.compile(r'^SCHEDULE_TZ="([A-Za-z0-9_+/-]+)"\s*$', re.M) _DECL_SOURCE_RE = re.compile(r'^SCHEDULE_SOURCE="([^"\n]+)"\s*$', re.M) +WALL_CLOCK_CRON = "* * * * *" def valid_timezone(name: str) -> bool: @@ -60,18 +66,48 @@ def parse_daily_cron(expr: str) -> tuple[int, int] | None: def materialize_zone_cron( zone_cron: str, tz_name: str, - *, - now: datetime | None = None, - server_tz=None, ) -> str: - """Expresses a zone-local daily cron in the server's current clock. + """Return the honest live cadence for a zone-local daily schedule. - ``now`` and ``server_tz`` exist for tests; production uses the real clock - and the process-local timezone. The conversion is exact for the current - offset pair — the caller re-materializes when offsets change, which is the - whole durability contract. + The actual wall-clock decision belongs to the supervised job gate. A static + offset expression is intentionally never returned: it would drift or misfire + at the next target-zone or server-zone transition. + """ + if parse_daily_cron(zone_cron) is None: + raise ValueError( + "A timezone-owned schedule must be a plain daily cron " + f"('m h * * *'), got: {zone_cron!r}" + ) + if not valid_timezone(tz_name): + raise ValueError(f"Unknown IANA timezone: {tz_name!r}") + return WALL_CLOCK_CRON + + +def _valid_wall_instants(local: datetime, tz: ZoneInfo) -> list[datetime]: + """UTC instants that round-trip to ``local``, ordered earliest first.""" + instants: set[datetime] = set() + for fold in (0, 1): + candidate = local.replace(tzinfo=tz, fold=fold) + instant = candidate.astimezone(timezone.utc) + round_trip = instant.astimezone(tz) + if ( + round_trip.replace(tzinfo=None) == local + and round_trip.fold == fold + ): + instants.add(instant) + return sorted(instants) + + +def wall_clock_occurrence( + local_date: date, + zone_cron: str, + tz_name: str, +) -> datetime | None: + """The UTC instant selected for one local civil date. - Raises ValueError for a non-daily expression or unknown timezone. + Ambiguous times choose the first occurrence. Nonexistent times advance to + the first valid minute on the same civil date. ``None`` means the remainder + of that civil date does not exist in the zone. """ parsed = parse_daily_cron(zone_cron) if parsed is None: @@ -79,31 +115,60 @@ def materialize_zone_cron( "A timezone-owned schedule must be a plain daily cron " f"('m h * * *'), got: {zone_cron!r}" ) - minute, hour = parsed if not valid_timezone(tz_name): raise ValueError(f"Unknown IANA timezone: {tz_name!r}") + minute, hour = parsed tz = ZoneInfo(tz_name) - moment = (now or datetime.now(tz)).astimezone(tz) - local = moment.replace(hour=hour, minute=minute, second=0, microsecond=0) - server = local.astimezone(server_tz) - return f"{server.minute} {server.hour} * * *" + requested = datetime( + local_date.year, local_date.month, local_date.day, hour, minute, + ) + candidate = requested + while candidate.date() == local_date: + instants = _valid_wall_instants(candidate, tz) + if instants: + return instants[0] + candidate += timedelta(minutes=1) + return None + + +def due_wall_clock_date( + zone_cron: str, + tz_name: str, + *, + now: datetime | None = None, +) -> date | None: + """The local date due at ``now``'s UTC minute, otherwise ``None``.""" + moment = now or datetime.now(timezone.utc) + if moment.tzinfo is None: + raise ValueError("now must be timezone-aware") + moment = moment.astimezone(timezone.utc).replace(second=0, microsecond=0) + tz = ZoneInfo(tz_name) if valid_timezone(tz_name) else None + if tz is None: + raise ValueError(f"Unknown IANA timezone: {tz_name!r}") + local_date = moment.astimezone(tz).date() + occurrence = wall_clock_occurrence(local_date, zone_cron, tz_name) + return local_date if occurrence == moment else None def parse_zone_declaration(init_cron_text: str) -> tuple[str, str] | None: """Extracts (timezone, zone_cron) from init-cron.sh text, if declared. - Returns None when either variable is absent or invalid — an app whose - declaration was hand-edited into an inconsistent state falls back to plain - server-local scheduling rather than guessing. + Returns ``None`` only when both variables are absent. A partial or invalid + platform-managed declaration raises ``ValueError`` so reconciliation fails + closed instead of turning its every-minute gate into an every-minute job. """ tz_match = _DECL_TZ_RE.search(init_cron_text or "") source_match = _DECL_SOURCE_RE.search(init_cron_text or "") - if not tz_match or not source_match: + if not tz_match and not source_match: return None + if not tz_match or not source_match: + raise ValueError("Incomplete IANA wall-clock schedule declaration") tz_name = tz_match.group(1) zone_cron = source_match.group(1).strip() - if not valid_timezone(tz_name) or parse_daily_cron(zone_cron) is None: - return None + if not valid_timezone(tz_name): + raise ValueError(f"Unknown IANA timezone: {tz_name!r}") + if parse_daily_cron(zone_cron) is None: + raise ValueError(f"Invalid zone-local daily cron: {zone_cron!r}") return tz_name, zone_cron diff --git a/backend/app/install.py b/backend/app/install.py index a820e6c3e..39fd21409 100644 --- a/backend/app/install.py +++ b/backend/app/install.py @@ -1133,11 +1133,11 @@ def _register_cron(slug: str, schedule_expr: str, job_path: Path, """Runs init-cron-scaffold.sh to install the crontab entry. ``timezone``/``zone_cron`` (given together) declare the schedule's durable - zone-aware identity: ``schedule_expr`` is then its server-local - materialization, and the identity is appended to ``init-cron.sh`` after the - scaffold rewrites it, so reconciliation can re-materialize the entry when - either zone's UTC offset changes. The scaffold's own contract stays - unchanged; the platform owns the declaration lines (see app.cron_tz). + zone-aware identity. The scaffold writes that identity into the complete + durable declaration atomically *before* installing the live entry. This + ordering means a persistence failure cannot change live behavior while + returning failure; a later live-write failure leaves a durable declaration + that boot reconciliation can safely retry. The scaffold writes both the durable ``init-cron.sh`` declaration and the live crontab entry. On restart, lifespan parses that declaration and rewrites @@ -1167,6 +1167,23 @@ def _register_cron(slug: str, schedule_expr: str, job_path: Path, 500, "Cron mutation is disabled in the test runtime.", ) + if (timezone is None) != (zone_cron is None): + raise HTTPException( + 500, "Cron registration bug: timezone and zone_cron must be paired.", + ) + if timezone is not None: + from app import cron_tz + + if app_id is None: + raise HTTPException( + 500, "A timezone-owned schedule requires an app id.", + ) + if not cron_tz.valid_timezone(timezone): + raise HTTPException(500, f"Unknown IANA timezone: {timezone!r}") + if cron_tz.parse_daily_cron(zone_cron) is None: + raise HTTPException( + 500, f"Zone-owned schedule must be a plain daily cron: {zone_cron!r}", + ) scaffold = _cron_scaffold() if not scaffold.exists(): # In tests we mock this away; in containers it's always present. @@ -1174,6 +1191,8 @@ def _register_cron(slug: str, schedule_expr: str, job_path: Path, cmd = [str(scaffold), slug, schedule_expr, job_path.name] if app_id is not None: cmd.append(str(app_id)) + if timezone is not None: + cmd.extend([timezone, zone_cron]) # Cron has a deliberately minimal environment. Materialize the configured # backend URL and the active supervisor path into its generated entry so # scheduled jobs use the same live runner and server as Run now. @@ -1189,37 +1208,6 @@ def _register_cron(slug: str, schedule_expr: str, job_path: Path, 500, f"Cron registration failed: {result.stderr.strip()[:400]}", ) - if (timezone is None) != (zone_cron is None): - raise HTTPException( - 500, "Cron registration bug: timezone and zone_cron must be paired.", - ) - if timezone is not None: - from app import cron_tz - - if not cron_tz.valid_timezone(timezone): - raise HTTPException(500, f"Unknown IANA timezone: {timezone!r}") - if cron_tz.parse_daily_cron(zone_cron) is None: - raise HTTPException( - 500, f"Zone-owned schedule must be a plain daily cron: {zone_cron!r}", - ) - init_path = job_path.parent / "init-cron.sh" - try: - # The scaffold just rewrote init-cron.sh from its template, so a plain - # append is deterministic and repeat-registration cannot duplicate. - with init_path.open("a", encoding="utf-8") as fh: - fh.write( - "\n# --- Zone-aware schedule identity " - "(platform-managed; parsed, never executed).\n" - "# ENTRY above is the server-local materialization of this durable\n" - "# identity, recomputed at boot and periodically so DST transitions\n" - "# reschedule the entry instead of drifting its wall time.\n" - f'SCHEDULE_TZ="{timezone}"\n' - f'SCHEDULE_SOURCE="{zone_cron}"\n' - ) - except OSError as exc: - raise HTTPException( - 500, f"Could not persist schedule timezone: {exc}", - ) from exc def _reconcile_cron_after_install_rollback() -> None: @@ -1274,7 +1262,11 @@ def _crontab_command_path(line: str) -> str: return "" for i, token in enumerate(toks): if token.endswith("/app-job-runner.py") and len(toks) > i + 2: - return toks[i + 2] + # The supervised runner's job path is always its final argument. A + # wall-clock schedule inserts its timezone and source expression before + # the app id; using the final argument keeps uninstall parsing aligned + # with both ordinary and gated invocations. + return toks[-1] return toks[0] diff --git a/backend/app/main.py b/backend/app/main.py index 91397dfb3..30d417a58 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -442,50 +442,6 @@ async def lifespan(app): _cron_ready.write_text(f"{_BOOT_ID}\n", encoding="utf-8") except Exception as exc: _log.error("app cron supervision wiring failed: %s", exc, exc_info=True) - # Periodic re-materialization for timezone-owned schedules (app.cron_tz): - # a schedule declared as "wall time in an IANA zone" must be rescheduled - # when either that zone's or the server's UTC offset changes (DST). Boot - # reconciliation covers restarts; this hourly pass covers long-running - # containers crossing a transition. It reuses the same idempotent - # reconciler, and only runs while any zone declaration actually exists. - _cron_tz_task = None - try: - from app.routes.apps import ( - _app_zone_declaration as _zone_decl, - reconcile_app_cron_supervision as _cron_reconcile, - ) - from app.database import SessionLocal as _CronTzSession - - async def _cron_tz_loop(): - while True: - await _asyncio.sleep(3600) - try: - def _pass(): - from app import models as _tz_models - _db = _CronTzSession() - try: - _live = _db.query(_tz_models.App).filter( - _tz_models.App.deleted_at.is_(None) - ).all() - if not any(_zone_decl(_a) for _a in _live): - return None - return _cron_reconcile(_db) - finally: - _db.close() - _result = await _asyncio.to_thread(_pass) - if _result is not None: - for _warning in _result[1]: - _log.warning("cron tz re-materialization: %s", _warning) - except _asyncio.CancelledError: - raise - except Exception as _exc: - _log.error( - "cron tz re-materialization failed: %s", _exc, exc_info=True, - ) - - _cron_tz_task = _asyncio.create_task(_cron_tz_loop()) - except Exception as exc: - _log.error("cron tz loop wiring failed: %s", exc, exc_info=True) record_memory_checkpoint("startup_metadata_reconciled") # Route provider model-registry and memory diagnostics to the durable # rotating chat.log handler. These loggers otherwise land only on @@ -709,8 +665,6 @@ async def _browser_profile_loop(): _reset_park_task.cancel() if _browser_profile_task is not None: _browser_profile_task.cancel() - if _cron_tz_task is not None: - _cron_tz_task.cancel() if _writer_supervisor_task is not None: _writer_supervisor_task.cancel() # Drain + join the chat-writer actor so any in-flight persistence diff --git a/backend/app/routes/apps.py b/backend/app/routes/apps.py index 0ff085708..297f0b12f 100644 --- a/backend/app/routes/apps.py +++ b/backend/app/routes/apps.py @@ -621,10 +621,9 @@ def reconcile_app_cron_supervision(db: Session) -> tuple[int, list[str]]: scaffold. Tombstoned apps are excluded and source trees must be ordinary, non-symlink direct children of ``/data/apps``. - A schedule owned in an IANA timezone (see ``app.cron_tz``) is - re-materialized here from its durable (timezone, zone_cron) identity - rather than preserved verbatim — this pass, run at boot and periodically - at runtime, is what reschedules the entry across DST transitions. + A schedule owned in an IANA timezone (see ``app.cron_tz``) is materialized + as an every-minute supervised gate. The gate, not a snapshot of today's UTC + offset, decides the declared wall-clock occurrence at runtime. """ from app import cron_tz from app.install import _register_cron @@ -775,7 +774,12 @@ def list_app_schedules( if schedule is None: continue cron, job = schedule - declaration = _app_zone_declaration(app) + try: + declaration = _app_zone_declaration(app) + except ValueError: + # Listing remains available for repair, while reconciliation itself + # fails closed and never installs the unguarded every-minute cadence. + declaration = None rows.append(schemas.AppScheduleOut( id=app.id, name=app.name, @@ -2914,9 +2918,9 @@ def update_app_schedule( raise HTTPException(status_code=400, detail=str(exc)) from exc timezone = (body.timezone or "").strip() or None if timezone is not None: - # A zone-owned schedule is durable data: body.cron is the daily wall - # time in that zone, and the crontab entry is its server-local - # materialization, re-materialized when either zone's offset changes. + # A zone-owned schedule is durable data: body.cron is the daily wall time + # in that zone, and the crontab entry is an every-minute materialization + # whose supervised gate decides the real due instant. if not cron_tz.valid_timezone(timezone): raise HTTPException( status_code=400, detail=f"Unknown IANA timezone: {timezone!r}", diff --git a/backend/app/schemas.py b/backend/app/schemas.py index a4039249f..388313a09 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -262,9 +262,10 @@ class AppScheduleUpdate(BaseModel): When ``timezone`` (an IANA identifier) is set, ``cron`` is a plain daily expression owned in that zone; the platform stores that identity durably - and materializes/re-materializes the server-local crontab entry itself. - Without it, ``cron`` is interpreted in the server's local timezone as - before. + and materializes an every-minute gate that resolves the real wall-clock + occurrence. Ambiguous times run once at their first occurrence; nonexistent + times run at the first valid minute after the gap. Without ``timezone``, + ``cron`` is interpreted in the server's local timezone as before. """ cron: str @@ -275,10 +276,9 @@ class AppScheduleUpdate(BaseModel): class AppScheduleOut(BaseModel): """Read-only metadata for an installed app's recurring cron job. - ``cron`` is always the materialized server-local entry. When the schedule - is owned in an IANA zone, ``timezone``/``zone_cron`` carry that durable - identity (the platform re-materializes ``cron`` when offsets change). - ``server_timezone`` is the server clock's own IANA identity. + ``cron`` is always the live crontab cadence. For an IANA-owned schedule it + is the every-minute gate; ``timezone``/``zone_cron`` carry the durable wall + clock identity. ``server_timezone`` is the server clock's own IANA identity. """ id: int diff --git a/backend/scripts/app-job-runner.py b/backend/scripts/app-job-runner.py index 191c0ca27..2a9441694 100755 --- a/backend/scripts/app-job-runner.py +++ b/backend/scripts/app-job-runner.py @@ -3,6 +3,7 @@ from __future__ import annotations +import fcntl import json import os import re @@ -18,7 +19,11 @@ _SCRIPT_DIR = Path(__file__).resolve().parent if str(_SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(_SCRIPT_DIR)) +_BACKEND_DIR = _SCRIPT_DIR.parent +if str(_BACKEND_DIR) not in sys.path: + sys.path.insert(0, str(_BACKEND_DIR)) from app_job_sandbox import JobAccess, select_executor +from app import cron_tz DATA_DIR = Path(os.environ.get("DATA_DIR", "/data")) @@ -34,6 +39,7 @@ SUPERVISOR_LOG = DATA_DIR / "cron-logs" / "app-jobs.log" SUPERVISOR_LOG_CAP = 2 * 1024 * 1024 READY_WAIT_SECONDS = 90 +WALL_CLOCK_STATE_DIR = DATA_DIR / "run" / "app-wall-clock" def _log(app_id: object, message: str) -> None: @@ -97,6 +103,44 @@ def _atomic_json(path: Path, value: dict) -> None: raise +def _claim_wall_clock_run( + app_id: int, + job: Path, + tz_name: str, + zone_cron: str, +) -> bool: + """Atomically claim today's due wall-clock occurrence. + + cron invokes this gate every minute. The durable state prevents the repeated + hour at a fall-back transition (and concurrent cron processes) from + launching the same app schedule twice. + """ + due_date = cron_tz.due_wall_clock_date(zone_cron, tz_name) + if due_date is None: + return False + WALL_CLOCK_STATE_DIR.mkdir(parents=True, exist_ok=True) + lock_path = WALL_CLOCK_STATE_DIR / f"{app_id}.lock" + state_path = WALL_CLOCK_STATE_DIR / f"{app_id}.json" + identity = { + "schema": 1, + "app_id": app_id, + "job": str(job), + "timezone": tz_name, + "zone_cron": zone_cron, + "local_date": due_date.isoformat(), + } + with lock_path.open("a", encoding="utf-8") as lock: + fcntl.flock(lock.fileno(), fcntl.LOCK_EX) + try: + previous = json.loads(state_path.read_text(encoding="utf-8")) + except (OSError, ValueError, TypeError): + previous = None + if previous == identity: + return False + _atomic_json(state_path, identity) + return True + + def _app_is_live(app_id: int, token: str | None = None) -> bool: token = (token or os.environ.get("APP_TOKEN", "")).strip() if not token: @@ -223,6 +267,13 @@ def run() -> int: wait_for_ready = argv[:1] == ["--wait-for-ready"] if wait_for_ready: argv = argv[1:] + wall_clock = None + if argv[:1] == ["--wall-clock"]: + if len(argv) < 3: + _log("?", "rejected: incomplete wall-clock argv") + return 2 + wall_clock = (argv[1], argv[2]) + argv = argv[3:] if len(argv) != 2 or not re.fullmatch(r"[0-9]+", argv[0]): _log(argv[0] if argv else "?", "rejected: bad argv") return 2 @@ -243,6 +294,14 @@ def run() -> int: ): _log(app_id, f"rejected: job outside apps root {resolved}") return 2 + if wall_clock is not None: + tz_name, zone_cron = wall_clock + try: + if not _claim_wall_clock_run(app_id, resolved, tz_name, zone_cron): + return 0 + except (OSError, ValueError) as exc: + _log(app_id, f"rejected: invalid wall-clock schedule ({exc})") + return 2 # API launches already create a session; cron launches do not. try: diff --git a/backend/scripts/init-cron-scaffold.sh b/backend/scripts/init-cron-scaffold.sh index c6006b236..816f22e4a 100755 --- a/backend/scripts/init-cron-scaffold.sh +++ b/backend/scripts/init-cron-scaffold.sh @@ -13,6 +13,7 @@ # # Usage: # init-cron-scaffold.sh "" [job-filename] [app-id] +# [iana-timezone zone-local-daily-cron] # # Example: # init-cron-scaffold.sh news "*/10 * * * *" # runs job.sh @@ -66,6 +67,11 @@ JOB_NAME="${3:-job.sh}" # for "Generate now". Empty (default) leaves the command bare, for # self-contained jobs that hardcode their own id. APP_ID="${4:-}" +# Optional 5th + 6th args are platform-owned and always paired. They preserve +# a daily IANA wall-clock identity while the live cron expression simply wakes +# the supervised gate every minute. +SCHEDULE_TZ="${5:-}" +SCHEDULE_SOURCE="${6:-}" # Guards use `case`, not `grep -qE`. A per-line regex (`grep`) passes a # multiline value as long as ONE line matches — so a newline could sneak @@ -99,6 +105,34 @@ case "$APP_ID" in exit 2 ;; esac +if [ "$#" -gt 6 ]; then + echo "ERROR: too many arguments" >&2 + exit 2 +fi +if [ -n "$SCHEDULE_TZ" ] || [ -n "$SCHEDULE_SOURCE" ]; then + if [ -z "$SCHEDULE_TZ" ] || [ -z "$SCHEDULE_SOURCE" ]; then + echo "ERROR: timezone and zone-local cron must be provided together" >&2 + exit 2 + fi + if [ -z "$APP_ID" ]; then + echo "ERROR: a timezone-owned schedule requires an app-id" >&2 + exit 2 + fi + if [[ ! "$SCHEDULE_TZ" =~ ^[A-Za-z0-9_+/-]+$ ]]; then + echo "ERROR: invalid IANA timezone syntax: $SCHEDULE_TZ" >&2 + exit 2 + fi + if [[ ! "$SCHEDULE_SOURCE" =~ ^([0-9]{1,2})[[:blank:]]+([0-9]{1,2})[[:blank:]]+\*[[:blank:]]+\*[[:blank:]]+\*$ ]]; then + echo "ERROR: zone-local cron must be a plain daily expression" >&2 + exit 2 + fi + if [ "$((10#${BASH_REMATCH[1]}))" -gt 59 ] \ + || [ "$((10#${BASH_REMATCH[2]}))" -gt 23 ]; then + echo "ERROR: zone-local cron has an invalid minute or hour" >&2 + exit 2 + fi +fi + # APP_BASE is overridable only so the scaffold is testable without a # container; production never sets it, so the base stays /data/apps. APP_BASE="${MOBIUS_APP_BASE:-/data/apps}" @@ -114,7 +148,11 @@ JOB_API_BASE_URL="${API_BASE_URL:-http://localhost:8000}" JOB_RUNNER="${MOBIUS_APP_JOB_RUNNER:-/app/scripts/app-job-runner.py}" JOB_API_BASE_ASSIGNMENT="API_BASE_URL=$(printf '%q' "$JOB_API_BASE_URL")" JOB_RUNNER_ARG="$(printf '%q' "$JOB_RUNNER")" -if [ -n "$APP_ID" ]; then +if [ -n "$SCHEDULE_TZ" ]; then + SCHEDULE_TZ_ARG="$(printf '%q' "$SCHEDULE_TZ")" + SCHEDULE_SOURCE_ARG="$(printf '%q' "$SCHEDULE_SOURCE")" + CRON_CMD="${JOB_API_BASE_ASSIGNMENT} python3 ${JOB_RUNNER_ARG} --wall-clock ${SCHEDULE_TZ_ARG} ${SCHEDULE_SOURCE_ARG} ${APP_ID} ${JOB_PATH}" +elif [ -n "$APP_ID" ]; then CRON_CMD="${JOB_API_BASE_ASSIGNMENT} python3 ${JOB_RUNNER_ARG} ${APP_ID} ${JOB_PATH}" else CRON_CMD="${JOB_PATH}" @@ -149,9 +187,12 @@ else echo "kept existing $JOB_PATH" fi -# 2. Write init-cron.sh. Always rewrite — the schedule is the only -# variable, and the script body is tiny + standardised. -cat > "$INIT_PATH" < "$INIT_TMP" < "$INIT_PATH" <"\$ERRFILE"); RC=\$? +STATUS=0 if [ "\$RC" -eq 0 ]; then # Authoritative read — keep every other app's line, replace only ours. (printf '%s\\n' "\$EXISTING" | grep -vF "$JOB_PATH"; echo "\$ENTRY") \\ - | crontab -u mobius - + | crontab -u mobius - || STATUS=\$? elif grep -qi 'no crontab for' "\$ERRFILE"; then # Genuinely no crontab yet — safe to install just this entry; lifespan # reconciles every other live app declaration before cron starts. - echo "\$ENTRY" | crontab -u mobius - + echo "\$ENTRY" | crontab -u mobius - || STATUS=\$? else # A real read error (not "no crontab"): do NOT rewrite, or we'd drop every - # other app's entry from a partial/empty read. Leave the crontab as-is. + # other app's entry from a partial/empty read. Leave the crontab as-is and + # report failure; the durable declaration is already safe for a later retry. echo "init-cron($SLUG): crontab read error (rc=\$RC); leaving crontab unchanged" >&2 cat "\$ERRFILE" >&2 + STATUS=\$RC fi rm -f "\$ERRFILE" +exit "\$STATUS" INIT -chmod +x "$INIT_PATH" +if [ -n "$SCHEDULE_TZ" ]; then + cat >> "$INIT_TMP" <&2; exit 1 ;;\n" + " -)\n" + " grep -q '^SCHEDULE_TZ=\"Europe/Belgrade\"$' \"$EXPECTED_INIT\" || exit 41\n" + " grep -q '^SCHEDULE_SOURCE=\"30 2 \\* \\* \\*\"$' \"$EXPECTED_INIT\" || exit 42\n" + " cat > \"$state\" ;;\n" + " *) exit 2 ;;\n" + "esac\n" + ) + crontab.chmod(0o755) + argv_state = tmp_path / "runner-argv.txt" + python = fake_bin / "python3" + python.write_text( + "#!/bin/sh\n" + "printf '%s\\n' \"$@\" > \"$RUNNER_ARGV_STATE\"\n" + ) + python.chmod(0o755) + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "CRONTAB_STATE": str(state), + "EXPECTED_INIT": str(init_path), + "RUNNER_ARGV_STATE": str(argv_state), + "MOBIUS_APP_BASE": str(app_base), + "MOBIUS_ALLOW_TEST_CRON": "1", + "DATA_DIR": str(tmp_path / "data"), + "MOBIUS_APP_JOB_RUNNER": "/live/scripts/app-job-runner.py", + } + script = Path(__file__).parents[1] / "scripts" / "init-cron-scaffold.sh" + + result = subprocess.run( + [ + str(script), "memory", "* * * * *", "fetch.sh", "57", + "Europe/Belgrade", "30 2 * * *", + ], + text=True, capture_output=True, env=env, check=False, + ) + + assert result.returncode == 0, result.stderr + assert "--wall-clock Europe/Belgrade" in init_path.read_text() + live_entry = state.read_text().strip() + assert live_entry.startswith("* * * * *") + command = live_entry.split(maxsplit=5)[5] + subprocess.run(["bash", "-c", command], env=env, check=True) + assert argv_state.read_text().splitlines() == [ + "/live/scripts/app-job-runner.py", + "--wall-clock", + "Europe/Belgrade", + "30 2 * * *", + "57", + str(app_dir / "fetch.sh"), + ] + + +def test_durable_replace_failure_cannot_change_live_crontab(tmp_path): + """A failed atomic declaration write stops before any live side effect.""" + app_base = tmp_path / "apps" + app_dir = app_base / "memory" + app_dir.mkdir(parents=True) + (app_dir / "fetch.sh").write_text("#!/bin/sh\n") + init_path = app_dir / "init-cron.sh" + init_path.write_text("old durable declaration\n") + touched = tmp_path / "crontab-called" + + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + mv = fake_bin / "mv" + mv.write_text("#!/bin/sh\nexit 23\n") + mv.chmod(0o755) + crontab = fake_bin / "crontab" + crontab.write_text(f"#!/bin/sh\ntouch {touched}\nexit 0\n") + crontab.chmod(0o755) + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "MOBIUS_APP_BASE": str(app_base), + "MOBIUS_ALLOW_TEST_CRON": "1", + "DATA_DIR": str(tmp_path / "data"), + } + script = Path(__file__).parents[1] / "scripts" / "init-cron-scaffold.sh" + + result = subprocess.run( + [ + str(script), "memory", "* * * * *", "fetch.sh", "57", + "Europe/Belgrade", "30 2 * * *", + ], + text=True, capture_output=True, env=env, check=False, + ) + + assert result.returncode == 23 + assert init_path.read_text() == "old durable declaration\n" + assert not touched.exists() + + +def test_live_write_failure_leaves_complete_durable_retry_point(tmp_path): + """After the durable commit, a live failure is honest and retryable.""" + app_base = tmp_path / "apps" + app_dir = app_base / "memory" + app_dir.mkdir(parents=True) + (app_dir / "fetch.sh").write_text("#!/bin/sh\n") + init_path = app_dir / "init-cron.sh" + state = tmp_path / "crontab.txt" + state.write_text("0 9 * * * /data/apps/news/fetch.sh 12\n") + + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + crontab = fake_bin / "crontab" + crontab.write_text( + "#!/bin/sh\n" + "state=\"$CRONTAB_STATE\"\n" + "if [ \"$1\" = \"-u\" ]; then shift 2; fi\n" + "case \"$1\" in\n" + " -l) cat \"$state\" ;;\n" + " -) cat >/dev/null; exit 17 ;;\n" + " *) exit 2 ;;\n" + "esac\n" + ) + crontab.chmod(0o755) + env = { + **os.environ, + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "CRONTAB_STATE": str(state), + "MOBIUS_APP_BASE": str(app_base), + "MOBIUS_ALLOW_TEST_CRON": "1", + "DATA_DIR": str(tmp_path / "data"), + } + script = Path(__file__).parents[1] / "scripts" / "init-cron-scaffold.sh" + + result = subprocess.run( + [ + str(script), "memory", "* * * * *", "fetch.sh", "57", + "Europe/Belgrade", "30 2 * * *", + ], + text=True, capture_output=True, env=env, check=False, + ) + + assert result.returncode == 17 + assert state.read_text() == "0 9 * * * /data/apps/news/fetch.sh 12\n" + init_text = init_path.read_text() + assert 'SCHEDULE_TZ="Europe/Belgrade"' in init_text + assert 'SCHEDULE_SOURCE="30 2 * * *"' in init_text diff --git a/backend/tests/test_cron_tz.py b/backend/tests/test_cron_tz.py index 800afe52d..f2dc6118d 100644 --- a/backend/tests/test_cron_tz.py +++ b/backend/tests/test_cron_tz.py @@ -1,21 +1,12 @@ -"""Zone-aware schedule materialization — including DST boundaries. +"""IANA wall-clock scheduling, including DST gaps and folds.""" -The durable schedule identity is (IANA timezone, zone-local daily cron); -the crontab entry is a server-local materialization recomputed when either -zone's UTC offset changes. These tests pin the conversion on both sides of -the 2026 European transitions (spring forward 2026-03-29, fall back -2026-10-25) and in both directions of ownership. -""" - -from datetime import datetime -from zoneinfo import ZoneInfo +from datetime import date, datetime, timezone import pytest from app import cron_tz -UTC = ZoneInfo("UTC") -BELGRADE = ZoneInfo("Europe/Belgrade") +UTC = timezone.utc def test_parse_daily_cron_accepts_plain_daily(): @@ -36,80 +27,88 @@ def test_parse_daily_cron_rejects_non_daily(expr): assert cron_tz.parse_daily_cron(expr) is None -def test_materialize_summer_offset_on_utc_server(): - # 2026-07-01: Belgrade is UTC+2 → 5:00 local = 3:00 server. - now = datetime(2026, 7, 1, 12, 0, tzinfo=UTC) +def test_materialization_is_a_gate_not_a_static_offset_snapshot(): assert cron_tz.materialize_zone_cron( - "0 5 * * *", "Europe/Belgrade", now=now, server_tz=UTC, - ) == "0 3 * * *" - - -def test_materialize_winter_offset_on_utc_server(): - # 2026-12-01: Belgrade is UTC+1 → 5:00 local = 4:00 server. - now = datetime(2026, 12, 1, 12, 0, tzinfo=UTC) + "0 5 * * *", "Europe/Belgrade", + ) == "* * * * *" assert cron_tz.materialize_zone_cron( - "0 5 * * *", "Europe/Belgrade", now=now, server_tz=UTC, - ) == "0 4 * * *" + "30 0 * * *", "Asia/Kathmandu", + ) == "* * * * *" -def test_materialize_across_spring_forward_boundary(): - # Europe's 2026 spring transition: 2026-03-29 02:00 CET → 03:00 CEST. - before = datetime(2026, 3, 28, 12, 0, tzinfo=UTC) - after = datetime(2026, 3, 29, 12, 0, tzinfo=UTC) - materialize = lambda now: cron_tz.materialize_zone_cron( - "0 5 * * *", "Europe/Belgrade", now=now, server_tz=UTC, - ) - assert materialize(before) == "0 4 * * *" # +1 before the switch - assert materialize(after) == "0 3 * * *" # +2 after — entry rescheduled +def test_ordinary_wall_clock_occurrence_tracks_seasonal_offset(): + # The durable identity stays 05:00 Belgrade; its real UTC instant changes. + assert cron_tz.wall_clock_occurrence( + date(2026, 7, 1), "0 5 * * *", "Europe/Belgrade", + ) == datetime(2026, 7, 1, 3, 0, tzinfo=UTC) + assert cron_tz.wall_clock_occurrence( + date(2026, 12, 1), "0 5 * * *", "Europe/Belgrade", + ) == datetime(2026, 12, 1, 4, 0, tzinfo=UTC) -def test_materialize_across_fall_back_boundary(): - # Europe's 2026 fall transition: 2026-10-25 03:00 CEST → 02:00 CET. - before = datetime(2026, 10, 24, 12, 0, tzinfo=UTC) - after = datetime(2026, 10, 25, 12, 0, tzinfo=UTC) - materialize = lambda now: cron_tz.materialize_zone_cron( - "0 5 * * *", "Europe/Belgrade", now=now, server_tz=UTC, +def test_nonexistent_wall_time_runs_at_first_valid_minute_after_gap(): + # 2026-03-29 jumps from 01:59:59 UTC / 02:59:59 CET to 03:00 CEST. + # The declared 02:30 does not exist, so the explicit policy selects 03:00. + occurrence = cron_tz.wall_clock_occurrence( + date(2026, 3, 29), "30 2 * * *", "Europe/Belgrade", + ) + assert occurrence == datetime(2026, 3, 29, 1, 0, tzinfo=UTC) + assert cron_tz.due_wall_clock_date( + "30 2 * * *", "Europe/Belgrade", now=occurrence, + ) == date(2026, 3, 29) + assert cron_tz.due_wall_clock_date( + "30 2 * * *", "Europe/Belgrade", + now=datetime(2026, 3, 29, 1, 30, tzinfo=UTC), + ) is None + + +def test_ambiguous_wall_time_runs_once_at_first_fold(): + # 02:30 occurs at both 00:30 UTC (CEST, fold=0) and 01:30 UTC (CET, + # fold=1). The first occurrence is selected and the repeated one is not due. + occurrence = cron_tz.wall_clock_occurrence( + date(2026, 10, 25), "30 2 * * *", "Europe/Belgrade", ) - assert materialize(before) == "0 3 * * *" - assert materialize(after) == "0 4 * * *" + assert occurrence == datetime(2026, 10, 25, 0, 30, tzinfo=UTC) + assert cron_tz.due_wall_clock_date( + "30 2 * * *", "Europe/Belgrade", now=occurrence, + ) == date(2026, 10, 25) + assert cron_tz.due_wall_clock_date( + "30 2 * * *", "Europe/Belgrade", + now=datetime(2026, 10, 25, 1, 30, tzinfo=UTC), + ) is None -def test_materialize_when_server_itself_observes_dst(): - # Server clock in Berlin, schedule owned in UTC: the SERVER side of the - # conversion moves at ITS transition, the schedule's identity does not. - berlin = ZoneInfo("Europe/Berlin") - materialize = lambda now: cron_tz.materialize_zone_cron( - "0 5 * * *", "UTC", now=now, server_tz=berlin, - ) - assert materialize(datetime(2026, 7, 1, 12, 0, tzinfo=UTC)) == "0 7 * * *" - assert materialize(datetime(2026, 12, 1, 12, 0, tzinfo=UTC)) == "0 6 * * *" +def test_occurrence_can_cross_the_utc_day_boundary(): + assert cron_tz.wall_clock_occurrence( + date(2026, 7, 2), "30 0 * * *", "Asia/Kathmandu", + ) == datetime(2026, 7, 1, 18, 45, tzinfo=UTC) -def test_materialize_wraps_across_the_day_boundary(): - # 00:30 in Kathmandu (UTC+5:45) is 18:45 the previous day in UTC; a - # daily job simply wraps — date fields are '*' by contract. - now = datetime(2026, 7, 1, 12, 0, tzinfo=UTC) - assert cron_tz.materialize_zone_cron( - "30 0 * * *", "Asia/Kathmandu", now=now, server_tz=UTC, - ) == "45 18 * * *" +def test_civil_date_with_no_remaining_valid_minute_is_skipped(): + # Pacific/Apia skipped 2011-12-30 when it moved across the date line. + assert cron_tz.wall_clock_occurrence( + date(2011, 12, 30), "0 0 * * *", "Pacific/Apia", + ) is None -def test_materialize_rejects_non_daily_and_unknown_zone(): - now = datetime(2026, 7, 1, 12, 0, tzinfo=UTC) +def test_wall_clock_functions_reject_bad_contracts(): + with pytest.raises(ValueError): + cron_tz.materialize_zone_cron("0 5 * * 1", "UTC") with pytest.raises(ValueError): - cron_tz.materialize_zone_cron("0 5 * * 1", "UTC", now=now, server_tz=UTC) + cron_tz.materialize_zone_cron("0 5 * * *", "Not/AZone") with pytest.raises(ValueError): - cron_tz.materialize_zone_cron( - "0 5 * * *", "Not/AZone", now=now, server_tz=UTC, + cron_tz.due_wall_clock_date( + "0 5 * * *", "UTC", now=datetime(2026, 7, 1, 5, 0), ) def test_parse_zone_declaration_round_trip(): text = ( "#!/bin/sh\n" - 'ENTRY="0 3 * * * python3 /app/scripts/app-job-runner.py 4 ' + 'ENTRY="* * * * * python3 /app/scripts/app-job-runner.py ' + '--wall-clock Europe/Belgrade 0\\ 5\\ \\*\\ \\*\\ \\* 4 ' '/data/apps/memory/fetch.sh"\n' - "# --- Zone-aware schedule identity (platform-managed).\n" + "# Zone-aware schedule identity (platform-managed).\n" 'SCHEDULE_TZ="Europe/Belgrade"\n' 'SCHEDULE_SOURCE="0 5 * * *"\n' ) @@ -118,21 +117,23 @@ def test_parse_zone_declaration_round_trip(): ) +def test_parse_zone_declaration_returns_none_when_identity_is_absent(): + assert cron_tz.parse_zone_declaration("") is None + + @pytest.mark.parametrize("text", [ - "", 'SCHEDULE_TZ="Europe/Belgrade"\n', # half a declaration 'SCHEDULE_SOURCE="0 5 * * *"\n', # half a declaration 'SCHEDULE_TZ="Nope"\nSCHEDULE_SOURCE="0 5 * * *"\n', # unknown zone 'SCHEDULE_TZ="UTC"\nSCHEDULE_SOURCE="0 5 * * 1"\n', # non-daily source ]) -def test_parse_zone_declaration_rejects_incomplete(text): - assert cron_tz.parse_zone_declaration(text) is None +def test_parse_zone_declaration_rejects_malformed_identity(text): + with pytest.raises(ValueError): + cron_tz.parse_zone_declaration(text) def test_server_timezone_name_prefers_valid_tz_env(monkeypatch): monkeypatch.setenv("TZ", "Europe/Belgrade") assert cron_tz.server_timezone_name() == "Europe/Belgrade" monkeypatch.setenv("TZ", "Total/Nonsense") - # Invalid TZ falls through to /etc/localtime or the UTC default — - # either way a valid IANA identifier comes back. assert cron_tz.valid_timezone(cron_tz.server_timezone_name())