diff --git a/.github/scripts/arxiv_fetch.py b/.github/scripts/arxiv_fetch.py index e6fb7253..8fd4706d 100644 --- a/.github/scripts/arxiv_fetch.py +++ b/.github/scripts/arxiv_fetch.py @@ -126,13 +126,17 @@ def announcement_band(now: dt.datetime) -> tuple: ) -def fetch(query: str, max_results: int) -> bytes: +def fetch(query: str, max_results: int, start: int = 0) -> bytes: + """One page of the API. `start` pages a query too broad for one request — + the strong-lensing digest never needs it (a band is ~1.5 papers), the + interests digest beside it always does (a band is a whole day of astro-ph). + """ params = urllib.parse.urlencode( { "search_query": query, "sortBy": "submittedDate", "sortOrder": "descending", - "start": 0, + "start": start, "max_results": max_results, } ) diff --git a/.github/scripts/arxiv_interests.py b/.github/scripts/arxiv_interests.py new file mode 100755 index 00000000..42272cf9 --- /dev/null +++ b/.github/scripts/arxiv_interests.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Shortlist the day's non-strong-lensing arXiv papers into arxiv_interest_candidates.json. + +The sibling of ``arxiv_fetch.py``, and deliberately its opposite. That script +asks a narrow question — *which of today's papers are about strong lensing?* — +and answers it with a keyword query precise enough that Claude only has to drop +the odd false positive. This one asks a broad one: *of everything announced +today, which ten would this reader most want to see?* Broad is the point (the +reading interest is black holes, dark matter, galaxy formation, statistics, and +whatever else is good), so there is no query narrow enough to answer it, and no +budget to hand Claude the whole day's astro-ph either. + +So the work is split the way it always is here — a deterministic, testable +stage that RANKS, and a Claude stage that JUDGES: + +1. Fetch the whole announcement band across the reading categories, paging + until the band is covered (a band is a few hundred papers, not the handful + the lensing query returns — hence ``fetch(..., start=)``). +2. Score each paper against the interest profile below: term hits in the title + count triple, in the abstract once, and the best-scoring topic becomes the + paper's suggested reading-queue section. Papers scoring nothing are dropped. +3. Write the top :data:`CANDIDATE_CAP` with full abstracts. Claude reads that + file, drops what the keywords oversold, and picks the final ten. + +The scoring is a shortlist, never a verdict: it exists to get the candidate +list down to something one prompt can read closely, and it is generous on +purpose — recall first, exactly as the lensing query is. + +**Strong lensing is not excluded here, only flagged.** Those papers have their +own list (``arxiv_papers.yml`` → PyAutoMemory's ``arxiv-inbox.md``) and must +not appear twice, but a *deterministic* exclusion would open a gap: a dark +matter paper that mentions a lensing constraint in passing matches the lensing +net, gets dropped by the lensing digest as off-topic, and then falls through +both lists. So each candidate carries ``strong_lensing``, and the prompt drops +the ones genuinely about it — one judgement, in the place that can make it. + +Window and band maths are ``arxiv_fetch.py``'s, imported rather than restated: +the two digests must take the same band or a paper can land in the seam. + +Usage: + python3 .github/scripts/arxiv_interests.py [--selftest] +""" +import datetime as dt +import json +import os +import sys +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import arxiv_fetch # noqa: E402 + +#: The reading categories. astro-ph.HE carries the black-hole and transient +#: work that .CO/.GA do not; .IM carries the instrumentation and methods +#: papers. `cat:` matches cross-lists, so a stats-heavy paper whose primary +#: category is stat.ME is still caught when it cross-lists to astro-ph. +CATEGORIES = ("astro-ph.CO", "astro-ph.GA", "astro-ph.HE", "astro-ph.IM") +QUERY = " OR ".join(f"cat:{c}" for c in CATEGORIES) + +#: How many papers the final list holds. The human asked for ten a day. +PICK_COUNT = 10 + +#: How many the prompt gets to choose from. Enough that ten good ones are +#: reliably in there, small enough that one prompt can read every abstract +#: closely — the whole reason for the scoring stage. +CANDIDATE_CAP = 60 + +#: Paging: the API caps a page at ~200 and a 3-day Monday band is several +#: hundred papers. The cap is a runaway guard, not an expected limit; hitting +#: it is reported rather than swallowed. +PAGE_SIZE = 200 +MAX_PAGES = 12 + +#: The interest profile. Each topic's terms are matched case-insensitively as +#: substrings against title and abstract; the topic that scores highest becomes +#: the paper's suggested reading-queue section, so these names are +#: PyAutoMemory `reading-queue.md` section headers, verbatim — a name that +#: matches nothing there falls back to `## Interests` on the Memory side rather +#: than stranding the paper. +#: +#: Generous by design. A term that pulls in the occasional off-topic paper +#: costs one line of a 60-paper shortlist that Claude then ignores; a term +#: missing costs a paper that never appears at all. Recall first. +INTERESTS = { + "SMBHs": ( + "black hole", "black holes", "supermassive", "smbh", "agn", + "active galactic nucle", "quasar", "blazar", "accretion disc", + "accretion disk", "tidal disruption", "event horizon", + "binary black hole", "gravitational wave", "pulsar timing", + "nanohertz", "m-sigma", "reverberation mapping", "jet", + "eddington", "seed black hole", "intermediate-mass black hole", + "little red dot", "sgr a*", "event horizon telescope", + ), + "Dark Matter": ( + "dark matter", "wimp", "axion", "sterile neutrino", "self-interacting", + "subhalo", "substructure", "halo mass function", "cold dark matter", + "warm dark matter", "fuzzy dark matter", "ultra-light", + "primordial black hole", "direct detection", "annihilation", + "dark energy", "modified gravity", "mond", "stellar stream", + "dwarf spheroidal", "core-cusp", "missing satellites", + "cosmological simulation", "n-body", + ), + "Galaxy Formation / Evolution": ( + "galaxy formation", "galaxy evolution", "star formation", + "stellar population", "initial mass function", "quenching", + "quiescent", "feedback", "outflow", "circumgalactic", + "interstellar medium", "morphology", "bulge", "disc galaxy", + "disk galaxy", "elliptical galaxy", "early-type galaxy", + "merger", "high-redshift", "high redshift", "jwst", "cosmic noon", + "reionization", "reionisation", "stellar halo", "globular cluster", + "metallicity", "chemical evolution", "ifu", "integral field", + "scaling relation", "stellar mass function", "dust", + ), + "Stats": ( + "bayesian", "inference", "posterior", "likelihood", "mcmc", + "nested sampling", "sampler", "hamiltonian monte carlo", + "variational", "simulation-based inference", "neural ratio", + "normalizing flow", "normalising flow", "emulator", + "gaussian process", "machine learning", "deep learning", + "neural network", "transformer", "diffusion model", + "uncertainty quantification", "model selection", "evidence", + "hierarchical model", "systematics", "calibration", + "probabilistic programming", "differentiable", "jax", + "convolutional", "anomaly detection", "interpretab", + ), +} + +#: Weights: a term in the title is what the paper is ABOUT; the same term in +#: the abstract may be one sentence of context. +TITLE_WEIGHT = 3 +ABSTRACT_WEIGHT = 1 + +#: The strong-lensing net, borrowed whole from the other digest so the two +#: cannot drift apart on what "strong lensing" means. Used only to FLAG. +LENSING_TERMS = tuple(t.lower() for t in (arxiv_fetch._ABS + arxiv_fetch._TI)) + + +def score(title: str, abstract: str) -> tuple[int, str | None, dict]: + """``(total, best_topic, per_topic)`` for one paper against the profile.""" + lo_title, lo_abs = title.lower(), abstract.lower() + per: dict[str, int] = {} + for topic, terms in INTERESTS.items(): + n = 0 + for term in terms: + if term in lo_title: + n += TITLE_WEIGHT + if term in lo_abs: + n += ABSTRACT_WEIGHT + if n: + per[topic] = n + if not per: + return (0, None, per) + best = max(per, key=lambda k: (per[k], k)) + return (sum(per.values()), best, per) + + +def is_lensing(title: str, abstract: str) -> bool: + """Whether the strong-lensing digest's own net would catch this paper.""" + text = f"{title}\n{abstract}".lower() + return any(term in text for term in LENSING_TERMS) + + +def rank(papers: list[dict], cap: int = CANDIDATE_CAP) -> tuple[list[dict], int]: + """Score, drop the unscored, sort, cap. Returns ``(candidates, scored)``. + + Ties break on the arXiv id rather than input order, so a re-run of the same + band produces the same shortlist — a digest whose output moves when nothing + moved is one nobody can debug. + """ + scored = [] + for p in papers: + total, topic, per = score(p["title"], p["abstract"]) + if not total: + continue + scored.append({**p, + "topic": topic, + "score": total, + "topic_scores": per, + "strong_lensing": is_lensing(p["title"], p["abstract"])}) + scored.sort(key=lambda p: (-p["score"], p["url"])) + return (scored[:cap], len(scored)) + + +def arxiv_id(url: str) -> str: + """`http://arxiv.org/abs/2608.21253v1` → `2608.21253`.""" + return url.rsplit("/", 1)[-1].split("v")[0] + + +def collect_band(band_start, band_end) -> tuple[list[dict], bool]: + """Every paper announced in the band, paging until it is covered. + + Returns ``(papers, truncated)`` — truncated when :data:`MAX_PAGES` ran out + before the band did, which is a real (if unexpected) loss of the band's + oldest papers and is reported rather than swallowed. + """ + seen: set[str] = set() + out: list[dict] = [] + truncated = True + for page in range(MAX_PAGES): + raw = arxiv_fetch.fetch(QUERY, PAGE_SIZE, start=page * PAGE_SIZE) + root = ET.fromstring(raw) + entries = root.findall(f"{arxiv_fetch.ATOM}entry") + if not entries: + truncated = False + break + oldest = None + for paper in arxiv_fetch.parse(raw, band_start, band_end): + if paper["url"] in seen: + continue + seen.add(paper["url"]) + out.append(paper) + for entry in entries: + published = entry.findtext(f"{arxiv_fetch.ATOM}published") + if published: + oldest = dt.datetime.fromisoformat(published.replace("Z", "+00:00")) + if oldest is not None and oldest <= band_start: + truncated = False + break + return (out, truncated) + + +def _selftest() -> int: + """Scoring and ranking, no network.""" + failures = 0 + + def check(label, ok): + nonlocal failures + failures += not ok + print(f" [{'ok' if ok else 'FAIL'}] {label}", file=sys.stderr) + + total, topic, _ = score("A tidal disruption event in a quiescent nucleus", + "We report an accretion disc flare around a " + "supermassive black hole.") + check(f"black-hole paper scores ({total}) and routes to SMBHs ({topic})", + total > 0 and topic == "SMBHs") + + _, topic, _ = score("Constraints on ultra-light dark matter", + "We use stellar streams to bound the axion mass.") + check(f"dark-matter paper routes to Dark Matter ({topic})", + topic == "Dark Matter") + + _, topic, _ = score("Simulation-based inference with misspecified models", + "A normalizing flow posterior over nuisance " + "parameters, validated by nested sampling.") + check(f"methods paper routes to Stats ({topic})", topic == "Stats") + + _, topic, _ = score("Quenching of massive galaxies at cosmic noon", + "JWST spectroscopy of the stellar populations of " + "quiescent galaxies.") + check(f"galaxies paper routes to Galaxy Formation ({topic})", + topic == "Galaxy Formation / Evolution") + + total, _, _ = score("A new species of Antarctic lichen", + "Nothing here is astronomy at all.") + check("an unrelated paper scores zero", total == 0) + + # The title carries the topic; the abstract only mentions it. + title_hit, _, _ = score("Dark matter in dwarf galaxies", "Unrelated prose.") + abs_hit, _, _ = score("Unrelated title", "We mention dark matter once.") + check(f"a title hit outweighs an abstract hit ({title_hit} > {abs_hit})", + title_hit > abs_hit) + + check("a lensing paper is flagged, not dropped", + is_lensing("An Einstein ring in COSMOS", "A strongly lensed source.")) + check("a non-lensing paper is not flagged", + not is_lensing("A quiescent galaxy at z=5", "JWST spectroscopy.")) + + papers = [ + {"title": f"Dark matter paper {i}", "abstract": "dark matter halo", + "url": f"https://arxiv.org/abs/2608.{i:05d}"} for i in range(20) + ] + [{"title": "Unrelated", "abstract": "nothing", "url": "x"}] + top, scored = rank(papers, cap=5) + check(f"rank drops the unscored and caps ({len(top)} of {scored})", + len(top) == 5 and scored == 20) + check("rank is deterministic", rank(papers, cap=5)[0] == top) + check("every candidate carries a topic and a lensing flag", + all(p["topic"] and "strong_lensing" in p for p in top)) + + print(f"selftest: {'PASS' if not failures else f'{failures} FAILURE(S)'}", + file=sys.stderr) + return 1 if failures else 0 + + +def main() -> int: + if "--selftest" in sys.argv: + return _selftest() + + now = dt.datetime.now(dt.timezone.utc) + override = os.environ.get("LOOKBACK_HOURS", "").strip() + if override: + mode = "lookback" + band_start, band_end = now - dt.timedelta(hours=float(override)), now + else: + mode = "announcement-band" + band_start, band_end = arxiv_fetch.announcement_band(now) + + uk_date = os.environ.get("UK_DATE") or now.strftime("%Y-%m-%d") + band, truncated = collect_band(band_start, band_end) + candidates, scored = rank(band) + if truncated: + print(f"::warning::paged out after {MAX_PAGES} pages without reaching " + f"the start of the band — the band's oldest papers were not " + f"considered today.", file=sys.stderr) + + out = { + "uk_date": uk_date, + "mode": mode, + "since": band_start.isoformat(), + "until": band_end.isoformat(), + "categories": list(CATEGORIES), + "pick": PICK_COUNT, + "band_count": len(band), + "scored_count": scored, + "truncated": truncated, + "count": len(candidates), + "papers": [{"title": p["title"], + "authors": p["authors"], + "abstract": p["abstract"], + "url": p["url"], + "id": arxiv_id(p["url"]), + "primary_category": p["primary_category"], + "published": p["published"], + "topic": p["topic"], + "score": p["score"], + "strong_lensing": p["strong_lensing"]} + for p in candidates], + } + with open("arxiv_interest_candidates.json", "w") as f: + json.dump(out, f, indent=2) + + print(f"mode={mode} band={band_start.isoformat()}..{band_end.isoformat()} " + f"announced={len(band)} scored={scored} shortlisted={len(candidates)}", + file=sys.stderr) + for p in candidates[:15]: + flag = " [lensing]" if p["strong_lensing"] else "" + print(f" {p['score']:>3} [{p['topic']}]{flag} {p['title'][:64]}", + file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/arxiv_interests.yml b/.github/workflows/arxiv_interests.yml new file mode 100644 index 00000000..c412e82e --- /dev/null +++ b/.github/workflows/arxiv_interests.yml @@ -0,0 +1,263 @@ +name: pyauto-arxiv-interests + +# The SECOND daily arXiv digest, and the sibling of arxiv_papers.yml. +# +# That one is strong lensing only, and it is the product: a curated Slack post +# in #papers, with the PyAutoMemory inbox as a convenience on top. This one is +# everything else the reader is interested in — black holes, dark matter, +# galaxy formation, statistics — and it has no Slack leg at all. It exists to +# fill one surface: PyAutoMemory's `arxiv-interests.md`, which the knowledge +# board renders under the strong-lensing inbox as the day's ten, each paper +# carrying the same 📄 ➕ 📥 📑 ✖️ actions (user, 2026-08-27). +# +# Why a separate workflow rather than a second leg of arxiv_papers.yml: the +# lensing digest works, is the morning's product, and its Claude step is the +# single point of failure for the Slack post. A broad second query, a much +# larger prompt and a second cross-repo push do not belong on that critical +# path — a bad day here must cost the interests list and nothing else. +# +# THE BACKLOG, which is the design and not an implementation detail: nothing +# here lapses. Each run appends one dated batch of ten; the board shows the +# OLDEST un-cleared batch and one 🧹 button clears that day and reveals the +# next. So a week away is a week of batches to walk through, not a week of +# lapsed suggestions — the inbox's seven-day timer is deliberately NOT copied. +# scripts/interests_actions.py in PyAutoMemory owns that; this workflow only +# supplies papers. +# +# Two stages, the house pattern: a deterministic ranker +# (.github/scripts/arxiv_interests.py) narrows a whole day of astro-ph to ~60 +# candidates, and Claude reads those abstracts and picks the ten. Neither could +# do the other's job — no keyword query answers "most interesting", and no +# prompt reads five hundred abstracts a day. +# +# Timing: 02:30 UTC, half an hour behind arxiv_papers.yml, on the same Mon-Fri +# cadence and the SAME announcement band (arxiv_fetch.announcement_band, shared +# by import — two digests on different bands would leave a seam a paper can +# fall through). Running second is load-bearing in one small way: by then the +# lensing digest has filed its papers into arxiv-inbox.md, and the append here +# dedupes against that file, so a paper never lands on both lists. It also +# keeps the two cross-repo pushes off each other's toes. + +on: + schedule: + - cron: "30 2 * * 1-5" + workflow_dispatch: + inputs: + lookback_hours: + description: "Rolling look-back window (hours) instead of the announcement band — for a manual test-fire or a backfill (e.g. 168 to sweep a week). Blank = the normal announcement band." + required: false + default: "" + +permissions: + contents: read + # claude-code-action needs OIDC to mint its short-lived token at startup; + # without it the action exits before reaching the prompt (arxiv_papers.yml). + id-token: write + +jobs: + arxiv-interests: + runs-on: ubuntu-latest + steps: + # claude-code-action runs `git config` in the workspace at startup and + # dies without a checkout — load-bearing even though the fetch is all API. + - uses: actions/checkout@v4 + + # The scoring and the band maths, no network. Fires before the fetch so a + # regression fails loudly here rather than quietly shortlisting nothing. + - name: Self-test the ranker + run: | + python3 .github/scripts/arxiv_fetch.py --selftest + python3 .github/scripts/arxiv_interests.py --selftest + + - name: Shortlist the day's candidates + id: fetch + run: | + set -euo pipefail + override="${{ github.event.inputs.lookback_hours }}" + if [ -n "$override" ]; then + export LOOKBACK_HOURS="$override" + fi + export UK_DATE="$(TZ=Europe/London date '+%Y-%m-%d (%a)')" + + python3 .github/scripts/arxiv_interests.py + + count=$(python3 -c "import json;print(json.load(open('arxiv_interest_candidates.json'))['count'])") + echo "count=$count" >> "$GITHUB_OUTPUT" + echo "Shortlisted $count candidate(s)." + + - name: Build the empty-day picks + # A band with nothing worth ranking. Skip Claude entirely (zero + # subscription usage) but still write the picks file, so the filing + # step below runs unconditionally and stamps the quiet day. Same + # invariant as arxiv_papers.yml: a MISSING picks file stays a real + # failure, never an empty day. + if: ${{ steps.fetch.outputs.count == '0' }} + run: | + set -euo pipefail + echo '{"papers": []}' > arxiv_interest_picks.json + echo "Nothing shortlisted — wrote an empty picks file." + + - name: Pick the day's ten with Claude + if: ${{ steps.fetch.outputs.count != '0' }} + uses: anthropics/claude-code-action@v1 + with: + # Claude subscription OAuth token — NOT an API key. The org pattern. + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + # Default-hidden output masks real failures (an expired token reads + # as "picks file missing" downstream) — surface everything. + show_full_output: true + # Write only. The runner's GITHUB_TOKEN is contents:read, so nothing + # here can leave the runner except via the filing step. + claude_args: '--allowedTools "Write"' + prompt: | + Read `arxiv_interest_candidates.json` in this directory. It has shape: + + { "uk_date": "YYYY-MM-DD (Mon)", + "pick": 10, + "band_count": , + "count": , + "papers": [ { "title": "", + "authors": ["<name>", ...], + "abstract": "<full abstract>", + "url": "https://arxiv.org/abs/XXXX.XXXXXvN", + "id": "<bare arXiv id>", + "primary_category": "astro-ph.GA", + "topic": "<suggested reading-queue section>", + "score": <keyword score>, + "strong_lensing": true|false }, ... ] } + + These are the papers announced on arXiv overnight that a keyword + ranker judged closest to the reader's interests. The ranker is a + shortlist, not a verdict: `score` and `topic` are hints from + substring matching, and you should overrule either when the + abstract says otherwise. + + The reader is a working astronomer whose interests are, roughly: + supermassive black holes and AGN; dark matter (particle nature, + substructure, cosmological probes); galaxy formation and evolution; + and statistical / computational methods — Bayesian inference, + samplers, simulation-based inference, machine learning applied to + astronomy. They also just like a genuinely interesting or + surprising result, so do not be mechanical about the four buckets. + + FIRST, drop: + - Any paper genuinely ABOUT strong gravitational lensing. + Those go on a separate list and must not appear twice. The + `strong_lensing` flag is a keyword hint, not the answer: a + dark matter or cosmology paper that merely uses or mentions a + lensing constraint is NOT a strong-lensing paper and should + stay in the running. Judge from the abstract. + - Anything the keywords oversold — a paper whose only match is + one incidental phrase, and which the reader would not care + about. + + THEN pick the `pick` most interesting of what remains (fewer only + if fewer survive; never more). Rank by how much this reader would + want to see it: a significant result, a new method they could use, + a dataset or survey they work with, a genuine surprise. Prefer + substance over volume — do not fill the list with near-duplicate + survey papers when a more varied ten is available. + + For each pick, assign `topic` — the PyAutoMemory reading-queue + section it belongs in. Use EXACTLY one of these four strings: + + "SMBHs" + "Dark Matter" + "Galaxy Formation / Evolution" + "Stats" + + Pick the closest one; a paper that fits none still gets the closest + (there is a fallback on the other side, but it is a last resort, + not a bucket). You may overrule the ranker's suggestion. + + Write `arxiv_interest_picks.json` in the repo root, with this exact + shape and nothing else: + + { "papers": [ { "title": "<title verbatim>", + "id": "<bare arXiv id, e.g. 2608.21253>", + "topic": "<one of the four strings above>" }, + ... ] } + + Most interesting first. Titles VERBATIM from the candidates file — + do not re-word or re-case them; they are matched against the + reading queue to avoid filing a paper twice. Strip any `arXiv:` + prefix and any trailing `vN` from the id. + + If nothing survives the drops, do NOT omit the file — write + `{"papers": []}`. An empty day must still be recorded. + + Do not commit, do not post anything anywhere, and do not modify any + file other than `arxiv_interest_picks.json`. + + # --- the Memory interests list ---------------------------------------- + # Runs on EVERY weekday, including days with nothing to file, because it + # also writes the `last digest: <date>` stamp — the board's evidence that + # a quiet list is a quiet day rather than a broken filing. Same invariant + # the strong-lensing inbox carries (PyAutoMemory#58); a quiet day is a + # one-line commit, not silence. + # + # PAT_PYAUTOLABS, not GITHUB_TOKEN: the job token is scoped to this repo + # and cannot write to PyAutoMemory. The line format, the batch cap and + # the transitions all live in PyAutoMemory's scripts/interests_actions.py + # — this step only supplies the papers. + - name: File the picks into the PyAutoMemory interests list + env: + PAT: ${{ secrets.PAT_PYAUTOLABS }} + run: | + set -euo pipefail + if [ ! -s arxiv_interest_picks.json ]; then + echo "::error::arxiv_interest_picks.json missing — the Claude step did not write it (check for an expired CLAUDE_CODE_OAUTH_TOKEN). Every path, empty days included, writes this file, so this is a real failure and today's papers were NOT filed." + exit 1 + fi + if [ -z "${PAT:-}" ]; then + echo "::error::PAT_PYAUTOLABS not set — cannot write to PyAutoMemory. Today's interests batch was NOT filed." + exit 1 + fi + + uk_date=$(python3 -c "import json;print(json.load(open('arxiv_interest_candidates.json'))['uk_date'])") + + git clone --depth 1 \ + "https://x-access-token:${PAT}@github.com/PyAutoLabs/PyAutoMemory.git" \ + /tmp/memory + cp arxiv_interest_picks.json /tmp/memory/ + cd /tmp/memory + + # append is idempotent and dedupes against the interests list, the + # reading queue AND the strong-lensing inbox, so a re-run adds + # nothing and a paper never lands on both lists. + added=$(python3 scripts/interests_actions.py append \ + --papers-file arxiv_interest_picks.json) + # Unconditional, and the whole reason this step does not skip empty + # days: it records that the digest ran at all. + stamped=$(python3 scripts/interests_actions.py stamp) + # NO sweep. The inbox has one; this list is a backlog the human + # clears a day at a time, and ageing batches out would silently + # delete the recommendations they asked to cycle through. + rm arxiv_interest_picks.json + echo "interests: ${added}, ${stamped}" + + if [ -z "$(git status --porcelain -- arxiv-interests.md)" ]; then + echo "interests list unchanged — nothing to commit." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git add arxiv-interests.md + git commit -m "interests: ${added}, ${stamped} (arXiv digest ${uk_date})" + # A rejected push is worth a rebase and another go — the board and the + # queue-action workflows commit to this same repo, and the lensing + # digest pushed half an hour ago. An auth failure is NOT that: the + # token either grants write on PyAutoMemory or it never will, and + # retrying only buries the reason. Split them (the arxiv_papers.yml + # lesson, PyAutoMemory#58). + for attempt in 1 2 3; do + if git push 2>/tmp/interests_push.err; then exit 0; fi + cat /tmp/interests_push.err + if grep -qiE 'denied|403|authentication failed' /tmp/interests_push.err; then + echo "::error::PAT_PYAUTOLABS cannot write to PyAutoLabs/PyAutoMemory (push denied) — the same scope the strong-lensing inbox needs. Today's batch was NOT filed, so the board shows yesterday's day at the top. After granting the scope, re-file with a workflow_dispatch carrying lookback_hours=168." + exit 1 + fi + git pull --rebase origin main || break + done + echo "::error::could not push the interests commit after 3 attempts — today's batch was NOT filed. Re-file with a workflow_dispatch carrying lookback_hours=168 once the cause is fixed." + exit 1 diff --git a/docs/pyautobrain/spawn_spec.md b/docs/pyautobrain/spawn_spec.md index 5fab491e..2b8f0a68 100644 --- a/docs/pyautobrain/spawn_spec.md +++ b/docs/pyautobrain/spawn_spec.md @@ -49,7 +49,7 @@ deliberately, never silently shipped into a template. | 9 | `.github/**` | **Per file, by the succeed-on-a-fresh-repo test below.** Not a blanket rule: owner substitution alone does NOT make a workflow work, because `YOURORG` is a literal placeholder — the template's own `spawn_drift` run failed `repository 'https://github.com/YOURORG/PyAutoMind/' not found`. See rules 9a–9d | | 9a | `.github/workflows/lifecycle_drift.yml` | KEEP verbatim — operates only on its own repo (checkout + local scripts) and contains no owner reference at all, so it needs no substitution and succeeds unmodified in a fresh org. Empirically the one green workflow in the template's run history | | 9b | `.github/workflows/spawn_drift.yml` | DROP — was "keep with the `schedule:` stripped", revised in #125. The self-heal added there makes this workflow depend on `secrets.PAT_PYAUTOLABS` AND on published `*-template` repos, neither of which a freshly-spawned org has, so **every** path in it is unrunnable there and the secret reference alone breaks the no-configured-secret condition. "When in doubt DROP" applies: an org that later publishes templates can adopt this workflow deliberately, having read it. The template still ships `scripts/spawn.py` + `tests/`, so the generator and its guards travel; only the org-coupled automation does not | -| 9c | `.github/workflows/{dashboard_refresh,registry_reconcile,morning_status,morning_health,arxiv_papers,firewall_gate,pages_dashboard,branch_sweep}.yml`, `.github/scripts/**` | DROP — instance automation. `dashboard_refresh.yml` checks out `PyAutoLabs/PyAutoBrain` (the dashboard renderer lives with the intake conductor, not in Mind), so it fails on checkout in any org that has no such sibling — and owner substitution only turns that into the literal `YOURORG/PyAutoBrain`. The rest hardcode sibling repo lists, organ-specific workflow names (`PyAutoHeart`/`PyAutoBrain`/`PyAutoHands`), org secrets (`PYAUTO_PAPERS_WEBHOOK_URL`, `CLAUDE_CODE_OAUTH_TOKEN`) and, in `arxiv_fetch.py`, strong-lensing search vocabulary plus dated incident notes. All 13 failing runs in the published template came from these. Two later additions join them (2026-08, first caught by the 2026-08-24 drift run): `firewall_gate.yml` checks out `PyAutoLabs/{PyAutoBrain,PyAutoHeart,PyAutoHands}` by name — `dashboard_refresh.yml`'s failure mode three times over; and `pages_dashboard.yml` needs a GitHub Pages site the default token cannot create on a fresh repo (the Hands lesson already recorded for Memory's `knowledge_board.yml`) and takes `pages: write` + `id-token: write`. A third joins them (2026-08-25): `branch_sweep.yml` checks out `PyAutoLabs/PyAutoBrain` for the sweep logic — `dashboard_refresh.yml`'s failure mode again — and carries a weekly cron, so it would also trip rule 9's no-unattended-trigger condition on arrival. The sweep is worth having in a mature organism and worth re-adding deliberately; it is not worth a fresh org inheriting a scheduled job that fails on checkout every Sunday | +| 9c | `.github/workflows/{dashboard_refresh,registry_reconcile,morning_status,morning_health,arxiv_papers,arxiv_interests,firewall_gate,pages_dashboard,branch_sweep}.yml`, `.github/scripts/**` | DROP — instance automation. `dashboard_refresh.yml` checks out `PyAutoLabs/PyAutoBrain` (the dashboard renderer lives with the intake conductor, not in Mind), so it fails on checkout in any org that has no such sibling — and owner substitution only turns that into the literal `YOURORG/PyAutoBrain`. The rest hardcode sibling repo lists, organ-specific workflow names (`PyAutoHeart`/`PyAutoBrain`/`PyAutoHands`), org secrets (`PYAUTO_PAPERS_WEBHOOK_URL`, `CLAUDE_CODE_OAUTH_TOKEN`) and, in `arxiv_fetch.py`, strong-lensing search vocabulary plus dated incident notes. All 13 failing runs in the published template came from these. Two later additions join them (2026-08, first caught by the 2026-08-24 drift run): `firewall_gate.yml` checks out `PyAutoLabs/{PyAutoBrain,PyAutoHeart,PyAutoHands}` by name — `dashboard_refresh.yml`'s failure mode three times over; and `pages_dashboard.yml` needs a GitHub Pages site the default token cannot create on a fresh repo (the Hands lesson already recorded for Memory's `knowledge_board.yml`) and takes `pages: write` + `id-token: write`. A third joins them (2026-08-25): `branch_sweep.yml` checks out `PyAutoLabs/PyAutoBrain` for the sweep logic — `dashboard_refresh.yml`'s failure mode again — and carries a weekly cron, so it would also trip rule 9's no-unattended-trigger condition on arrival. The sweep is worth having in a mature organism and worth re-adding deliberately; it is not worth a fresh org inheriting a scheduled job that fails on checkout every Sunday. A fourth joins them (2026-08-27): `arxiv_interests.yml`, the second daily digest, is `arxiv_papers.yml`'s sibling in every relevant way — the same `CLAUDE_CODE_OAUTH_TOKEN` and `PAT_PYAUTOLABS` org secrets, a cron, and a cross-repo push into `PyAutoLabs/PyAutoMemory` by name — and its ranker `.github/scripts/arxiv_interests.py` carries one reader's personal interest vocabulary, which is instance content by definition | | 9d | any other `.github/**` | **No catch-all rule — UNMATCHED by design.** A fallback here is fail-*open*: a workflow added to Mind later would ride it into the template carrying whatever schedule and secrets it has, which is precisely the defect 9a–9c fix. A new `.github` file must fail the run and get an explicit entry above, like every other new file class | | 10 | `.claude/**`, `.codex/**` | DROP — agent-discovery symlinks are install artifacts recreated by the PyAutoBrain installer, not source content | diff --git a/scripts/spawn.py b/scripts/spawn.py index ab7ec630..df4c818a 100644 --- a/scripts/spawn.py +++ b/scripts/spawn.py @@ -131,6 +131,9 @@ (".github/workflows/morning_status.yml", "DROP"), (".github/workflows/morning_health.yml", "DROP"), (".github/workflows/arxiv_papers.yml", "DROP"), + # rule 9c, same as its sibling above: scheduled, needs the papers + # webhook-less cross-repo PAT, and pushes to another org repo. + (".github/workflows/arxiv_interests.yml", "DROP"), # 9c also: the tenant-firewall gate checks out three sibling organ repos by # name (PyAutoBrain/PyAutoHeart/PyAutoHands). A fresh org has none of them, # and owner substitution only turns those into YOURORG/... placeholders — @@ -209,6 +212,9 @@ # content, the instance's overnight suggestions are not. A fresh repo gets # the header and no papers; PyAutoMemory#57. ("arxiv-inbox.md", "EMPTY"), + # EMPTY for the same reason: the day-batch format is template content, the + # instance's backlog of recommendations is not. + ("arxiv-interests.md", "EMPTY"), ("README.md", "SPECIAL:memory_readme"), ] @@ -245,6 +251,7 @@ "queue.md": "# Queue", "reading-queue.md": "# Reading queue", "arxiv-inbox.md": "# arXiv inbox", + "arxiv-interests.md": "# arXiv interests", } # Generated header comments for EMPTY files matched by a glob rather than by diff --git a/tests/test_spawn_template_contract.py b/tests/test_spawn_template_contract.py index 433bb540..298addd9 100644 --- a/tests/test_spawn_template_contract.py +++ b/tests/test_spawn_template_contract.py @@ -114,6 +114,13 @@ def _real(name): ".github/workflows/firewall_gate.yml": _real("firewall_gate.yml"), ".github/workflows/pages_dashboard.yml": _real("pages_dashboard.yml"), ".github/scripts/arxiv_fetch.py": "QUERY = 'strong lensing OR lensed quasar'\n", + ".github/workflows/arxiv_interests.yml": ( + "name: interests\non:\n schedule:\n - cron: \"30 2 * * 1-5\"\n" + "jobs:\n i:\n runs-on: ubuntu-latest\n steps:\n" + " - env:\n PAT: ${{ secrets.PAT_PYAUTOLABS }}\n" + " run: echo x\n" + ), + ".github/scripts/arxiv_interests.py": "CATEGORIES = ('astro-ph.CO',)\n", } DROPPED_GITHUB = [ @@ -135,6 +142,8 @@ def _real(name): # fresh repo, and takes pages:write + id-token:write. ".github/workflows/pages_dashboard.yml", ".github/scripts/arxiv_fetch.py", + ".github/workflows/arxiv_interests.yml", + ".github/scripts/arxiv_interests.py", ]