fix: refresh private report history within one bounded run

.github/workflows/publish-discord-backup.yml

@@ -1,15 +1,21 @@
 name: publish-discord-backup
 
 on:
-  schedule:
-    - cron: "*/15 * * * *"
-    - cron: "17 6 * * *"
+  # Admit a measured manual run before choosing a recurring cadence.
   workflow_dispatch:
     inputs:
-      reconcile_cloud:
-        description: Reconcile the configured private identity plan against Discord before publication
-        type: boolean
-        default: false
+      policy_commit:
+        description: Exact commit on the private maintainer-report-policy branch
+        type: string
+        required: true
+      policy_sha256:
+        description: SHA-256 of the reviewed private selector policy
+        type: string
+        required: true
+      since:
+        description: Optional UTC midnight less than 48 hours ago; defaults to previous UTC midnight
+        type: string
+        required: false
 
 permissions:
   contents: read
@@ -37,11 +43,10 @@ jobs:
           go-version-file: go.mod
           cache: true
 
-      - name: Build native entry points
+      - name: Build native entry point
         timeout-minutes: 10
         run: |
           go build -o "$RUNNER_TEMP/discrawl-native" ./cmd/discrawl
-          go build -o "$RUNNER_TEMP/discrawl-reconcile" scripts/reconcile_cloud_source.go
 
       - name: Configure runtime paths
         timeout-minutes: 1
@@ -52,6 +57,22 @@ jobs:
           echo "DB=$GITHUB_WORKSPACE/.discrawl-ci/discrawl.db" >> "$GITHUB_ENV"
           echo "BACKUP_REPO=$RUNNER_TEMP/discord-backup" >> "$GITHUB_ENV"
           echo "CHECKPOINT_RECEIPT=$RUNNER_TEMP/discrawl/checkpoint.json" >> "$GITHUB_ENV"
+          echo "ACQUISITION_POLICY=$RUNNER_TEMP/discrawl/report-selectors.json" >> "$GITHUB_ENV"
+
+      - name: Verify pinned private acquisition policy
+        timeout-minutes: 2
+        id: policy
+        env:
+          DISCORD_BACKUP_TOKEN: ${{ secrets.DISCORD_BACKUP_TOKEN }}
+          DISCRAWL_BACKUP_REPOSITORY: ${{ secrets.DISCRAWL_BACKUP_REPOSITORY }}
+          DISCRAWL_GUILD_ID: ${{ secrets.DISCRAWL_GUILD_ID }}
+          POLICY_COMMIT: ${{ inputs.policy_commit }}
+          POLICY_SHA256: ${{ inputs.policy_sha256 }}
+          HISTORY_SINCE: ${{ inputs.since }}
+        run: |
+          python3 scripts/private_discord_checkpoint.py policy --policy "$ACQUISITION_POLICY" \
+            --policy-commit "$POLICY_COMMIT" --policy-sha256 "$POLICY_SHA256" \
+            --since "$HISTORY_SINCE" >> "$GITHUB_OUTPUT"
 
       - name: Restore verified private checkpoint
         timeout-minutes: 10
@@ -66,42 +87,23 @@ jobs:
           python3 scripts/private_discord_checkpoint.py restore --prune --db "$DB" --receipt "$CHECKPOINT_RECEIPT"
 
       - name: Sync native Discord state
-        timeout-minutes: ${{ inputs.reconcile_cloud && 7 || 22 }}
+        timeout-minutes: 22
         id: sync
         continue-on-error: true
         env:
           DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
           DISCRAWL_GUILD_ID: ${{ secrets.DISCRAWL_GUILD_ID }}
-          RECONCILE_CLOUD: ${{ inputs.reconcile_cloud && '1' || '0' }}
-          MEMBER_REFRESH: ${{ github.event.schedule == '17 6 * * *' && '1' || '0' }}
+          POLICY_SHA256: ${{ inputs.policy_sha256 }}
+          HISTORY_SINCE: ${{ steps.policy.outputs.since }}
         run: |
           umask 077
           test -n "$DISCORD_BOT_TOKEN" && test -n "$DISCRAWL_GUILD_ID"
           python3 scripts/private_discord_checkpoint.py run --timeout 60 -- \
             "$RUNNER_TEMP/discrawl-native" --config "$CONFIG" init --db "$DB" --guild "$DISCRAWL_GUILD_ID"
-          sync_timeout=1200
-          if [ "$RECONCILE_CLOUD" = "1" ]; then sync_timeout=300; fi
-          sync_args=(--guild "$DISCRAWL_GUILD_ID" --skip-members --latest-only --no-update)
-          if [ "$MEMBER_REFRESH" = "1" ]; then
-            sync_args=(--guild "$DISCRAWL_GUILD_ID" --with-members --latest-only --no-update)
-          fi
-          python3 scripts/private_discord_checkpoint.py run --timeout "$sync_timeout" \
-            --writer-receipt "$RUNNER_TEMP/discrawl/sync.json" -- \
-            "$RUNNER_TEMP/discrawl-native" --config "$CONFIG" sync "${sync_args[@]}"
-
-      - name: Reconcile requested native source
-        timeout-minutes: 16
-        id: reconcile
-        if: ${{ steps.sync.outcome == 'success' && github.event_name == 'workflow_dispatch' && inputs.reconcile_cloud }}
-        continue-on-error: true
-        env:
-          DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
-          DISCRAWL_GUILD_ID: ${{ secrets.DISCRAWL_GUILD_ID }}
-          DISCRAWL_CLOUD_RECONCILE_PLAN: ${{ secrets.DISCRAWL_CLOUD_RECONCILE_PLAN }}
-          DISCRAWL_CLOUD_ARCHIVE: ${{ secrets.DISCRAWL_CLOUD_ARCHIVE }}
-        run: |
-          python3 scripts/private_discord_checkpoint.py run --timeout 900 \
-            --writer-receipt "$RUNNER_TEMP/discrawl/reconcile.json" -- "$RUNNER_TEMP/discrawl-reconcile"
+          python3 scripts/private_discord_checkpoint.py acquire --timeout 1200 \
+            --writer-receipt "$RUNNER_TEMP/discrawl/sync.json" \
+            --native "$RUNNER_TEMP/discrawl-native" --config "$CONFIG" \
+            --policy "$ACQUISITION_POLICY" --policy-sha256 "$POLICY_SHA256" --since "$HISTORY_SINCE"
 
       - name: Verify private ingestion checkpoint
         timeout-minutes: 15
@@ -110,14 +112,13 @@ jobs:
         env:
           DISCORD_BACKUP_TOKEN: ${{ secrets.DISCORD_BACKUP_TOKEN }}
           DISCRAWL_BACKUP_REPOSITORY: ${{ secrets.DISCRAWL_BACKUP_REPOSITORY }}
-          WRITER_RECEIPT: ${{ steps.reconcile.outcome != 'skipped' && 'reconcile.json' || 'sync.json' }}
         run: |
           python3 scripts/private_discord_checkpoint.py save --db "$DB" --receipt "$CHECKPOINT_RECEIPT" \
-            --writer-receipt "$RUNNER_TEMP/discrawl/$WRITER_RECEIPT"
+            --writer-receipt "$RUNNER_TEMP/discrawl/sync.json"
 
       - name: Require successful native ingestion
         timeout-minutes: 1
-        if: ${{ always() && !cancelled() && (steps.sync.outcome != 'success' || steps.reconcile.outcome == 'failure') }}
+        if: ${{ always() && !cancelled() && steps.sync.outcome != 'success' }}
         run: |
           echo "::error::Native ingestion did not complete. Publication was not requested; check the private checkpoint step for retained progress."
           exit 1
@@ -150,31 +151,3 @@ jobs:
             --message "sync: discord archive" --push >/dev/null 2>&1
           echo "published=true" >> "$GITHUB_OUTPUT"
           echo "Archive publication completed."
-
-      - name: Setup Node for cloud publication
-        timeout-minutes: 3
-        if: ${{ vars.DISCRAWL_CLOUD_PUBLISH_ENABLED == '1' && steps.git-publish.outputs.published == 'true' }}
-        uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
-        with:
-          node-version: '24'
-
-      - name: Publish the same current state to D1 and R2
-        if: ${{ vars.DISCRAWL_CLOUD_PUBLISH_ENABLED == '1' && steps.git-publish.outputs.published == 'true' }}
-        timeout-minutes: 40
-        env:
-          DISCRAWL_CLOUD_ENDPOINT: ${{ secrets.DISCRAWL_CLOUD_ENDPOINT }}
-          DISCRAWL_CLOUD_ARCHIVE: ${{ secrets.DISCRAWL_CLOUD_ARCHIVE }}
-          DISCRAWL_CLOUD_AUTH_GITHUB_TOKEN: ${{ secrets.DISCORD_BACKUP_TOKEN }}
-          DISCRAWL_CLOUD_ACCESS_CLIENT_ID: ${{ secrets.DISCRAWL_CLOUD_ACCESS_CLIENT_ID }}
-          DISCRAWL_CLOUD_ACCESS_CLIENT_SECRET: ${{ secrets.DISCRAWL_CLOUD_ACCESS_CLIENT_SECRET }}
-          DISCRAWL_CLOUD_SNAPSHOT: ${{ runner.temp }}/discrawl/cloud.db
-          DISCRAWL_CLOUD_SOURCE_DB: ${{ github.workspace }}/.discrawl-ci/discrawl.db
-        run: |
-          umask 077
-          trap 'rm -f "$DISCRAWL_CLOUD_SNAPSHOT"' EXIT
-          trap 'echo "::error::Cloud publication failed after archive publication."; exit 1' ERR
-          DISCRAWL_CLOUD_SOURCE_COMMIT="$(git -C "$BACKUP_REPO" rev-parse HEAD 2>/dev/null)"
-          export DISCRAWL_CLOUD_SOURCE_COMMIT
-          "$RUNNER_TEMP/discrawl-native" --config "$CONFIG" cloud publish --export-only "$DISCRAWL_CLOUD_SNAPSHOT" >/dev/null 2>&1
-          node scripts/publish-cloud.mjs >/dev/null 2>&1
-          echo "Cloud publication completed."

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add manual, privately scoped report-history acquisition after the normal guild refresh, sharing one time budget and preserving failed progress in private checkpoints; hold recurring and cloud publication until separately qualified.
 - Identify private checkpoint API failures without exposing private output, and inspect a retained failed migration through a bounded read-only diagnostic.
 
 - Preserve complete native Discord crawl state in verified private checkpoints, with atomic restore, bounded retention, and an explicit migration from the retired workflow cache transport.

docs/guides/private-checkpoints.md

@@ -22,7 +22,7 @@ process group to disappear before issuing its completion receipt. Only then can
 the checkpoint helper open a read transaction and copy SQLite through its backup
 API. The pinned image supplies schema, table counts and integrity results.
 
-The checkpoint is the **native ingestion boundary before Git/cloud publication**.
+The checkpoint is the **native ingestion boundary before Git publication**.
 It deliberately precedes the publisher's later `share:*` bookkeeping. Replaying
 that boundary preserves messages, native cursors and failure history. Every
 workflow sets `DISCRAWL_NO_AUTO_UPDATE=1`; sync also uses `--no-update`. The daily
@@ -36,7 +36,7 @@ Workflow concurrency provides serialization; GitHub's latest update is not a
 server-side compare-and-swap operation.
 
 Normal ingestion failure or the supervisor's time limit can still save committed
-partial progress, but the job fails and skips Git/cloud publication. A forced
+partial progress, but the job fails and skips Git publication. A forced
 runner cancellation or loss cannot guarantee a new upload. The previously
 verified checkpoint remains the recovery point until a new one is promoted.
 The reporter visibly skips a checkpoint unless its successful producer tuple
@@ -58,13 +58,72 @@ the next ingestion until that same deletion succeeds. An interrupted draft also
 blocks new ingestion; an operator must reconcile that exact attempt. Do not
 bulk-delete releases by age or prefix.
 
-The publisher has a 125-minute job ceiling. Phase caps total at most 120 minutes:
-checkout/setup/build/configuration 18, restore 10, ingestion/reap 23, checkpoint
-15, outcome gate 1, Git publication 10, Node setup 3 and cloud publication 40.
-Normal sync gets 20 minutes; optional reconciliation shares that allowance with
-a five-minute sync and a fifteen-minute reconciliation. This ceiling does not
+The publisher has a 125-minute job ceiling. Phase caps total at most 79 minutes:
+checkout/setup/build/configuration 18, policy validation 2, restore 10,
+ingestion/reap 23, checkpoint 15, outcome gate 1 and Git publication 10.
+The two acquisition phases share one 20-minute deadline. This ceiling does not
 promise a fifteen-minute completion time or durable FIFO scheduling.
 
+## Private bounded acquisition
+
+The publisher currently has **only a manual trigger**. Keep it disabled until
+the full-state migration is verified. Enabling it later permits a measured
+manual run without also starting a cron schedule. Recurring acquisition and the
+daily member refresh need a later reviewed cadence change, based on measured
+restore, acquisition, checkpoint and publication durations.
+
+Each manual run requires `policy_commit` and `policy_sha256`. The existing
+backup credential reads `policy/report-selectors.json` from the private backup
+repository. Its `refs/heads/maintainer-report-policy` must point to exactly the
+requested commit; the file is then fetched by that commit and its bytes must
+match the requested SHA-256. A missing, changed or invalid policy stops before
+native state is restored. No policy branch, repository variable or credential
+is created by the workflow.
+
+The policy is a JSON object with these exact fields:
+
+- `schema`: integer `1`.
+- `policy`: `repository`, full `revision`, repository-relative `path` and
+  `sha256` identifying the reviewed canonical report selector source.
+- `catalog`: `archive_commit` and `manifest_sha256` identifying the catalog used
+  to select the roots and retained children.
+- `guild_id`: the exact configured Discord guild ID.
+- `message_root_ids` and `forum_root_ids`: arrays of native decimal IDs.
+- `retained_threads`: objects with `id` and `parent_id`; each parent must be one
+  of the declared message or forum roots. Retained children of text roots must
+  be explicit targets: native targeted discovery expands forum children only.
+
+The approved policy digest binds that provenance and scope together. The file
+is limited to 64 KiB and 4,096 distinct targets in total. It cannot contain
+commands, arbitrary flags, wildcards, names or additional fields. Creation and
+review of this private policy are separate operator actions. Do not store it on
+archive `main`, in SQLite, or in public workflow inputs, logs or artifacts. The
+workflow keeps its temporary copy private and outside all export paths.
+
+The supervisor first runs the existing guild-wide latest-only refresh. It then
+runs native `sync --full --since` on the policy targets, starting at the previous
+UTC midnight by default. An optional `since` input may pin a UTC midnight less
+than 48 hours ago. It is fixed before restore and revalidated before acquisition;
+a stale or future value stops rather than silently selecting a different day.
+The second phase uses `--skip-members`, `--with-media=false` and `--no-update`.
+Both phases share the same 1,200-second deadline, and the second
+starts only after the first process group has exited. Any failure prevents Git
+publication; committed partial work can still enter a verified private
+checkpoint after all owned processes are reaped.
+
+This policy limits **acquisition cost and scope**, not archive privacy. Native
+forum selection includes all bot-visible children of the selected forums,
+including newly discovered children. Full forum discovery traverses archived
+thread metadata even though message acquisition is date-bounded. The normal
+Discord permissions and native unavailable markers still apply. A successful
+process does not mean every channel is available or complete; report projection
+must inspect each channel's bounded receipts and preserve its privacy rules.
+
+This publisher does not export or upload to cloud storage. Existing cloud data
+is unchanged and is not made current by this private-only run. Restoring cloud
+publication requires separate verification of its configured destination and
+access policy.
+
 ## One-time migration
 
 Keep the publisher, daily reporter and retired cache-repair workflow disabled.

scripts/private_checkpoint_workflow_test.go

@@ -56,8 +56,13 @@ func TestPrivateCheckpointWorkflowBoundaries(t *testing.T) {
 			require.Contains(t, text[checkpoint:guard], "always() && !cancelled()")
 			require.Contains(t, text[checkpoint:guard], "--writer-receipt")
 			require.Contains(t, text[publish:], "success() && steps.checkpoint.outcome == 'success'")
-			require.Contains(t, text[publish:], "steps.git-publish.outputs.published == 'true'")
-			require.Contains(t, text, "--no-update")
+			require.Contains(t, text, "private_discord_checkpoint.py acquire --timeout 1200")
+			require.Contains(t, text, "private_discord_checkpoint.py policy")
+			require.NotContains(t, text, "schedule:")
+			require.NotContains(t, text, "cloud publish")
+			require.NotContains(t, text, "publish-cloud.mjs")
+			require.NotContains(t, text, "DISCRAWL_CLOUD_")
+			require.NotContains(t, text, "reconcile_cloud")
 			require.NotContains(t, text, "actions/cache/restore")
 		case "discord-backup-report":
 			require.NotContains(t, text, " subscribe ")