fix(seal): stop design/real-seal epoch desync

crates/design-challenge/src/lib.rs

@@ -44,7 +44,10 @@ pub use design_store_pg::DbDesignStore;
 pub use host_sim::{
     force_sim_refusal_reason, host_sim_allowed, is_prod_env, require_host_sim_for_force,
 };
-pub use orchestrator::{ErrorClass, Orchestrator, OrchestratorConfig};
+pub use orchestrator::{
+    design_emit_plan, DesignEmitPlan, ErrorClass, Orchestrator, OrchestratorConfig,
+    DESIGN_EMIT_LATE_BLOCKS,
+};
 
 /// Crate identity smoke.
 #[must_use]
@@ -62,4 +65,30 @@ mod tests {
         assert_eq!(CHALLENGE_ID, "design");
         assert_eq!(SCORING_VERSION, 3);
     }
+
+    #[test]
+    fn emit_plan_waits_until_late_tempo_for_current_epoch() {
+        assert!(design_emit_plan(10, 11, 200, 360, 1000).is_none());
+        let p = design_emit_plan(10, 11, 360 - DESIGN_EMIT_LATE_BLOCKS, 360, 1000).unwrap();
+        assert_eq!(
+            p,
+            DesignEmitPlan {
+                epoch: 11,
+                pin_block: 1000
+            }
+        );
+    }
+
+    #[test]
+    fn emit_plan_catches_up_skipped_epochs_without_waiting() {
+        // Prod failure mode: award/boundary race skipped 24413 while chain is 24423.
+        let p = design_emit_plan(24412, 24423, 50, 360, 8_815_687).unwrap();
+        assert_eq!(p.epoch, 24413);
+        assert_eq!(p.pin_block, 8_815_687 - (24423 - 24413) * 360);
+    }
+
+    #[test]
+    fn emit_plan_noop_when_already_emitted_current() {
+        assert!(design_emit_plan(11, 11, 350, 360, 1000).is_none());
+    }
 }

crates/design-challenge/src/orchestrator.rs

@@ -285,8 +285,8 @@ impl<C: ChainClient + Send + Sync + 'static> Orchestrator<C> {
         }
     }
 
-    /// Late-tempo D24 filler: `NotAttempted` coverage when no admin award fired
-    /// (waits ~last 48 blocks so `award_round` can land Score leaves first).
+    /// D24 filler + catch-up: covers epochs with no admin award, and repairs
+    /// gaps left by the end-of-epoch boundary race (see [`design_emit_plan`]).
     pub async fn run_emitter(self: Arc<Self>)
     where
         C: Sync,
@@ -310,14 +310,21 @@ impl<C: ChainClient + Send + Sync + 'static> Orchestrator<C> {
         let state = chain::gather_schedule_state(self.chain.as_ref(), self.cfg.netuid)
             .map_err(|e| format!("schedule: {e}"))?;
         let epoch = state.subnet_epoch_index;
-        if epoch == 0 || self.emitted_epoch.load(Ordering::Relaxed) >= epoch {
-            return Ok(false);
-        }
         let tempo = u64::from(state.tempo.max(1));
-        if state.blocks_since_last_step.saturating_add(48) < tempo {
+        let last = self.emitted_epoch.load(Ordering::Relaxed);
+        let Some(plan) = design_emit_plan(
+            last,
+            epoch,
+            state.blocks_since_last_step,
+            tempo,
+            state.last_epoch_block,
+        ) else {
             return Ok(false);
-        }
-        self.emit_leaves().await?;
+        };
+        // Pin epoch + block from this tick's schedule snapshot — do **not**
+        // re-read chain inside emit (end-of-epoch flip used to relabel the set
+        // as E+1 and permanently skip E, starving real-seal with D24 409s).
+        self.emit_leaves_at(plan.epoch, plan.pin_block).await?;
         Ok(true)
     }
 
@@ -1089,15 +1096,22 @@ impl<C: ChainClient + Send + Sync + 'static> Orchestrator<C> {
     async fn emit_leaves(&self) -> Result<(), String> {
         let state = chain::gather_schedule_state(self.chain.as_ref(), self.cfg.netuid)
             .map_err(|e| format!("schedule: {e}"))?;
-        // Label with the *current* chain epoch (see the prism emitter for the
-        // full rationale): the expected set is pinned at `last_epoch_block`,
-        // so the label and the covered metagraph must refer to the same epoch
-        // or D24 exact-match against the other challenge 409s under churn.
-        let epoch = state.subnet_epoch_index;
+        self.emit_leaves_at(state.subnet_epoch_index, state.last_epoch_block)
+            .await
+    }
+
+    /// Submit a D24-complete design leaf set for a pinned `(epoch, pin_block)`.
+    ///
+    /// `pin_block` must be that epoch's start block (`LastEpochBlock` while the
+    /// epoch is current, or `current_last_epoch_block - k*tempo` when catching up).
+    async fn emit_leaves_at(&self, epoch: u64, pin_block: u64) -> Result<(), String> {
+        if epoch == 0 {
+            return Err("refuse emit for epoch 0".into());
+        }
         let block_hash = self
             .chain
-            .block_hash(state.last_epoch_block)
-            .map_err(|e| format!("block_hash: {e}"))?;
+            .block_hash(pin_block)
+            .map_err(|e| format!("block_hash@{pin_block}: {e}"))?;
         let expected: ExpectedSet = expected_set_at_chain(
             &trustroot::ParticipantPolicy::AllMetagraphHotkeys,
             PinnedBlockHash::new(block_hash),
@@ -1131,17 +1145,74 @@ impl<C: ChainClient + Send + Sync + 'static> Orchestrator<C> {
         submit_signed_leaf_set(self.gateway.as_ref(), &signed)
             .await
             .map_err(|e| e.to_string())?;
-        self.emitted_epoch.store(epoch, Ordering::Relaxed);
+        self.emitted_epoch.fetch_max(epoch, Ordering::Relaxed);
         info!(
             epoch,
             participants = expected_set.len(),
-            last_epoch_block = state.last_epoch_block,
+            pin_block,
             "design leaf set submitted"
         );
         Ok(())
     }
 }
 
+/// How many blocks before epoch end the NotAttempted filler may run.
+///
+/// Wider than the historical 48-block window so `base-real-seal` (10 min) still
+/// has time to seal after design emits, while leaving most of the epoch for
+/// `award_round` to land Score leaves first (first-write-wins).
+pub const DESIGN_EMIT_LATE_BLOCKS: u64 = 96;
+
+/// Planned design leaf emission for one emitter tick.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct DesignEmitPlan {
+    /// Epoch label for the leaf set.
+    pub epoch: u64,
+    /// Metagraph pin block (epoch start).
+    pub pin_block: u64,
+}
+
+/// Decide whether/which epoch the design filler should emit.
+///
+/// - Catch up `last_emitted+1` when behind by more than one epoch (repairs the
+///   end-of-epoch relabel race that skipped alternate epochs in prod).
+/// - Otherwise wait until the last [`DESIGN_EMIT_LATE_BLOCKS`] of the current
+///   epoch so admin awards can submit Score leaves first.
+#[must_use]
+pub fn design_emit_plan(
+    last_emitted: u64,
+    current_epoch: u64,
+    blocks_since_last_step: u64,
+    tempo: u64,
+    current_last_epoch_block: u64,
+) -> Option<DesignEmitPlan> {
+    if current_epoch == 0 {
+        return None;
+    }
+    let tempo = tempo.max(1);
+    if last_emitted >= current_epoch {
+        return None;
+    }
+    // Sequential catch-up for skipped epochs (award path / boundary race).
+    if last_emitted + 1 < current_epoch {
+        let target = last_emitted + 1;
+        let epochs_back = current_epoch.saturating_sub(target);
+        let pin_block = current_last_epoch_block.saturating_sub(epochs_back.saturating_mul(tempo));
+        return Some(DesignEmitPlan {
+            epoch: target,
+            pin_block,
+        });
+    }
+    // Current epoch: late-tempo filler only.
+    if blocks_since_last_step.saturating_add(DESIGN_EMIT_LATE_BLOCKS) < tempo {
+        return None;
+    }
+    Some(DesignEmitPlan {
+        epoch: current_epoch,
+        pin_block: current_last_epoch_block,
+    })
+}
+
 #[async_trait]
 impl<C: ChainClient + Send + Sync + 'static> AdminAwardHook for Orchestrator<C> {
     async fn on_winners(&self, round_id: u64, _harness_ids: &[String]) -> Result<(), String> {

crates/validator-verify/src/coordination.rs

@@ -276,6 +276,9 @@ pub struct WeightsLatestView {
     /// IP) fails loudly instead of surfacing as an opaque bundle verify error.
     #[serde(default)]
     pub netuid: Option<u16>,
+    /// Metagraph block pinned by the seal (for prune / lag pressure checks).
+    #[serde(default)]
+    pub metagraph_block: Option<u64>,
 }
 
 impl WeightsLatestView {

crates/validator/src/epoch_loop.rs

@@ -270,6 +270,39 @@ where
         // Fail-closed burn (sealed=false) or incomplete legacy body — no Match path.
         return Ok(None);
     };
+    // Pressure / verify: sealed vector must stay near the live chain epoch.
+    // A stuck real-seal (D24 incomplete) leaves `/v1/weights/latest` on an old
+    // chain-scale bundle; Match can still succeed while emission is stale.
+    if let Some(cfg) = submit {
+        if let Ok(chain_epoch) = chain.subnet_epoch_index(cfg.netuid) {
+            let lag = chain_epoch.saturating_sub(epoch);
+            if lag > 1 {
+                warn!(
+                    event = "validator_seal_lag",
+                    sealed_epoch = epoch,
+                    chain_epoch,
+                    lag_epochs = lag,
+                    metagraph_block = ?latest.metagraph_block,
+                    "pressure verify: sealed weights lag chain epoch; check design/prism emit + base-real-seal"
+                );
+            }
+        }
+        if let (Ok(tip), Some(mg_block)) = (chain.current_block(), latest.metagraph_block) {
+            let block_lag = tip.saturating_sub(mg_block);
+            // Public Finney RPC prunes ~256 blocks; beyond that Match cannot
+            // re-fetch the seal's metagraph on a cold RPC (ops red line).
+            if block_lag > 256 {
+                warn!(
+                    event = "validator_seal_metagraph_stale",
+                    sealed_epoch = epoch,
+                    metagraph_block = mg_block,
+                    tip_block = tip,
+                    block_lag,
+                    "pressure verify: seal metagraph_block outside ~256-block prune window"
+                );
+            }
+        }
+    }
     let outcome = fetch_and_compare(client, epoch, chain, trust).await;
     match &outcome {
         ComparisonOutcome::Match {
@@ -708,4 +741,44 @@ mod tests {
         maybe_submit_match(&outcome, &chain, &ReadyDrand, None, &dedupe);
         assert!(chain.call_log().is_empty());
     }
+
+    #[tokio::test]
+    async fn tick_pressure_verify_allows_match_when_seal_lags_chain() {
+        // Same metagraph as the sealed fixture, but chain epoch/tip far ahead —
+        // pressure-verify warns (validator_seal_lag) yet Match must still proceed.
+        let epoch = 77u64;
+        let (client, _chain, trust, merkle_root, _) = sealed_match_fixture(epoch).await;
+        let miner = [0xA1u8; 32];
+        let chain = FakeChain::new(FakeChainConfig {
+            current_block: 10_000,
+            subnet_epoch_index: epoch + 11,
+            hotkeys: vec![miner.to_vec()],
+            owner_hotkey: miner.to_vec(),
+            commit_reveal_enabled: true,
+            last_epoch_block: 500,
+            ..FakeChainConfig::default()
+        });
+        let dedupe = EpochSubmitDedupe::new();
+        let submit = CoordinationSubmitConfig {
+            netuid: 1,
+            hotkey: vec![0xBBu8; 32],
+            version_key: 3,
+            epoch_length: 360,
+        };
+        let out = coordination_compare_once(&client, &chain, &trust, Some(&submit), &dedupe)
+            .await
+            .expect("ok")
+            .expect("some");
+        match out {
+            ComparisonOutcome::Match {
+                epoch: e,
+                merkle_root: root,
+                ..
+            } => {
+                assert_eq!(e, epoch);
+                assert_eq!(root, merkle_root);
+            }
+            other => panic!("expected Match despite seal lag, got {other:?}"),
+        }
+    }
 }

deploy/AGENTS.md

@@ -96,7 +96,7 @@ cargo run -q --release -p weights-smoke -- \
 
 A seal older than ~256 blocks can never be verified by the validator (public RPC prunes state) — if `GET /v1/weights/latest` shows `metagraph_block` lagging tip by thousands of blocks, check `systemctl status base-burn-seal.timer` and `/var/log/base-burn-seal.log` on the master.
 
-**Real-epoch sealer (post burn-seal retirement):** `base-real-seal.timer` (every 10 min) drives [`scripts/prod-real-seal.sh`](scripts/prod-real-seal.sh), which seals the **current chain epoch** with `block_b = LastEpochBlock` (the epoch's start block — exactly the metagraph both challenges pin their leaf sets against, so D24 participant matching holds by construction). The attempt 409s until both challenges have emitted for that epoch; that is the expected steady state. The gateway prefers chain-scale bundles over the reserved smoke range (`>= 8_000_000`), so once a real seal lands it outranks every interim burn bundle — retire the burn timer (`systemctl disable --now base-burn-seal.timer`) after the first real seal verifies end-to-end. Install:
+**Real-epoch sealer (post burn-seal retirement):** `base-real-seal.timer` (every 10 min) drives [`scripts/prod-real-seal.sh`](scripts/prod-real-seal.sh), which walks **current … current−N** chain epochs (`REAL_SEAL_WALK_BACK`, default 16) with `block_b = LastEpochBlock − k×tempo` so a skipped design/prism leaf epoch does not pin `/v1/weights/latest` on a stale real seal forever (burn seals cannot outrank chain-scale bundles). 409 `incomplete_participant_set` on a candidate is expected and the script continues walking. The gateway prefers chain-scale bundles over the reserved smoke range (`>= 8_000_000`) — retire the burn timer (`systemctl disable --now base-burn-seal.timer`) after the first real seal verifies end-to-end. Install:
 
 ```bash
 install -m 0755 deploy/scripts/prod-real-seal.sh /opt/base/deploy/scripts/prod-real-seal.sh