Merge pull request #101 from BaseIntelligence/fix/master-no-validator-submitter

fix(deploy): keep on-chain validator off master hosts

Author: echobt

bins/design-challenge/src/main.rs

@@ -546,6 +546,7 @@ fn build_app_state(
             staging_root: cli.staging_root.clone(),
             stage_delay,
             auto_retry_max: cli.auto_retry_max,
+            emit_poll: Duration::from_secs(15),
         },
         Arc::clone(&store),
         sandbox,
@@ -673,6 +674,7 @@ async fn cmd_serve(cli: Cli) -> Result<(), String> {
     }
     tokio::spawn(Arc::clone(&orch).run_round_loop());
     tokio::spawn(Arc::clone(&orch).run_sweeper());
+    tokio::spawn(Arc::clone(&orch).run_emitter());
 
     let app = design_router(state);
     let listener = TcpListener::bind(cli.bind)

crates/design-challenge/src/orchestrator.rs

@@ -149,6 +149,8 @@ pub struct OrchestratorConfig {
     /// Auto-retry budget for infra-class failures (initial attempt + this
     /// many retries). Default 3; `cheat` / `rejected` are always terminal.
     pub auto_retry_max: u32,
+    /// Poll interval for the late-tempo D24 filler.
+    pub emit_poll: Duration,
 }
 
 impl Default for OrchestratorConfig {
@@ -161,6 +163,7 @@ impl Default for OrchestratorConfig {
             staging_root: PathBuf::from("/var/lib/design/staging"),
             stage_delay: Duration::ZERO,
             auto_retry_max: 3,
+            emit_poll: Duration::from_secs(15),
         }
     }
 }
@@ -177,6 +180,8 @@ pub struct Orchestrator<C: ChainClient + Send + Sync> {
     gating: Option<Arc<dyn GatingStore>>,
     /// Shared chain-epoch cache (HTTP `AppState.epoch` + sweeper clock).
     epoch_cache: Option<Arc<AtomicU64>>,
+    /// Last epoch successfully covered by [`Self::emit_leaves`].
+    emitted_epoch: AtomicU64,
 }
 
 impl<C: ChainClient + Send + Sync> std::fmt::Debug for Orchestrator<C> {
@@ -209,6 +214,7 @@ impl<C: ChainClient + Send + Sync + 'static> Orchestrator<C> {
             sk,
             gating: None,
             epoch_cache: None,
+            emitted_epoch: AtomicU64::new(0),
         }
     }
 
@@ -279,6 +285,42 @@ impl<C: ChainClient + Send + Sync + 'static> Orchestrator<C> {
         }
     }
 
+    /// Late-tempo D24 filler: `NotAttempted` coverage when no admin award fired
+    /// (waits ~last 48 blocks so `award_round` can land Score leaves first).
+    pub async fn run_emitter(self: Arc<Self>)
+    where
+        C: Sync,
+    {
+        loop {
+            if let Err(e) = self.emitter_tick().await {
+                warn!(error = %e, "design emitter tick error");
+            }
+            sleep(self.cfg.emit_poll).await;
+        }
+    }
+
+    /// One emitter tick. `Ok(true)` when a leaf set was submitted this tick.
+    ///
+    /// # Errors
+    /// Chain / sign / submit failures (retried next tick).
+    pub async fn emitter_tick(&self) -> Result<bool, String>
+    where
+        C: Sync,
+    {
+        let state = chain::gather_schedule_state(self.chain.as_ref(), self.cfg.netuid)
+            .map_err(|e| format!("schedule: {e}"))?;
+        let epoch = state.subnet_epoch_index;
+        if epoch == 0 || self.emitted_epoch.load(Ordering::Relaxed) >= epoch {
+            return Ok(false);
+        }
+        let tempo = u64::from(state.tempo.max(1));
+        if state.blocks_since_last_step.saturating_add(48) < tempo {
+            return Ok(false);
+        }
+        self.emit_leaves().await?;
+        Ok(true)
+    }
+
     async fn sweep_unscored_timeouts(&self) -> Result<(), String> {
         let current = self.current_epoch();
         if current == 0 {
@@ -1089,6 +1131,13 @@ impl<C: ChainClient + Send + Sync + 'static> Orchestrator<C> {
         submit_signed_leaf_set(self.gateway.as_ref(), &signed)
             .await
             .map_err(|e| e.to_string())?;
+        self.emitted_epoch.store(epoch, Ordering::Relaxed);
+        info!(
+            epoch,
+            participants = expected_set.len(),
+            last_epoch_block = state.last_epoch_block,
+            "design leaf set submitted"
+        );
         Ok(())
     }
 }

deploy/AGENTS.md

@@ -19,8 +19,8 @@ Terraform: [`terraform/`](terraform/). Firewall: SSH from operator IP; CI uses e
 
 | File | Purpose |
 |------|---------|
-| `compose/role-master.yml` | gateway profile, VPC publish |
-| `compose/role-validator.yml` | no gateway; external gateway endpoint |
+| `compose/role-master.yml` | gateway profile, VPC publish; **no validator** (avoids dual CRV4 submit) |
+| `compose/role-validator.yml` | no gateway; external gateway endpoint; sole on-chain submitter |
 | `compose/env-staging.yml` | testnet 541, faster coordination |
 | `compose/env-prod.yml` | mainnet, conservative intervals |
 | `compose/env-local.yml` | **local only** — ports/smoke knobs/tunnel env; always on top of `env-staging` |
@@ -120,7 +120,9 @@ Prod (`env-prod.yml` + `base-burn-seal.service`): onfinality `public-ws` primary
 
 Validator logs should show `Match epoch=` then `Match → submit_intent` / `submit_timelocked ok`. Keep legacy Python weight submit **stopped** to avoid double-commit.
 
-**Legacy Python agents (mainnet):** `validator-5gzi` (`95.133.252.120`) may point `master_url` / `weights_url` / `registry_url` at `https://chain.joinbase.ai` with **`submit_on_chain_enabled: false`**. Coordination shims live in `gateway-compat` (`/v1/validators/*`, `/v1/registry`, empty assignments). `GET /v1/weights/latest` refreshes `computed_at` / `expires_at` at serve time so Python pydantic clients accept sealed vectors older than 720s. Sole on-chain submitter for hotkey `5Gzi…` is the Rust validator on `192.81.218.11` — do **not** start `base-weight-submitter-5gzi` on `validator-root` unless CR ownership is moved off Rust.
+**Sole on-chain submitter (mainnet hotkey `5Gzi…`):** Rust `base-validator-1` on **`base-prod-validator` (`192.81.218.11`) only**. `role-master.yml` profiles the validator under `never`; `remote-deploy.sh --role master` force-removes any leftover container. Do **not** run a second validator (or Python weight submitter) with the same wallet — dual submitters fight `WeightsSetRateLimit` and can leave CRV4 commits stuck while incentive still shows a prior monopoly UID.
+
+**Legacy Python agents (mainnet):** `validator-5gzi` (`95.133.252.120`) may point `master_url` / `weights_url` / `registry_url` at `https://chain.joinbase.ai` with **`submit_on_chain_enabled: false`**. Coordination shims live in `gateway-compat` (`/v1/validators/*`, `/v1/registry`, empty assignments). `GET /v1/weights/latest` refreshes `computed_at` / `expires_at` at serve time so Python pydantic clients accept sealed vectors older than 720s. Do **not** start `base-weight-submitter-5gzi` on `validator-root` unless CR ownership is moved off Rust.
 
 **Challenge verification:** on **master** only (validator has **no challenge exec**). Simulate submissions end-to-end — submit **baseline** + submit **cheat**, poll `/v1/runs/{id}` + `/events` + `/logs`, probe edges (bad harness, sanitize, quota, routes), then **admin winners** (`GET/POST /v1/admin/rounds/{id}/…` with bearer from `deploy/secrets/design/annotator_tokens`) and confirm leaf → seal → `GET /v1/weights/latest` **`sealed: true`**. **Never host Sim in staging/prod** (`BASE_ALLOW_HOST_SIM` / host `SimSandbox` are CI/local only). Healthz alone is insufficient.
 

deploy/compose/env-local.yml

@@ -39,6 +39,9 @@ services:
       - ./.local/trust-root:/etc/base/config:ro
 
   validator:
+    # role-master disables on-chain submit on droplet masters; local-e2e needs
+    # a co-located validator for Match/CRV4 smoke against the local gateway.
+    profiles: !override []
     # Avoid role-master 18080 — often taken by staging SSH tunnels on laptops.
     ports: !override
       - "127.0.0.1:${LOCAL_VALIDATOR_HOST_PORT:-28080}:8080"

deploy/compose/role-master.yml

@@ -1,10 +1,12 @@
 # Master role — gateway enabled, published on VPC for validator peers.
 # Used with: docker compose -f docker-compose.yml -f deploy/compose/role-master.yml --profile master
+#
+# On-chain weight submit must NOT run here (sole submitter = validator host).
+# Co-located validator is local-e2e only — env-local.yml clears the profile.
 services:
   gateway:
     profiles: ["master"]
     ports:
       - "8080:8080"
   validator:
-    ports:
-      - "127.0.0.1:18080:8080"
+    profiles: ["never"]

deploy/scripts/assert-compose-matrix.sh

@@ -58,7 +58,7 @@ for banned in design-challenge design-egress-proxy prism-challenge socket-proxy;
 done
 echo "OK: validator role does not render gateway or challenge services"
 
-# --- master role: gateway + challenge services present ---
+# --- master role: gateway + challenge services present; no on-chain validator ---
 services=$(render \
   -f docker-compose.yml \
   -f deploy/compose/role-master.yml \
@@ -72,7 +72,10 @@ for required in design-challenge design-egress-proxy prism-challenge socket-prox
     fail "master role does not render $required (must)"
   fi
 done
-echo "OK: master role renders gateway and challenge services"
+if echo "$services" | grep -qx "validator"; then
+  fail "master role renders validator (dual submitter; must not — use validator host)"
+fi
+echo "OK: master role renders gateway and challenge services (no validator)"
 
 # --- evil-gateway not in default or master ---
 services=$(render \
@@ -206,6 +209,8 @@ local_services=$(render \
   config --services)
 echo "$local_services" | grep -qx "gateway" \
   || fail "env-local master stack does not render gateway"
+echo "$local_services" | grep -qx "validator" \
+  || fail "env-local master stack does not render co-located validator"
 echo "$local_services" | grep -qx "prism-challenge" \
   || fail "env-local master stack does not render prism-challenge"
 echo "$local_services" | grep -qx "design-challenge" \

deploy/scripts/remote-deploy.sh

@@ -416,23 +416,35 @@ fi
 docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} \$UP_PROFILE "\${UP_ARGS[@]}"
 # Profile-disabled services are not started, but an older compose project may
 # still be running them. On validator, force-remove master-only challenge
-# surfaces so smoke health does not see stale unhealthy containers.
+# surfaces so smoke health does not see stale unhealthy containers. On master,
+# force-remove the validator so a prior dual-submitter cannot fight the
+# validator-host wallet for WeightsSetRateLimit / CRV4 commits.
 if [[ '$ROLE' == 'validator' ]]; then
   docker compose ${COMPOSE_FILES[*]} rm -sf \
     prism-challenge design-challenge design-egress-proxy socket-proxy \
     >/dev/null 2>&1 || true
+elif [[ '$ROLE' == 'master' ]]; then
+  docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} rm -sf validator \
+    >/dev/null 2>&1 || true
 fi
 docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} \$UP_PROFILE ps
 # Local health probes via published tunnels if present, else container exec.
 sleep 5
-if curl -fsS -m 5 http://127.0.0.1:18080/healthz >/dev/null 2>&1; then
-  echo "validator tunnel health: \$(curl -fsS -m 5 http://127.0.0.1:18080/healthz)"
-elif docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} exec -T validator curl -fsS -m 5 http://127.0.0.1:8080/healthz >/dev/null 2>&1; then
-  echo "validator health: ok (in-container)"
-else
-  echo "validator health: probe deferred (container may still be starting)"
+if [[ '$ROLE' == 'validator' ]]; then
+  if curl -fsS -m 5 http://127.0.0.1:18080/healthz >/dev/null 2>&1; then
+    echo "validator tunnel health: \$(curl -fsS -m 5 http://127.0.0.1:18080/healthz)"
+  elif docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} exec -T validator curl -fsS -m 5 http://127.0.0.1:8080/healthz >/dev/null 2>&1; then
+    echo "validator health: ok (in-container)"
+  else
+    echo "validator health: probe deferred (container may still be starting)"
+  fi
 fi
 if [[ '$ROLE' == 'master' ]]; then
+  if docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} ps --status running --services 2>/dev/null | grep -qx validator; then
+    echo "remote-deploy: ERROR: validator still running on master (dual submitter)" >&2
+    exit 1
+  fi
+  echo "master: validator absent (sole on-chain submitter is validator host)"
   if docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} exec -T gateway curl -fsS -m 5 http://127.0.0.1:8080/healthz >/dev/null 2>&1; then
     echo "gateway health: ok"
   else

docs/DESIGN_CHALLENGE.md

@@ -479,7 +479,9 @@ Expected set `E` = all metagraph hotkeys for the pinned epoch (policy
 
 - Exactly one signed leaf per `h ∈ E`
 - **Refuses subset and superset** — Silence is a bug
-- Emit at round close and at each epoch boundary via `POST /v1/weights/raw`
+- Emit at round close (scored) and near each epoch boundary via `POST /v1/weights/raw`
+  (`Orchestrator::run_emitter` fills `NotAttempted` when no admin award fired, so
+  D24 seals keep advancing under 50/50 emission shares)
 
 Absence codes used on this path include `NotAttempted`, `Timeout`,
 `InvalidResponse`, `MinerError`, `RateLimited`, `ChallengeInternal` (bundle enum).