fix(prism): keep BYOK seal on measure Err + hard-pin 1×5090

crates/prism-challenge/src/api.rs

@@ -410,13 +410,19 @@ async fn post_retry(
         Err(e) => return json_err(StatusCode::INTERNAL_SERVER_ERROR, "store", &e.to_string()),
     };
     if row.status != Stage::Failed {
+        let has_lium = headers
+            .get(prism_lium_payer::LIUM_API_KEY_HEADER)
+            .or_else(|| headers.get("X-Lium-Api-Key"))
+            .is_some();
+        let hint = if has_lium {
+            "status is not failed — /retry only recovers failed rows. Re-POSTing the same ZIP is an idempotent already-queued no-op; wait for the run or use events/logs. Infra recovery after failure needs X-Lium-Api-Key on /retry (hotkey/Bearer alone is not enough)."
+        } else {
+            "status is not failed — /retry only accepts failed rows. Identical ZIP re-POST returns already-queued (no new GPU). After a failed infra run, POST /retry with X-Lium-Api-Key (not only X-Miner-Hotkey / Bearer)."
+        };
         return json_err(
             StatusCode::CONFLICT,
             "not_failed",
-            &format!(
-                "status={} — /retry only accepts failed rows; for miner infra retry send X-Lium-Api-Key (and the usual X-Miner-Hotkey / body hotkey). Admin Bearer is for operator retries of non-infra failures",
-                row.status.as_str()
-            ),
+            &format!("status={} — {hint}", row.status.as_str()),
         );
     }
     let gate_key = prism_pipeline::gating_key(row.arch_id.as_deref());

crates/prism-challenge/src/orchestrator.rs

@@ -379,6 +379,20 @@ impl<C: ChainClient + Send> Orchestrator<C> {
             Ok((m, r)) => (Some(m), Some(r)),
             Err(e) => {
                 let msg = format!("measure: {e}");
+                // Harness EVAL_FAIL is miner/model code, not Lium infra — do not
+                // burn auto-retries (and never drop the BYOK seal on Err; see
+                // finish_measure).
+                if msg.contains("EVAL_FAIL") {
+                    fail_terminal(
+                        self.store.as_ref(),
+                        self.gating.as_ref(),
+                        &row,
+                        "install",
+                        &msg,
+                    )
+                    .await;
+                    return Ok(());
+                }
                 if self.maybe_auto_retry(&row, "install", &msg).await {
                     return Ok(());
                 }
@@ -669,9 +683,15 @@ impl<C: ChainClient + Send> Orchestrator<C> {
         row: &SubmissionState,
     ) -> Result<(prism_lium::RemoteExecResult, prism_lium::EvalReceipt), String> {
         self.to_stage(id, Stage::Provisioning).await?;
-        // Extend BYOK seal before any Lium call so a long train cannot race TTL.
+        // Re-seal immediately before measure so a full train+eval wall cannot
+        // race TTL; fail closed when BYOK is required and the vault is empty.
         if let Some(p) = &self.payer {
-            let _ = p.vault.refresh(id);
+            if !p.vault.refresh(id) && !p.allow_operator_fallback {
+                return Err(
+                    "miner Lium API key missing for this submission — resubmit with X-Lium-Api-Key"
+                        .into(),
+                );
+            }
         }
         let backend = self.backend_for(id)?;
         let resume = mid_pod_resume(row);

crates/prism-lium-types/src/types.rs

@@ -160,32 +160,35 @@ pub struct Instance {
     pub ssh_connect_cmd: Option<String>,
 }
 
-/// Ordered GPU preference list (capability filter, not hard SKU lock).
+/// Ordered GPU preference list — Prism live rents are a **hard SKU pin**.
 #[derive(Debug, Clone, Default)]
 pub struct GpuPreference {
     /// Substrings matched against `Offer.gpu_type` (case-insensitive), first wins.
     pub prefer: Vec<String>,
 }
 
 impl GpuPreference {
-    /// Default PRISM pin: **RTX 5090 first**, then an explicit ordered
-    /// fallback. The 5090 (Blackwell, 32 GB) is the price/performance sweet
-    /// spot for the ≤350M-param recipe; the fallback chain orders by
-    /// capability so provisioning never silently lands on whatever happens
-    /// to be cheapest when the pin is out of capacity.
+    /// Default PRISM pin: **1× RTX 5090 only** (fail-closed).
+    ///
+    /// Ranking fairness requires a single SKU: wall-capped trains on a slower
+    /// card (e.g. 4090) see fewer tokens → worse bpb. Non-5090 / multi-GPU
+    /// offers are rejected at rent time, not normalized after the fact.
     #[must_use]
     pub fn default_prism() -> Self {
         Self {
-            prefer: vec![
-                "RTX 5090".into(),
-                "B200".into(),
-                "H100".into(),
-                "A100".into(),
-                "RTX 4090".into(),
-            ],
+            prefer: vec!["RTX 5090".into()],
         }
     }
 
+    /// True when `gpu_type` matches any pin needle (case-insensitive substring).
+    #[must_use]
+    pub fn matches_pin(&self, gpu_type: &str) -> bool {
+        let upper = gpu_type.to_ascii_uppercase();
+        self.prefer
+            .iter()
+            .any(|needle| upper.contains(&needle.to_ascii_uppercase()))
+    }
+
     /// Rank offers: lower is better. Unmatched get large rank.
     #[must_use]
     pub fn rank(&self, gpu_type: &str) -> usize {
@@ -344,14 +347,14 @@ mod tests {
     use super::*;
 
     #[test]
-    fn default_prism_pins_5090_first_with_ordered_fallback() {
+    fn default_prism_hard_pins_5090_only() {
         let p = GpuPreference::default_prism();
-        assert_eq!(p.prefer.first().map(String::as_str), Some("RTX 5090"));
-        assert!(p.rank("NVIDIA GeForce RTX 5090") < p.rank("NVIDIA B200"));
-        assert!(p.rank("NVIDIA B200") < p.rank("NVIDIA H100-SXM5-80GB"));
-        assert!(p.rank("NVIDIA H100") < p.rank("NVIDIA A100-SXM4-80GB"));
-        assert!(p.rank("NVIDIA A100") < p.rank("NVIDIA GeForce RTX 4090"));
-        assert!(p.rank("NVIDIA GeForce RTX 4090") < p.rank("NVIDIA L4"));
+        assert_eq!(p.prefer.as_slice(), ["RTX 5090"]);
+        assert!(p.matches_pin("NVIDIA GeForce RTX 5090"));
+        assert!(!p.matches_pin("NVIDIA GeForce RTX 4090"));
+        assert!(!p.matches_pin("NVIDIA H100"));
+        assert!(!p.matches_pin("NVIDIA A100-SXM4-80GB"));
+        assert!(p.rank("NVIDIA GeForce RTX 5090") < p.rank("NVIDIA GeForce RTX 4090"));
     }
 
     #[test]

crates/prism-lium/src/client.rs

@@ -409,6 +409,17 @@ impl LiumClient {
         let target = self.resolve_ssh_target(instance_id).await?;
         let key = resolve_private_key(self.ssh.private_key_path.as_deref())?;
         let gpu_type = self.gpu_smoke(&target, &key).await?;
+        if !GpuPreference::default_prism().matches_pin(&gpu_type) {
+            warn!(
+                instance_id,
+                gpu = %gpu_type,
+                "nvidia-smi reported non-pin GPU — terminate for requeue"
+            );
+            let _ = self.terminate(instance_id).await;
+            return Err(LiumError::Exec(format!(
+                "non-pin GPU ({gpu_type}); Prism requires 1× RTX 5090 — resubmit/retry"
+            )));
+        }
         self.ensure_python_deps(&target, &key).await?;
 
         let train_cap_secs = (self.ssh.train_hours_cap * 3600.0) as u64;
@@ -443,6 +454,7 @@ impl LiumClient {
             &key,
             assets.as_deref(),
             train_cap_secs.saturating_add(3900),
+            Some(gpu_type.as_str()),
         )
         .await
     }
@@ -452,6 +464,18 @@ impl LiumClient {
         let _running = self.wait_until_running(instance_id).await?;
         let target = self.resolve_ssh_target(instance_id).await?;
         let key = resolve_private_key(self.ssh.private_key_path.as_deref())?;
+        let gpu_type = self.gpu_smoke(&target, &key).await?;
+        if !GpuPreference::default_prism().matches_pin(&gpu_type) {
+            warn!(
+                instance_id,
+                gpu = %gpu_type,
+                "resume saw non-pin GPU — terminate for requeue"
+            );
+            let _ = self.terminate(instance_id).await;
+            return Err(LiumError::Exec(format!(
+                "non-pin GPU ({gpu_type}); Prism requires 1× RTX 5090 — resubmit/retry"
+            )));
+        }
         let (att, rty) = (self.ssh.ssh_attempts, self.ssh.ssh_retry_secs);
         let probe_out =
             ssh_exec_allow_fail(&target, &key, HARNESS_PROBE_CMD, att.max(1), rty, 60).await?;
@@ -469,6 +493,7 @@ impl LiumClient {
             &key,
             assets.as_deref(),
             train_cap_secs.saturating_add(3900),
+            Some(gpu_type.as_str()),
         )
         .await
     }
@@ -481,6 +506,7 @@ impl LiumClient {
         key: &Path,
         assets: Option<&Path>,
         timeout_secs: u64,
+        fill_gpu: Option<&str>,
     ) -> Result<RemoteExecResult, LiumError> {
         let start = Instant::now();
         let period = Duration::from_secs(20);
@@ -524,7 +550,17 @@ impl LiumClient {
                             res.eval_tier
                         )));
                     }
-                    return Ok(*res);
+                    let mut res = *res;
+                    if res
+                        .gpu_type
+                        .as_deref()
+                        .is_none_or(|g| g.is_empty() || g.eq_ignore_ascii_case("null"))
+                    {
+                        if let Some(g) = fill_gpu {
+                            res.gpu_type = Some(g.to_owned());
+                        }
+                    }
+                    return Ok(res);
                 }
                 HarnessProgress::NeedsAssets | HarnessProgress::Running
                     if assets.is_some()
@@ -796,6 +832,9 @@ impl EvalJobBackend for LiumClient {
         // fallback to `selected.gpu_count` could silently rent 8×5090.
         offers.retain(|o| o.matches_gpu_count(spec.gpu_count));
         let pref = GpuPreference::default_prism();
+        // Fail-closed SKU pin: never fall through to 4090/A100/H100 when 5090
+        // rent fails — that is ranking unfair under a wall-clock train cap.
+        offers.retain(|o| pref.matches_pin(&o.gpu_type));
         offers.sort_by(|a, b| {
             pref.rank(&a.gpu_type)
                 .cmp(&pref.rank(&b.gpu_type))
@@ -883,7 +922,20 @@ impl EvalJobBackend for LiumClient {
                         continue;
                     };
                     match self.wait_until_running(&id).await {
-                        Ok(inst) => return Ok(inst),
+                        Ok(inst) => {
+                            let labeled = inst.gpu_type.as_deref().unwrap_or("");
+                            if !labeled.is_empty() && !pref.matches_pin(labeled) {
+                                warn!(
+                                    pod_id = %id,
+                                    gpu = %labeled,
+                                    "lium rented non-pin GPU — terminate and try next 5090 offer"
+                                );
+                                let _ = self.terminate(&id).await;
+                                last_err = format!("non-pin GPU after rent: {labeled}");
+                                continue;
+                            }
+                            return Ok(inst);
+                        }
                         Err(e) => {
                             last_err = format!("offer {} wait_running: {e}", selected.id);
                             self.cleanup_after_rent(&id).await;
@@ -1206,23 +1258,28 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn provision_falls_back_in_declared_order_when_no_5090() {
+    async fn provision_fail_closed_when_no_5090() {
         let server = MockServer::start().await;
         mount_common(
             &server,
             serde_json::json!([
                 {"id": "cheap-a100", "gpu_type": "NVIDIA A100-SXM4-80GB", "gpu_count": 1, "price_per_hour": 0.5},
                 {"id": "mid-h100", "gpu_type": "NVIDIA H100", "gpu_count": 1, "price_per_hour": 1.5},
-                {"id": "weak-l4", "gpu_type": "NVIDIA L4", "gpu_count": 1, "price_per_hour": 0.2}
+                {"id": "weak-4090", "gpu_type": "NVIDIA GeForce RTX 4090", "gpu_count": 1, "price_per_hour": 0.27}
             ]),
         )
         .await;
         mount_rent_path(&server, "cheap-a100", "pod-a100").await;
         mount_rent_path(&server, "mid-h100", "pod-h100").await;
-        mount_rent_path(&server, "weak-l4", "pod-l4").await;
+        mount_rent_path(&server, "weak-4090", "pod-4090").await;
         let c = LiumClient::with_base_url("test-key", server.uri()).unwrap();
-        let inst = c.provision(&provision_spec()).await.unwrap();
-        assert_eq!(inst.id, "pod-h100");
+        let err = c.provision(&provision_spec()).await.unwrap_err();
+        assert!(
+            matches!(err, LiumError::Cost(CostGuardrailError::NoCapacity))
+                || err.to_string().contains("NoCapacity")
+                || err.to_string().to_ascii_lowercase().contains("capacity"),
+            "got {err}"
+        );
     }
 
     #[tokio::test]

crates/prism-orphan/src/terminal.rs

@@ -131,8 +131,12 @@ pub async fn finish_measure(
         tokio::time::sleep(Duration::from_secs(5)).await;
         termination_verified = backend.verify_terminated(pod_id).await.unwrap_or(false);
     }
-    if let Some(p) = payer {
-        p.vault.remove(id);
+    // Keep BYOK seal on measure Err so auto-/miner-retry can re-rent.
+    // Drop only after a successful metrics harvest (pod already terminated).
+    if metrics.is_ok() {
+        if let Some(p) = payer {
+            p.vault.remove(id);
+        }
     }
     let receipt = EvalReceipt {
         provider,

docs/PRISM.md

@@ -79,7 +79,9 @@ not SIGHUP GPU work. On boot (and every ~30s) orphan reconcile is
 **resume-first**: mid-flight `provisioning`/`running` rows whose Lium pod is
 still alive and whose BYOK key can be restored from the sealed vault
 (`PRISM_PAYER_VAULT_DIR`, default TTL ≥**36h** / train+eval+skew; heartbeats
-re-seal) are requeued with `pod_id` kept — the orchestrator reattaches
+re-seal; measure start refreshes the seal and **measure Err keeps the vault
+entry** so auto-/miner-retry can re-rent) are requeued with `pod_id` kept — the
+orchestrator reattaches
 (log/event poll → wait terminal → harvest → score) without terminating the
 pod. Only unreattachable rows fail-closed (`control_plane_restart` /
 `harness_detached`) with best-effort terminate. Post-measure review stages
@@ -551,7 +553,7 @@ the harness semantics listed above, and the baseline sources they may reuse.
 
 | Dimension | Real | Fallback |
 |-----------|------|----------|
-| Eval backend | Live Lium when not `PRISM_FORCE_SIM` — miners bill via `X-Lium-Api-Key` (operator `LIUM_API_KEY` optional fallback if `PRISM_ALLOW_OPERATOR_LIUM=1`) | `SimLiumBackend` |
+| Eval backend | Live Lium when not `PRISM_FORCE_SIM` — miners bill via `X-Lium-Api-Key` (operator `LIUM_API_KEY` optional fallback if `PRISM_ALLOW_OPERATOR_LIUM=1`). **Hard pin: 1× RTX 5090** (non-5090 / multi-GPU rejected at rent; no silent fallback) | `SimLiumBackend` |
 | Reviewer | `/run/base/openrouter/api_key` exists → OpenRouter LLM | `SimReviewer` (deterministic) |
 | Agentic | same OpenRouter key → `OpenRouterAgent` | `SimAgent` (AST + metrics heuristics) |
 | Store | `BASE_DATABASE_URL` set → Postgres w/ migrations | in-memory (dev only) |

docs/external-miner/prism.md

@@ -134,10 +134,24 @@ versioned descriptor). Trust `/v1/recipe`, not marketing chart labels.
 - If your hotkey **leaves the metagraph**, the watcher reopens your slot(s)
   automatically — resubmit under your new uid.
 - Infra failures (Lium pod, review/similarity/LLM infra) **auto-retry up to 3
-  times**; cheat / rejected verdicts are terminal. After an infra failure
-  (`ChallengeInternal`), you may **resubmit within 30 minutes** (new POST or
-  `POST /v1/submissions/{id}/retry`). After 30 minutes the slot stays blocked
-  until your hotkey leaves the metagraph.
+  times**; harness `EVAL_FAIL` (miner/model code) is terminal for that attempt
+  and is **not** auto-retried. Cheat / rejected verdicts are terminal. After an
+  infra failure (`ChallengeInternal`), you may **recover within 30 minutes**
+  via `POST /v1/submissions/{id}/retry` with **`X-Lium-Api-Key`** (required on
+  live when another GPU rent is needed). After 30 minutes the slot stays
+  blocked until your hotkey leaves the metagraph.
+
+### Retry vs re-POST
+
+| Action | When | Headers |
+|--------|------|---------|
+| Re-POST the **same** ZIP | Always safe | Same as submit | Returns `200 already-queued` — **no new GPU run**; does not recover a failed row |
+| `POST /v1/submissions/{id}/retry` | Row status is **`failed`** only | **`X-Lium-Api-Key`** on live (infra recovery); admin Bearer for operator non-infra retries | Requeues measure; wrong/missing Lium key → `400 missing_lium_api_key` |
+| `/retry` on non-failed | — | — | `409 not_failed` — hotkey or Bearer alone does not change that |
+
+Do **not** expect `X-Miner-Hotkey` or admin Bearer alone to fund a new Lium
+pod. Seal TTL is ≥36h and master re-seals on measure + heartbeats; the key is
+kept across measure Err so auto-/miner-retry can re-rent without a new submit.
 
 ## Anti-copy rule (patch / delta)
 

docs/external-miner/troubleshoot.md

@@ -35,7 +35,9 @@
 | `similar: true` on precheck | Would hit intake copy gate | Change the patch vs prior champions; starting from the operator pin is fine |
 | `429 precheck_quota_exceeded` | 3 prechecks/coldkey/UTC day used | Wait until next UTC day; rotating hotkeys does not reset |
 | `400 missing_lium_api_key` | Live path needs miner-funded Lium | Pass `X-Lium-Api-Key` (your Lium account); see [`prism.md`](prism.md) |
-| Stuck `Provisioning` | Lium market / underfunded key | Check your Lium balance; watch `GET /v1/jobs` / events |
+| `409 not_failed` on `/retry` | Row is not `failed` (queued/running/scored) | `/retry` is only for failed rows. Identical ZIP re-POST → `already-queued` (no-op). After infra failure use `/retry` + `X-Lium-Api-Key` |
+| `400 missing_lium_api_key` on `/retry` | Failed infra row needs another GPU rent | Send `X-Lium-Api-Key` (hotkey / Bearer alone is not enough) |
+| Stuck `Provisioning` | Lium market / underfunded key / no 1×5090 | Check Lium balance; Prism hard-pins **1× RTX 5090** (non-5090 rejected) |
 | Idempotent replay | Same `submission_id` (pin id + patch bytes) | Expected — returns prior row |
 
 ## Shared