fix(prism): keep BYOK seal alive for full train wall

crates/prism-challenge/src/api.rs

@@ -413,7 +413,10 @@ async fn post_retry(
         return json_err(
             StatusCode::CONFLICT,
             "not_failed",
-            &format!("status={}", row.status.as_str()),
+            &format!(
+                "status={} — /retry only accepts failed rows; for miner infra retry send X-Lium-Api-Key (and the usual X-Miner-Hotkey / body hotkey). Admin Bearer is for operator retries of non-infra failures",
+                row.status.as_str()
+            ),
         );
     }
     let gate_key = prism_pipeline::gating_key(row.arch_id.as_deref());

crates/prism-challenge/src/orchestrator.rs

@@ -669,6 +669,10 @@ impl<C: ChainClient + Send> Orchestrator<C> {
         row: &SubmissionState,
     ) -> Result<(prism_lium::RemoteExecResult, prism_lium::EvalReceipt), String> {
         self.to_stage(id, Stage::Provisioning).await?;
+        // Extend BYOK seal before any Lium call so a long train cannot race TTL.
+        if let Some(p) = &self.payer {
+            let _ = p.vault.refresh(id);
+        }
         let backend = self.backend_for(id)?;
         let resume = mid_pod_resume(row);
         let (pod_id, provider) = if let Some(pid) = row.pod_id.clone() {
@@ -708,11 +712,13 @@ impl<C: ChainClient + Send> Orchestrator<C> {
                 None,
             )
             .await;
+        let payer_vault = self.payer.as_ref().map(|p| Arc::clone(&p.vault));
         let stop_tx = spawn_log_watch(
             Arc::clone(&self.logs),
             Arc::clone(&self.store),
             Arc::clone(&backend),
             Arc::clone(&self.active),
+            payer_vault,
             id.to_owned(),
             pod_id.clone(),
             DEFAULT_LOG_POLL_SECS,

crates/prism-lium-payer/src/lib.rs

@@ -1,9 +1,11 @@
 //! Miner-funded Lium keys (BYOK): vault + live-client factory.
 //!
-//! Keys are held in process memory and optionally in a short-TTL
-//! encrypted seal directory (`PRISM_PAYER_VAULT_DIR` + key file). They are
-//! never written to the submission store and never logged. Master still SSHs
-//! with the operator keypair; the miner key pays for rent/terminate only.
+//! Keys are held in process memory and optionally in a TTL-bounded
+//! encrypted seal directory (`PRISM_PAYER_VAULT_DIR` + key file). Default TTL
+//! is ≥36h (train wall + eval + skew) and heartbeats re-seal so long GPU runs
+//! survive control-plane restarts. Keys are never written to the submission
+//! store and never logged. Master still SSHs with the operator keypair; the
+//! miner key pays for rent/terminate only.
 
 #![forbid(unsafe_code)]
 
@@ -15,7 +17,10 @@ use std::sync::{Arc, Mutex};
 
 use prism_lium::{EvalJobBackend, LiumClient, LiumError, LiumSshConfig, LIUM_API_BASE_URL};
 
-pub use sealed::{SealedVaultConfig, DEFAULT_TTL_SECS, DIR_ENV, KEY_ENV};
+pub use sealed::{
+    expiry_secs, recommended_ttl_secs, SealedVaultConfig, DEFAULT_TTL_SECS, DIR_ENV,
+    EVAL_BUDGET_SECS, KEY_ENV, SEAL_SKEW_SECS, TRAIN_WALL_SECS, TTL_ENV,
+};
 
 /// In-memory map `submission_id → Lium API key`, with optional sealed persist.
 pub struct PayerKeyVault {
@@ -103,6 +108,9 @@ impl PayerKeyVault {
     }
 
     /// Clone of the stored key, if any (memory, then sealed file).
+    ///
+    /// Measure / resume paths rely on this hydrate-from-seal fallback when the
+    /// in-memory map is empty after a restart (as long as the seal is unexpired).
     #[must_use]
     pub fn get(&self, submission_id: &str) -> Option<String> {
         if let Ok(g) = self.inner.lock() {
@@ -118,6 +126,26 @@ impl PayerKeyVault {
         Some(key)
     }
 
+    /// Re-persist the seal with a fresh TTL window (memory and/or disk).
+    ///
+    /// Call before measure and on heartbeats so full-budget trains cannot
+    /// outlive the on-disk seal. Returns `false` when no key is available.
+    pub fn refresh(&self, submission_id: &str) -> bool {
+        let Some(key) = self.get(submission_id) else {
+            return false;
+        };
+        if let Some(cfg) = &self.sealed {
+            if let Err(e) = sealed::persist(cfg, submission_id, &key) {
+                tracing::warn!(error = %e, "payer seal refresh failed");
+                return false;
+            }
+        }
+        if let Ok(mut g) = self.inner.lock() {
+            g.insert(submission_id.to_owned(), key);
+        }
+        true
+    }
+
     /// Drop the key after the eval finishes (memory + seal).
     pub fn remove(&self, submission_id: &str) {
         if let Ok(mut g) = self.inner.lock() {
@@ -226,6 +254,7 @@ pub fn require_miner_lium(backend_mode: &str) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use tempfile::tempdir;
 
     #[test]
     fn vault_roundtrip_and_redacted_debug() {
@@ -244,4 +273,57 @@ mod tests {
         assert!(!require_miner_lium("sim"));
         assert!(!require_miner_lium("sim/openrouter"));
     }
+
+    #[test]
+    fn default_ttl_covers_train_wall() {
+        const {
+            assert!(DEFAULT_TTL_SECS >= TRAIN_WALL_SECS + EVAL_BUDGET_SECS + SEAL_SKEW_SECS);
+        }
+        assert!(recommended_ttl_secs() >= TRAIN_WALL_SECS);
+        assert!(recommended_ttl_secs() >= 36 * 3600);
+    }
+
+    #[test]
+    fn refresh_extends_seal_expiry() {
+        let dir = tempdir().unwrap();
+        let cfg = SealedVaultConfig {
+            dir: dir.path().to_path_buf(),
+            key: [3u8; 32],
+            ttl_secs: 120,
+        };
+        let v = PayerKeyVault::new().with_sealed(cfg.clone());
+        v.insert("job1", "sk_miner");
+        let first = sealed::expiry_secs(&cfg, "job1").expect("sealed");
+        // Advance perceived lifetime by re-sealing after a short sleep so the
+        // unix expiry second is guaranteed to move forward under CI load.
+        std::thread::sleep(std::time::Duration::from_secs(1));
+        assert!(v.refresh("job1"));
+        let second = sealed::expiry_secs(&cfg, "job1").expect("refreshed");
+        assert!(
+            second > first,
+            "refresh must extend expiry (first={first} second={second})"
+        );
+        assert_eq!(v.get("job1").as_deref(), Some("sk_miner"));
+    }
+
+    #[test]
+    fn measure_hydrates_from_seal_without_ram() {
+        let dir = tempdir().unwrap();
+        let cfg = SealedVaultConfig {
+            dir: dir.path().to_path_buf(),
+            key: [5u8; 32],
+            ttl_secs: 3600,
+        };
+        sealed::persist(&cfg, "onlyseal", "sk_from_disk").unwrap();
+        // Fresh vault: empty RAM, sealed backend only — same as post-restart
+        // before hydrate_all, or when get() mid-run must reload from disk.
+        let v = PayerKeyVault::new().with_sealed(cfg);
+        assert_eq!(v.get("onlyseal").as_deref(), Some("sk_from_disk"));
+        assert!(v.refresh("onlyseal"));
+        // Clear RAM and prove seal-only hydrate still works.
+        if let Ok(mut g) = v.inner.lock() {
+            g.clear();
+        }
+        assert_eq!(v.get("onlyseal").as_deref(), Some("sk_from_disk"));
+    }
 }

crates/prism-lium-payer/src/sealed.rs

@@ -1,7 +1,11 @@
-//! Short-TTL encrypted-at-rest BYOK seals (not the submission DB).
+//! Encrypted-at-rest BYOK seals (not the submission DB).
 //!
 //! ChaCha20-Poly1305 with a process key file. Plaintext never lands in Postgres.
 //! Files are mode-0600 under `PRISM_PAYER_VAULT_DIR`.
+//!
+//! TTL must outlast a full train wall + eval + control-plane skew. Heartbeats
+//! re-seal so mid-flight restarts never hydrate an expired file after a long
+//! GPU run.
 
 use std::fs;
 use std::io::Write;
@@ -13,12 +17,41 @@ use chacha20poly1305::{ChaCha20Poly1305, Nonce};
 use rand::RngCore;
 use sha2::{Digest, Sha256};
 
-/// Default TTL for sealed payer keys (covers max healthy train + margin).
-pub const DEFAULT_TTL_SECS: u64 = 12 * 3600;
+/// Recipe train wall (6h = 21600s). Overridden by `PRISM_TRAIN_HOURS_CAP` when set.
+pub const TRAIN_WALL_SECS: u64 = 6 * 3600;
+/// Post-train eval / harvest / terminate budget.
+pub const EVAL_BUDGET_SECS: u64 = 2 * 3600;
+/// Queue, pre-pod screens, restart skew, and clock margin.
+pub const SEAL_SKEW_SECS: u64 = 4 * 3600;
+/// Default TTL floor (≥36h): covers full-budget runs with substantial queue wait.
+pub const DEFAULT_TTL_SECS: u64 = 36 * 3600;
 /// Env: directory for `*.seal` files.
 pub const DIR_ENV: &str = "PRISM_PAYER_VAULT_DIR";
 /// Env: 32-byte key file (raw or 64-hex).
 pub const KEY_ENV: &str = "PRISM_PAYER_VAULT_KEY_FILE";
+/// Env: soft TTL seconds (floored by [`recommended_ttl_secs`] when unset).
+pub const TTL_ENV: &str = "PRISM_PAYER_VAULT_TTL_SECS";
+
+/// Soft TTL: `max(default_floor, train_wall + eval + skew)`.
+///
+/// `PRISM_TRAIN_HOURS_CAP` (hours, float) raises the train component when set.
+#[must_use]
+pub fn recommended_ttl_secs() -> u64 {
+    let train = std::env::var("PRISM_TRAIN_HOURS_CAP")
+        .ok()
+        .and_then(|s| s.parse::<f64>().ok())
+        .filter(|h| h.is_finite() && *h > 0.0)
+        .map_or(TRAIN_WALL_SECS, |h| {
+            #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
+            {
+                (h * 3600.0).ceil() as u64
+            }
+        });
+    let computed = train
+        .saturating_add(EVAL_BUDGET_SECS)
+        .saturating_add(SEAL_SKEW_SECS);
+    DEFAULT_TTL_SECS.max(computed)
+}
 
 /// On-disk sealed vault settings.
 #[derive(Debug, Clone)]
@@ -42,10 +75,11 @@ impl SealedVaultConfig {
             .ok()
             .filter(|s| !s.trim().is_empty())?;
         let key = load_key(Path::new(&key_path))?;
-        let ttl_secs = std::env::var("PRISM_PAYER_VAULT_TTL_SECS")
+        let floor = recommended_ttl_secs();
+        let ttl_secs = std::env::var(TTL_ENV)
             .ok()
             .and_then(|s| s.parse().ok())
-            .unwrap_or(DEFAULT_TTL_SECS);
+            .map_or(floor, |configured: u64| configured.max(floor));
         let dir = PathBuf::from(dir);
         let _ = fs::create_dir_all(&dir);
         Some(Self { dir, key, ttl_secs })
@@ -119,6 +153,23 @@ pub fn persist(cfg: &SealedVaultConfig, submission_id: &str, api_key: &str) -> R
     Ok(())
 }
 
+/// Absolute unix expiry embedded in the seal, if present and decryptable.
+#[must_use]
+pub fn expiry_secs(cfg: &SealedVaultConfig, submission_id: &str) -> Option<u64> {
+    let path = seal_path(&cfg.dir, submission_id);
+    let bytes = fs::read(&path).ok()?;
+    if bytes.len() < 13 {
+        return None;
+    }
+    let (nonce_bytes, ct) = bytes.split_at(12);
+    let cipher = ChaCha20Poly1305::new_from_slice(&cfg.key).ok()?;
+    let nonce = Nonce::from_slice(nonce_bytes);
+    let pt = cipher.decrypt(nonce, ct).ok()?;
+    let text = String::from_utf8(pt).ok()?;
+    let (exp_s, _) = text.split_once('\n')?;
+    exp_s.parse().ok()
+}
+
 /// Decrypt seal when present and unexpired.
 pub fn load(cfg: &SealedVaultConfig, submission_id: &str) -> Option<String> {
     let path = seal_path(&cfg.dir, submission_id);
@@ -175,6 +226,20 @@ mod tests {
     use super::*;
     use tempfile::tempdir;
 
+    #[test]
+    fn recommended_ttl_covers_train_wall() {
+        let ttl = recommended_ttl_secs();
+        assert!(
+            ttl >= TRAIN_WALL_SECS + EVAL_BUDGET_SECS + SEAL_SKEW_SECS,
+            "ttl {ttl} must cover train+eval+skew"
+        );
+        assert!(
+            ttl >= DEFAULT_TTL_SECS,
+            "ttl {ttl} must be at least the 36h floor"
+        );
+        assert!(ttl >= 36 * 3600);
+    }
+
     #[test]
     fn seal_roundtrip_and_ttl_expiry() {
         let dir = tempdir().unwrap();
@@ -185,9 +250,32 @@ mod tests {
         };
         persist(&cfg, "abc123", "sk_test_secret").unwrap();
         assert_eq!(load(&cfg, "abc123").as_deref(), Some("sk_test_secret"));
+        let exp = expiry_secs(&cfg, "abc123").unwrap();
+        assert!(exp > now_secs());
         let all = hydrate_all(&cfg);
         assert_eq!(all.len(), 1);
         remove(&cfg, "abc123");
         assert!(load(&cfg, "abc123").is_none());
     }
+
+    #[test]
+    fn refresh_extends_expiry() {
+        let dir = tempdir().unwrap();
+        let mut cfg = SealedVaultConfig {
+            dir: dir.path().to_path_buf(),
+            key: [9u8; 32],
+            ttl_secs: 60,
+        };
+        persist(&cfg, "sub1", "sk_live").unwrap();
+        let first = expiry_secs(&cfg, "sub1").unwrap();
+        // Simulate a later refresh with a longer TTL window.
+        cfg.ttl_secs = 3600;
+        persist(&cfg, "sub1", "sk_live").unwrap();
+        let second = expiry_secs(&cfg, "sub1").unwrap();
+        assert!(
+            second >= first + 3000,
+            "refresh must push expiry forward (first={first} second={second})"
+        );
+        assert_eq!(load(&cfg, "sub1").as_deref(), Some("sk_live"));
+    }
 }

crates/prism-orphan/src/lib.rs

@@ -35,6 +35,7 @@ pub fn spawn_log_watch(
     store: std::sync::Arc<dyn prism_store::PrismStore>,
     backend: std::sync::Arc<dyn prism_lium::EvalJobBackend>,
     active: std::sync::Arc<ActiveJobs>,
+    payer_vault: Option<std::sync::Arc<prism_lium_payer::PayerKeyVault>>,
     submission_id: String,
     pod_id: String,
     poll_secs: u64,
@@ -46,6 +47,7 @@ pub fn spawn_log_watch(
             store,
             backend,
             active,
+            payer_vault,
             submission_id,
             pod_id,
             stop_rx,
@@ -62,17 +64,27 @@ pub async fn watch_pod_logs(
     store: std::sync::Arc<dyn prism_store::PrismStore>,
     backend: std::sync::Arc<dyn prism_lium::EvalJobBackend>,
     active: std::sync::Arc<ActiveJobs>,
+    payer_vault: Option<std::sync::Arc<prism_lium_payer::PayerKeyVault>>,
     submission_id: String,
     pod_id: String,
     mut cancel: tokio::sync::watch::Receiver<bool>,
     poll_secs: u64,
 ) {
     let period = std::time::Duration::from_secs(poll_secs.max(5));
+    // Re-seal every ~5 minutes so a 6h+ train cannot outlive the on-disk TTL.
+    let refresh_every = (300 / period.as_secs().max(1)).max(1);
+    let mut ticks: u64 = 0;
     loop {
         if *cancel.borrow() {
             break;
         }
         active.touch(&submission_id);
+        ticks = ticks.saturating_add(1);
+        if ticks == 1 || ticks.is_multiple_of(refresh_every) {
+            if let Some(vault) = &payer_vault {
+                let _ = vault.refresh(&submission_id);
+            }
+        }
         match backend.harvest_logs(&pod_id).await {
             Ok(text) if !text.trim().is_empty() => {
                 logs.replace_tail(&submission_id, &text);

deploy/secrets/README.md

@@ -72,8 +72,8 @@ chmod 0400 deploy/secrets/huggingface/token
   `POST|GET /v1/admin/artifacts/...`. Read via `PRISM_ADMIN_TOKENS_FILE`
   (`/run/base/prism/admin_tokens`). Empty/missing → those routes answer
   **503 `auth_unconfigured`** (fail-closed). Mode **0400**, uid **65532**.
-- `prism/payer_vault_key` — 32-byte (or 64-hex) key for short-TTL encrypted
-  miner BYOK seals (`PRISM_PAYER_VAULT_KEY_FILE`). Host dir
+- `prism/payer_vault_key` — 32-byte (or 64-hex) key for TTL-bounded encrypted
+  miner BYOK seals (`PRISM_PAYER_VAULT_KEY_FILE`; default TTL ≥36h). Host dir
   `/var/lib/prism/payer-vault` is mounted RW for `*.seal` files. **Never
   commit the key.** Generate once:
 

docs/PRISM.md

@@ -77,16 +77,17 @@ lives in `prism_submission`. Live measure runs the harness **detached** on the
 pod (`setsid` + `harness.log` / `harness.pid`) so a control-plane restart does
 not SIGHUP GPU work. On boot (and every ~30s) orphan reconcile is
 **resume-first**: mid-flight `provisioning`/`running` rows whose Lium pod is
-still alive and whose BYOK key can be restored from the short-TTL sealed vault
-(`PRISM_PAYER_VAULT_DIR`) are requeued with `pod_id` kept — the orchestrator
-reattaches (log/event poll → wait terminal → harvest → score) without
-terminating the pod. Only unreattachable rows fail-closed
-(`control_plane_restart` / `harness_detached`) with best-effort terminate.
-Post-measure review stages still requeue. Residual gap: expired seal + no
-operator fallback ⇒ cannot call Lium API ⇒ fail-orphan (miner must stop the
-pod and resubmit). The stuck sweeper remains a **10h** backstop and skips live
-workers. `GET /v1/submissions/{id}/logs?since=` exposes harvested harness tails
-+ heartbeats while a pod is measuring.
+still alive and whose BYOK key can be restored from the sealed vault
+(`PRISM_PAYER_VAULT_DIR`, default TTL ≥**36h** / train+eval+skew; heartbeats
+re-seal) are requeued with `pod_id` kept — the orchestrator reattaches
+(log/event poll → wait terminal → harvest → score) without terminating the
+pod. Only unreattachable rows fail-closed (`control_plane_restart` /
+`harness_detached`) with best-effort terminate. Post-measure review stages
+still requeue. Residual gap: expired seal + no operator fallback ⇒ cannot
+call Lium API ⇒ fail-orphan (miner must stop the pod and resubmit). The stuck
+sweeper remains a **10h** backstop and skips live workers.
+`GET /v1/submissions/{id}/logs?since=` exposes harvested harness tails +
+heartbeats while a pod is measuring.
 
 Evaluation (Lium / Sim, review, agentic, leaf emit) is **master-only**.
 Validators never run `prism-challenge` — they fetch sealed weights only.