diff --git a/crates/prism-challenge/src/api.rs b/crates/prism-challenge/src/api.rs index 07f5d6fa8..b7fd2b940 100644 --- a/crates/prism-challenge/src/api.rs +++ b/crates/prism-challenge/src/api.rs @@ -413,7 +413,10 @@ async fn post_retry( return json_err( StatusCode::CONFLICT, "not_failed", - &format!("status={}", row.status.as_str()), + &format!( + "status={} — /retry only accepts failed rows; for miner infra retry send X-Lium-Api-Key (and the usual X-Miner-Hotkey / body hotkey). Admin Bearer is for operator retries of non-infra failures", + row.status.as_str() + ), ); } let gate_key = prism_pipeline::gating_key(row.arch_id.as_deref()); diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index b47f77f6e..cbd3f8af8 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -669,6 +669,10 @@ impl Orchestrator { row: &SubmissionState, ) -> Result<(prism_lium::RemoteExecResult, prism_lium::EvalReceipt), String> { self.to_stage(id, Stage::Provisioning).await?; + // Extend BYOK seal before any Lium call so a long train cannot race TTL. + if let Some(p) = &self.payer { + let _ = p.vault.refresh(id); + } let backend = self.backend_for(id)?; let resume = mid_pod_resume(row); let (pod_id, provider) = if let Some(pid) = row.pod_id.clone() { @@ -708,11 +712,13 @@ impl Orchestrator { None, ) .await; + let payer_vault = self.payer.as_ref().map(|p| Arc::clone(&p.vault)); let stop_tx = spawn_log_watch( Arc::clone(&self.logs), Arc::clone(&self.store), Arc::clone(&backend), Arc::clone(&self.active), + payer_vault, id.to_owned(), pod_id.clone(), DEFAULT_LOG_POLL_SECS, diff --git a/crates/prism-lium-payer/src/lib.rs b/crates/prism-lium-payer/src/lib.rs index c5041b60f..f028bba45 100644 --- a/crates/prism-lium-payer/src/lib.rs +++ b/crates/prism-lium-payer/src/lib.rs @@ -1,9 +1,11 @@ //! Miner-funded Lium keys (BYOK): vault + live-client factory. //! -//! Keys are held in process memory and optionally in a short-TTL -//! encrypted seal directory (`PRISM_PAYER_VAULT_DIR` + key file). They are -//! never written to the submission store and never logged. Master still SSHs -//! with the operator keypair; the miner key pays for rent/terminate only. +//! Keys are held in process memory and optionally in a TTL-bounded +//! encrypted seal directory (`PRISM_PAYER_VAULT_DIR` + key file). Default TTL +//! is ≥36h (train wall + eval + skew) and heartbeats re-seal so long GPU runs +//! survive control-plane restarts. Keys are never written to the submission +//! store and never logged. Master still SSHs with the operator keypair; the +//! miner key pays for rent/terminate only. #![forbid(unsafe_code)] @@ -15,7 +17,10 @@ use std::sync::{Arc, Mutex}; use prism_lium::{EvalJobBackend, LiumClient, LiumError, LiumSshConfig, LIUM_API_BASE_URL}; -pub use sealed::{SealedVaultConfig, DEFAULT_TTL_SECS, DIR_ENV, KEY_ENV}; +pub use sealed::{ + expiry_secs, recommended_ttl_secs, SealedVaultConfig, DEFAULT_TTL_SECS, DIR_ENV, + EVAL_BUDGET_SECS, KEY_ENV, SEAL_SKEW_SECS, TRAIN_WALL_SECS, TTL_ENV, +}; /// In-memory map `submission_id → Lium API key`, with optional sealed persist. pub struct PayerKeyVault { @@ -103,6 +108,9 @@ impl PayerKeyVault { } /// Clone of the stored key, if any (memory, then sealed file). + /// + /// Measure / resume paths rely on this hydrate-from-seal fallback when the + /// in-memory map is empty after a restart (as long as the seal is unexpired). #[must_use] pub fn get(&self, submission_id: &str) -> Option { if let Ok(g) = self.inner.lock() { @@ -118,6 +126,26 @@ impl PayerKeyVault { Some(key) } + /// Re-persist the seal with a fresh TTL window (memory and/or disk). + /// + /// Call before measure and on heartbeats so full-budget trains cannot + /// outlive the on-disk seal. Returns `false` when no key is available. + pub fn refresh(&self, submission_id: &str) -> bool { + let Some(key) = self.get(submission_id) else { + return false; + }; + if let Some(cfg) = &self.sealed { + if let Err(e) = sealed::persist(cfg, submission_id, &key) { + tracing::warn!(error = %e, "payer seal refresh failed"); + return false; + } + } + if let Ok(mut g) = self.inner.lock() { + g.insert(submission_id.to_owned(), key); + } + true + } + /// Drop the key after the eval finishes (memory + seal). pub fn remove(&self, submission_id: &str) { if let Ok(mut g) = self.inner.lock() { @@ -226,6 +254,7 @@ pub fn require_miner_lium(backend_mode: &str) -> bool { #[cfg(test)] mod tests { use super::*; + use tempfile::tempdir; #[test] fn vault_roundtrip_and_redacted_debug() { @@ -244,4 +273,57 @@ mod tests { assert!(!require_miner_lium("sim")); assert!(!require_miner_lium("sim/openrouter")); } + + #[test] + fn default_ttl_covers_train_wall() { + const { + assert!(DEFAULT_TTL_SECS >= TRAIN_WALL_SECS + EVAL_BUDGET_SECS + SEAL_SKEW_SECS); + } + assert!(recommended_ttl_secs() >= TRAIN_WALL_SECS); + assert!(recommended_ttl_secs() >= 36 * 3600); + } + + #[test] + fn refresh_extends_seal_expiry() { + let dir = tempdir().unwrap(); + let cfg = SealedVaultConfig { + dir: dir.path().to_path_buf(), + key: [3u8; 32], + ttl_secs: 120, + }; + let v = PayerKeyVault::new().with_sealed(cfg.clone()); + v.insert("job1", "sk_miner"); + let first = sealed::expiry_secs(&cfg, "job1").expect("sealed"); + // Advance perceived lifetime by re-sealing after a short sleep so the + // unix expiry second is guaranteed to move forward under CI load. + std::thread::sleep(std::time::Duration::from_secs(1)); + assert!(v.refresh("job1")); + let second = sealed::expiry_secs(&cfg, "job1").expect("refreshed"); + assert!( + second > first, + "refresh must extend expiry (first={first} second={second})" + ); + assert_eq!(v.get("job1").as_deref(), Some("sk_miner")); + } + + #[test] + fn measure_hydrates_from_seal_without_ram() { + let dir = tempdir().unwrap(); + let cfg = SealedVaultConfig { + dir: dir.path().to_path_buf(), + key: [5u8; 32], + ttl_secs: 3600, + }; + sealed::persist(&cfg, "onlyseal", "sk_from_disk").unwrap(); + // Fresh vault: empty RAM, sealed backend only — same as post-restart + // before hydrate_all, or when get() mid-run must reload from disk. + let v = PayerKeyVault::new().with_sealed(cfg); + assert_eq!(v.get("onlyseal").as_deref(), Some("sk_from_disk")); + assert!(v.refresh("onlyseal")); + // Clear RAM and prove seal-only hydrate still works. + if let Ok(mut g) = v.inner.lock() { + g.clear(); + } + assert_eq!(v.get("onlyseal").as_deref(), Some("sk_from_disk")); + } } diff --git a/crates/prism-lium-payer/src/sealed.rs b/crates/prism-lium-payer/src/sealed.rs index 914da7d70..902b9c30c 100644 --- a/crates/prism-lium-payer/src/sealed.rs +++ b/crates/prism-lium-payer/src/sealed.rs @@ -1,7 +1,11 @@ -//! Short-TTL encrypted-at-rest BYOK seals (not the submission DB). +//! Encrypted-at-rest BYOK seals (not the submission DB). //! //! ChaCha20-Poly1305 with a process key file. Plaintext never lands in Postgres. //! Files are mode-0600 under `PRISM_PAYER_VAULT_DIR`. +//! +//! TTL must outlast a full train wall + eval + control-plane skew. Heartbeats +//! re-seal so mid-flight restarts never hydrate an expired file after a long +//! GPU run. use std::fs; use std::io::Write; @@ -13,12 +17,41 @@ use chacha20poly1305::{ChaCha20Poly1305, Nonce}; use rand::RngCore; use sha2::{Digest, Sha256}; -/// Default TTL for sealed payer keys (covers max healthy train + margin). -pub const DEFAULT_TTL_SECS: u64 = 12 * 3600; +/// Recipe train wall (6h = 21600s). Overridden by `PRISM_TRAIN_HOURS_CAP` when set. +pub const TRAIN_WALL_SECS: u64 = 6 * 3600; +/// Post-train eval / harvest / terminate budget. +pub const EVAL_BUDGET_SECS: u64 = 2 * 3600; +/// Queue, pre-pod screens, restart skew, and clock margin. +pub const SEAL_SKEW_SECS: u64 = 4 * 3600; +/// Default TTL floor (≥36h): covers full-budget runs with substantial queue wait. +pub const DEFAULT_TTL_SECS: u64 = 36 * 3600; /// Env: directory for `*.seal` files. pub const DIR_ENV: &str = "PRISM_PAYER_VAULT_DIR"; /// Env: 32-byte key file (raw or 64-hex). pub const KEY_ENV: &str = "PRISM_PAYER_VAULT_KEY_FILE"; +/// Env: soft TTL seconds (floored by [`recommended_ttl_secs`] when unset). +pub const TTL_ENV: &str = "PRISM_PAYER_VAULT_TTL_SECS"; + +/// Soft TTL: `max(default_floor, train_wall + eval + skew)`. +/// +/// `PRISM_TRAIN_HOURS_CAP` (hours, float) raises the train component when set. +#[must_use] +pub fn recommended_ttl_secs() -> u64 { + let train = std::env::var("PRISM_TRAIN_HOURS_CAP") + .ok() + .and_then(|s| s.parse::().ok()) + .filter(|h| h.is_finite() && *h > 0.0) + .map_or(TRAIN_WALL_SECS, |h| { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + { + (h * 3600.0).ceil() as u64 + } + }); + let computed = train + .saturating_add(EVAL_BUDGET_SECS) + .saturating_add(SEAL_SKEW_SECS); + DEFAULT_TTL_SECS.max(computed) +} /// On-disk sealed vault settings. #[derive(Debug, Clone)] @@ -42,10 +75,11 @@ impl SealedVaultConfig { .ok() .filter(|s| !s.trim().is_empty())?; let key = load_key(Path::new(&key_path))?; - let ttl_secs = std::env::var("PRISM_PAYER_VAULT_TTL_SECS") + let floor = recommended_ttl_secs(); + let ttl_secs = std::env::var(TTL_ENV) .ok() .and_then(|s| s.parse().ok()) - .unwrap_or(DEFAULT_TTL_SECS); + .map_or(floor, |configured: u64| configured.max(floor)); let dir = PathBuf::from(dir); let _ = fs::create_dir_all(&dir); Some(Self { dir, key, ttl_secs }) @@ -119,6 +153,23 @@ pub fn persist(cfg: &SealedVaultConfig, submission_id: &str, api_key: &str) -> R Ok(()) } +/// Absolute unix expiry embedded in the seal, if present and decryptable. +#[must_use] +pub fn expiry_secs(cfg: &SealedVaultConfig, submission_id: &str) -> Option { + let path = seal_path(&cfg.dir, submission_id); + let bytes = fs::read(&path).ok()?; + if bytes.len() < 13 { + return None; + } + let (nonce_bytes, ct) = bytes.split_at(12); + let cipher = ChaCha20Poly1305::new_from_slice(&cfg.key).ok()?; + let nonce = Nonce::from_slice(nonce_bytes); + let pt = cipher.decrypt(nonce, ct).ok()?; + let text = String::from_utf8(pt).ok()?; + let (exp_s, _) = text.split_once('\n')?; + exp_s.parse().ok() +} + /// Decrypt seal when present and unexpired. pub fn load(cfg: &SealedVaultConfig, submission_id: &str) -> Option { let path = seal_path(&cfg.dir, submission_id); @@ -175,6 +226,20 @@ mod tests { use super::*; use tempfile::tempdir; + #[test] + fn recommended_ttl_covers_train_wall() { + let ttl = recommended_ttl_secs(); + assert!( + ttl >= TRAIN_WALL_SECS + EVAL_BUDGET_SECS + SEAL_SKEW_SECS, + "ttl {ttl} must cover train+eval+skew" + ); + assert!( + ttl >= DEFAULT_TTL_SECS, + "ttl {ttl} must be at least the 36h floor" + ); + assert!(ttl >= 36 * 3600); + } + #[test] fn seal_roundtrip_and_ttl_expiry() { let dir = tempdir().unwrap(); @@ -185,9 +250,32 @@ mod tests { }; persist(&cfg, "abc123", "sk_test_secret").unwrap(); assert_eq!(load(&cfg, "abc123").as_deref(), Some("sk_test_secret")); + let exp = expiry_secs(&cfg, "abc123").unwrap(); + assert!(exp > now_secs()); let all = hydrate_all(&cfg); assert_eq!(all.len(), 1); remove(&cfg, "abc123"); assert!(load(&cfg, "abc123").is_none()); } + + #[test] + fn refresh_extends_expiry() { + let dir = tempdir().unwrap(); + let mut cfg = SealedVaultConfig { + dir: dir.path().to_path_buf(), + key: [9u8; 32], + ttl_secs: 60, + }; + persist(&cfg, "sub1", "sk_live").unwrap(); + let first = expiry_secs(&cfg, "sub1").unwrap(); + // Simulate a later refresh with a longer TTL window. + cfg.ttl_secs = 3600; + persist(&cfg, "sub1", "sk_live").unwrap(); + let second = expiry_secs(&cfg, "sub1").unwrap(); + assert!( + second >= first + 3000, + "refresh must push expiry forward (first={first} second={second})" + ); + assert_eq!(load(&cfg, "sub1").as_deref(), Some("sk_live")); + } } diff --git a/crates/prism-orphan/src/lib.rs b/crates/prism-orphan/src/lib.rs index 4e53f7ec3..3edfae7ee 100644 --- a/crates/prism-orphan/src/lib.rs +++ b/crates/prism-orphan/src/lib.rs @@ -35,6 +35,7 @@ pub fn spawn_log_watch( store: std::sync::Arc, backend: std::sync::Arc, active: std::sync::Arc, + payer_vault: Option>, submission_id: String, pod_id: String, poll_secs: u64, @@ -46,6 +47,7 @@ pub fn spawn_log_watch( store, backend, active, + payer_vault, submission_id, pod_id, stop_rx, @@ -62,17 +64,27 @@ pub async fn watch_pod_logs( store: std::sync::Arc, backend: std::sync::Arc, active: std::sync::Arc, + payer_vault: Option>, submission_id: String, pod_id: String, mut cancel: tokio::sync::watch::Receiver, poll_secs: u64, ) { let period = std::time::Duration::from_secs(poll_secs.max(5)); + // Re-seal every ~5 minutes so a 6h+ train cannot outlive the on-disk TTL. + let refresh_every = (300 / period.as_secs().max(1)).max(1); + let mut ticks: u64 = 0; loop { if *cancel.borrow() { break; } active.touch(&submission_id); + ticks = ticks.saturating_add(1); + if ticks == 1 || ticks.is_multiple_of(refresh_every) { + if let Some(vault) = &payer_vault { + let _ = vault.refresh(&submission_id); + } + } match backend.harvest_logs(&pod_id).await { Ok(text) if !text.trim().is_empty() => { logs.replace_tail(&submission_id, &text); diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 65fe385de..e818567bf 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -72,8 +72,8 @@ chmod 0400 deploy/secrets/huggingface/token `POST|GET /v1/admin/artifacts/...`. Read via `PRISM_ADMIN_TOKENS_FILE` (`/run/base/prism/admin_tokens`). Empty/missing → those routes answer **503 `auth_unconfigured`** (fail-closed). Mode **0400**, uid **65532**. -- `prism/payer_vault_key` — 32-byte (or 64-hex) key for short-TTL encrypted - miner BYOK seals (`PRISM_PAYER_VAULT_KEY_FILE`). Host dir +- `prism/payer_vault_key` — 32-byte (or 64-hex) key for TTL-bounded encrypted + miner BYOK seals (`PRISM_PAYER_VAULT_KEY_FILE`; default TTL ≥36h). Host dir `/var/lib/prism/payer-vault` is mounted RW for `*.seal` files. **Never commit the key.** Generate once: diff --git a/docs/PRISM.md b/docs/PRISM.md index 579dd29c5..1d7031d78 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -77,16 +77,17 @@ lives in `prism_submission`. Live measure runs the harness **detached** on the pod (`setsid` + `harness.log` / `harness.pid`) so a control-plane restart does not SIGHUP GPU work. On boot (and every ~30s) orphan reconcile is **resume-first**: mid-flight `provisioning`/`running` rows whose Lium pod is -still alive and whose BYOK key can be restored from the short-TTL sealed vault -(`PRISM_PAYER_VAULT_DIR`) are requeued with `pod_id` kept — the orchestrator -reattaches (log/event poll → wait terminal → harvest → score) without -terminating the pod. Only unreattachable rows fail-closed -(`control_plane_restart` / `harness_detached`) with best-effort terminate. -Post-measure review stages still requeue. Residual gap: expired seal + no -operator fallback ⇒ cannot call Lium API ⇒ fail-orphan (miner must stop the -pod and resubmit). The stuck sweeper remains a **10h** backstop and skips live -workers. `GET /v1/submissions/{id}/logs?since=` exposes harvested harness tails -+ heartbeats while a pod is measuring. +still alive and whose BYOK key can be restored from the sealed vault +(`PRISM_PAYER_VAULT_DIR`, default TTL ≥**36h** / train+eval+skew; heartbeats +re-seal) are requeued with `pod_id` kept — the orchestrator reattaches +(log/event poll → wait terminal → harvest → score) without terminating the +pod. Only unreattachable rows fail-closed (`control_plane_restart` / +`harness_detached`) with best-effort terminate. Post-measure review stages +still requeue. Residual gap: expired seal + no operator fallback ⇒ cannot +call Lium API ⇒ fail-orphan (miner must stop the pod and resubmit). The stuck +sweeper remains a **10h** backstop and skips live workers. +`GET /v1/submissions/{id}/logs?since=` exposes harvested harness tails + +heartbeats while a pod is measuring. Evaluation (Lium / Sim, review, agentic, leaf emit) is **master-only**. Validators never run `prism-challenge` — they fetch sealed weights only. diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index e170c054e..2f2d89465 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -66,19 +66,21 @@ X-Lium-Api-Key: ``` The key is held in master memory for that submission and may also land in a -**short-TTL encrypted seal file** on the master host (never in Postgres, never -logged) so a control-plane restart can **resume** your pod (or stop it if -reattach is impossible). Missing key on live → `400 missing_lium_api_key`. -Cost guardrails (`max_price_per_hour`, lifetime) still apply so a bad key -cannot rent unbounded SKUs through the orchestrator. +**TTL-bounded encrypted seal file** on the master host (default ≥36h; never in +Postgres, never logged). Master **re-seals** on measure start and heartbeats +so a full 6h train wall cannot outlive the seal across a control-plane +restart. Missing key on live → `400 missing_lium_api_key`. Cost guardrails +(`max_price_per_hour`, lifetime) still apply so a bad key cannot rent +unbounded SKUs through the orchestrator. If the challenge process restarts mid-run while your Lium pod is still training/evaling, master **reattaches** quietly (same submission id; pod is not killed). You only see `control_plane_restart` / `harness_detached` when -the pod is already dead or the sealed key expired and master cannot talk to -Lium — then stop the pod yourself and resubmit with `X-Lium-Api-Key`. Poll -`GET /v1/submissions/{id}/events` and `GET /v1/submissions/{id}/logs?since=` -for live stage heartbeats and harness tails while the run is healthy. +the pod is already dead or the sealed key cannot be restored and master +cannot talk to Lium — then stop the pod yourself and resubmit with +`X-Lium-Api-Key`. Poll `GET /v1/submissions/{id}/events` and +`GET /v1/submissions/{id}/logs?since=` for live stage heartbeats and harness +tails while the run is healthy. ## Submit diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index 6ca016600..9d830ba63 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -12,7 +12,7 @@ | `409 schedule` "daily manual run quota exceeded" | Manual anti-spam cap (10/day) — round-loop auto-enqueue does **not** spend it | `GET /v1/quota/{hotkey}` → `manual.remaining`; wait until next UTC day | | Active harness but no runs this round | Rare race / restart before auto-enqueue; or eliminated cooldown | Wait for the round tick / ask ops `admin/rounds/current/requeue`; check `eliminated_until_round` | | `auto_retry` events, class `install` | Dep won't install (bad name/version, heavy source build) | Design: `GET /v1/runs/{id}/logs`; Prism: `GET /v1/submissions/{id}/logs?since=` | -| `control_plane_restart` / `harness_detached` | Restart could not reattach (dead pod or expired BYOK seal) | Stop the Lium pod if still billing; resubmit with `X-Lium-Api-Key`. Healthy pods are resumed automatically — do not kill them on a routine master redeploy. | +| `control_plane_restart` / `harness_detached` | Restart could not reattach (dead pod or unrecoverable BYOK seal) | Stop the Lium pod if still billing; resubmit with `X-Lium-Api-Key`. Healthy pods are resumed automatically — do not kill them on a routine master redeploy. | | Run `failed` / Score 0 | Missing pages, timeout, crash | `GET /v1/runs/{id}/events`; ensure three required HTML pages | | External call refused (`403`) | Target is internal-blocklisted (metadata IP, loopback, RFC1918/VPC, control plane) | Call public endpoints only; egress is otherwise open | | Pages look empty in viewer | Sanitize stripped content | Scripts/`on*` handlers are removed; use static HTML/CSS | diff --git a/docs/runbooks/prism-enable-lium-and-emission.md b/docs/runbooks/prism-enable-lium-and-emission.md index 86fdbd473..9a6db2bc3 100644 --- a/docs/runbooks/prism-enable-lium-and-emission.md +++ b/docs/runbooks/prism-enable-lium-and-emission.md @@ -10,9 +10,12 @@ semaphore). Dial down only if the Lium lease pool cannot absorb the load. 6. **Control-plane restart / redeploy (GPU-safe):** keep `PRISM_PAYER_VAULT_DIR` + `PRISM_PAYER_VAULT_KEY_FILE` on a durable volume. - Healthy mid-flight pods are resumed (not terminated). Do not manually kill - Lium pods after a routine `prism-challenge` bounce — only stop pods that - surface `control_plane_restart` / `harness_detached` (dead pod or expired + Seal TTL defaults to ≥**36h** (`PRISM_PAYER_VAULT_TTL_SECS`, floored by + train wall + eval + skew); measure + heartbeats re-seal so full-budget + runs survive a bounce. Healthy mid-flight pods are resumed (not + terminated). Do not manually kill Lium pods after a routine + `prism-challenge` bounce — only stop pods that surface + `control_plane_restart` / `harness_detached` (dead pod or unrecoverable seal). Prefer rolling the challenge image when no pods are in `provisioning`/`running`, or accept resume after boot.