diff --git a/bins/prism-challenge/src/main.rs b/bins/prism-challenge/src/main.rs index a750d055a..05da7cbca 100644 --- a/bins/prism-challenge/src/main.rs +++ b/bins/prism-challenge/src/main.rs @@ -34,7 +34,8 @@ use prism_review::{OpenRouterClient, ReviewBackend, SimReviewer}; use submission_gating::{ watch_once, GatingStore, MemoryGatingStore, MetagraphCache, PgGatingStore, }; -const MAX_ATTEMPTS: u32 = 2; +/// Manual `/retry` ceiling; keep ≥ `PRISM_AUTO_RETRY_MAX` so infra retries work. +const MAX_ATTEMPTS: u32 = 3; use tokio::net::TcpListener; use tokio::sync::Semaphore; diff --git a/crates/prism-challenge/src/api.rs b/crates/prism-challenge/src/api.rs index 2c9f4d463..6bb5f3080 100644 --- a/crates/prism-challenge/src/api.rs +++ b/crates/prism-challenge/src/api.rs @@ -1,21 +1,6 @@ -//! Public PRISM HTTP API (miners + operators). -//! -//! | Route | Purpose | -//! |-------|---------| -//! | `GET /health` | liveness | -//! | `POST /v1/submissions` | accept a two-script recipe | -//! | `POST /v1/submissions/precheck` | advisory copy-gate (quota 3/coldkey/UTC day) | -//! | `GET /v1/submissions` | list (`status` / `miner` filter, limit) | -//! | `GET /v1/submissions/{id}` | full detail + event timeline | -//! | `GET /v1/submissions/{id}/events` | journal only | -//! | `GET /v1/status` | queue sizes + backend + recipe pin | -//! | `GET /v1/jobs` | orchestrator jobs view (active/last per pod) | -//! | `GET /v1/recipe` | recipe descriptor (full data contract) | -//! | `GET /v1/recipe/baseline` | baseline sources pairs | -//! -//! The API never blocks on the chain: acceptance timestamps the chain epoch -//! read at boot loop; if that read fails the epoch stays at the last known -//! value (still `>= 0`), never guessed. +//! Public PRISM HTTP API (miners + operators). Routes: health, submissions +//! (+ precheck / retry / events), status, jobs, recipe, architectures. +//! Epoch at accept is the last chain read (never guessed). use std::sync::Arc; @@ -26,7 +11,7 @@ use axum::routing::{get, post}; use axum::{Json, Router}; use serde::Deserialize; use serde_json::{json, Value}; -use submission_gating::{GatingState, GatingStore, MetagraphCache}; +use submission_gating::{infra_resubmit_allowed, GatingState, GatingStore, MetagraphCache}; use prism_recipe::{BASELINE_ARCHITECTURE_PY, BASELINE_TRAINING_PY}; @@ -41,24 +26,15 @@ use prism_store::{FinalScore, PrismStore, Stage, StoreError, SubmissionState}; /// Shared HTTP app state. #[derive(Debug)] pub struct AppState { - /// Store. pub store: Arc, - /// Current chain epoch cache (advanced by the worker loop). pub epoch: std::sync::atomic::AtomicU64, - /// Netuid. pub netuid: u16, - /// Eval backend label (`lium` / `sim`) for the status view. pub backend_mode: &'static str, - /// Max orchestrator attempts per submission (retry guard). pub retry_max: u32, - /// Submission gating (1-max). `None` disables intake gating (tests/dev). pub gating: Option>, - /// Cached metagraph snapshot for intake membership. `None` disables the - /// membership check (tests/dev). pub metagraph: Option>, } -/// Router over the full API surface. pub fn submission_router(state: Arc) -> Router { Router::new() .route("/health", get(health)) @@ -120,8 +96,7 @@ fn parse_submission_body( serde_json::from_slice(body).map_err(|e| format!("invalid_json: {e}")) } -/// Metagraph membership only (fail closed when configured but empty). -#[allow(clippy::result_large_err)] // mirrors other intake helpers returning `Response` +#[allow(clippy::result_large_err)] fn metagraph_uid(st: &AppState, hotkey: &str) -> Result, Response> { let Some(cache) = &st.metagraph else { return Ok(None); @@ -143,8 +118,6 @@ fn metagraph_uid(st: &AppState, hotkey: &str) -> Result, Response> { } } -/// Intake gates: metagraph membership + one accepted submission per -/// `(challenge, hotkey)`. Returns the metagraph uid on pass. async fn intake_gates( st: &AppState, hotkey: &str, @@ -155,8 +128,6 @@ async fn intake_gates( Ok(uid) } -/// The 1-max gating check alone (metagraph not consulted when no cache is -/// configured, e.g. unit tests). async fn gate_one_max( st: &AppState, hotkey: &str, @@ -164,7 +135,9 @@ async fn gate_one_max( ) -> Result, Response> { if let Some(g) = &st.gating { match g.get(challenge, hotkey).await { - Ok(Some(row)) if row.state != GatingState::Open => { + Ok(Some(row)) + if row.state != GatingState::Open && !infra_resubmit_allowed(&row, now_ms()) => + { return Err(json_err( StatusCode::CONFLICT, "submission_gated", @@ -187,8 +160,6 @@ async fn gate_one_max( Ok(None) } -/// Materialize a training-only request's architecture from the registry -/// (`Ok(())` for architecture submissions — nothing to pull). async fn materialize_arch(st: &AppState, req: &mut SubmissionRequest) -> Result<(), Response> { let Some(arch_id) = req .arch_id @@ -217,8 +188,6 @@ async fn materialize_arch(st: &AppState, req: &mut SubmissionRequest) -> Result< } } -/// `POST /v1/submissions/precheck` — advisory copy-gate (same logic as -/// intake), no submission row, no 1-max gate, no Lium. Quota: 3/coldkey/UTC day. async fn post_precheck( State(st): State>, headers: axum::http::HeaderMap, @@ -444,7 +413,6 @@ async fn get_events(State(st): State>, Path(id): Path) -> } } -/// `POST /v1/submissions/{id}/retry` — requeue a failed row (guard: max attempts). async fn post_retry(State(st): State>, Path(id): Path) -> Response { let row = match st.store.get(&id).await { Ok(Some(r)) => r, @@ -458,15 +426,33 @@ async fn post_retry(State(st): State>, Path(id): Path) -> &format!("status={}", row.status.as_str()), ); } - if row.retry_count >= st.retry_max { + let gate_key = prism_pipeline::gating_key(row.arch_id.as_deref()); + let mut infra = matches!(row.final_score, Some(FinalScore::NoScore(6))); + if infra { + if let Some(g) = &st.gating { + infra = g + .get(&gate_key, &row.miner_hotkey) + .await + .ok() + .flatten() + .is_some_and(|gr| infra_resubmit_allowed(&gr, now_ms())); + } + } + if row.retry_count >= st.retry_max && !infra { return json_err( StatusCode::CONFLICT, "retry_exhausted", &format!("retry_count={} max={}", row.retry_count, st.retry_max), ); } + if infra { + if let Some(g) = &st.gating { + let _ = g.reset_open(&gate_key, &row.miner_hotkey).await; + let _ = g.mark_registered(&gate_key, &row.miner_hotkey, None).await; + } + } match st.store.reset_for_retry(&id).await { - Ok(_row) => ( + Ok(_) => ( StatusCode::ACCEPTED, Json(json!({"submission_id": id, "status": "queued"})), ) @@ -507,30 +493,21 @@ async fn get_status(State(st): State>) -> Response { .into_response() } -/// Orchestrator job view: one row per active/recent pod (for ops). async fn get_jobs(State(st): State>) -> Response { let rows = match st.store.list(None, None, 200).await { Ok(v) => v, - Err(e) => { - return json_err(StatusCode::INTERNAL_SERVER_ERROR, "store", &e.to_string()); - } + Err(e) => return json_err(StatusCode::INTERNAL_SERVER_ERROR, "store", &e.to_string()), }; - let actionable: Vec = rows + let jobs: Vec = rows .iter() .filter(|r| !r.status.is_terminal() || r.pod_id.is_some()) .take(200) .map(|r| { - json!({ - "submission_id": r.id, - "miner_hotkey": r.miner_hotkey, - "status": r.status.as_str(), - "pod_id": r.pod_id, - "bpb": r.bpb, - "retry_count": r.retry_count, - }) + json!({"submission_id": r.id, "miner_hotkey": r.miner_hotkey, "status": r.status.as_str(), + "pod_id": r.pod_id, "bpb": r.bpb, "retry_count": r.retry_count}) }) .collect(); - Json(json!({"jobs": actionable})).into_response() + Json(json!({"jobs": jobs})).into_response() } async fn get_recipe() -> impl IntoResponse { @@ -544,17 +521,12 @@ async fn get_recipe_baseline() -> impl IntoResponse { })) } -/// `GET /v1/architectures` — published architecture registry (leaderboard -/// source: per-arch best bpb across all trainers). async fn get_architectures(State(st): State>) -> Response { match st.store.list_archs(200).await { Ok(rows) => Json(json!({ "architectures": rows.iter().map(|a| json!({ - "arch_id": a.arch_id, - "owner_hotkey": a.owner_hotkey, - "arch_digest": a.arch_digest, - "source_submission": a.source_submission, - "best_bpb": a.best_bpb, + "arch_id": a.arch_id, "owner_hotkey": a.owner_hotkey, "arch_digest": a.arch_digest, + "source_submission": a.source_submission, "best_bpb": a.best_bpb, "created_at_ms": a.created_at_ms, })).collect::>() })) @@ -563,7 +535,6 @@ async fn get_architectures(State(st): State>) -> Response { } } -/// Feed cache: call this from the worker loop every tick. pub fn record_epoch(st: &AppState, epoch: u64) { st.epoch.store(epoch, std::sync::atomic::Ordering::Relaxed); } @@ -645,14 +616,7 @@ fn map_submission_err(err: &SubmissionError) -> Response { } fn json_err(status: StatusCode, code: &str, message: &str) -> Response { - ( - status, - Json(json!({ - "error": message, - "code": code, - })), - ) - .into_response() + (status, Json(json!({"error": message, "code": code}))).into_response() } #[cfg(test)] @@ -660,6 +624,7 @@ mod tests { #![allow(clippy::unwrap_used)] use super::*; + use crate::NoScoreReasonCode; use axum::body::Body; use axum::http::Request; use http_body_util::BodyExt; @@ -1138,6 +1103,121 @@ mod tests { assert_eq!(v["status"], "already-queued"); } + #[tokio::test] + async fn infra_blocked_allows_resubmit_within_window() { + let (st, gating) = gated_state(&[[0x11; 32]]); + let hk = "11".repeat(32); + gating.mark_registered("prism", &hk, Some(0)).await.unwrap(); + gating + .set_terminal( + "prism", + &hk, + submission_gating::GatingState::Blocked, + Some("install"), + ) + .await + .unwrap(); + let app = submission_router(Arc::clone(&st)); + let mut req = crate::example_valid_request(); + req.architecture_py.push_str("\n# infra-retry\n"); + let body = serde_json::to_vec(&req).unwrap(); + let (s, v) = call( + app, + Request::post("/v1/submissions") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await; + assert_eq!(s, StatusCode::ACCEPTED, "{v}"); + } + + #[tokio::test] + async fn infra_blocked_rejects_after_window() { + let (st, gating) = gated_state(&[[0x11; 32]]); + let hk = "11".repeat(32); + gating.mark_registered("prism", &hk, Some(0)).await.unwrap(); + gating + .set_terminal( + "prism", + &hk, + submission_gating::GatingState::Blocked, + Some("install"), + ) + .await + .unwrap(); + assert!(gating.set_updated_at_ms( + "prism", + &hk, + now_ms().saturating_sub(submission_gating::INFRA_RESUBMIT_WINDOW_MS + 1), + )); + let app = submission_router(st); + let mut req = crate::example_valid_request(); + req.architecture_py.push_str("\n# too-late\n"); + let body = serde_json::to_vec(&req).unwrap(); + let (s, v) = call( + app, + Request::post("/v1/submissions") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await; + assert_eq!(s, StatusCode::CONFLICT, "{v}"); + assert_eq!(v["code"], "submission_gated"); + } + + #[tokio::test] + async fn challenge_internal_retry_bypasses_retry_max_in_window() { + let (st, gating) = gated_state(&[[0x11; 32]]); + let app = submission_router(Arc::clone(&st)); + let body = serde_json::to_vec(&crate::example_valid_request()).unwrap(); + let (s, v) = call( + app.clone(), + Request::post("/v1/submissions") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await; + assert_eq!(s, StatusCode::ACCEPTED, "{v}"); + let id = v["submission_id"].as_str().unwrap().to_owned(); + let hk = "11".repeat(32); + // Burn retry_count past retry_max with a ChallengeInternal terminal. + st.store + .apply( + &id, + &StatePatch { + status: Some(Stage::Failed), + final_score: Some(FinalScore::NoScore( + NoScoreReasonCode::ChallengeInternal as u8, + )), + retry_bump: st.retry_max.saturating_add(1), + ..StatePatch::default() + }, + None, + ) + .await + .unwrap(); + gating + .set_terminal( + "prism", + &hk, + submission_gating::GatingState::Blocked, + Some("install"), + ) + .await + .unwrap(); + let (s, v) = call( + app, + Request::post(format!("/v1/submissions/{id}/retry")) + .body(Body::empty()) + .unwrap(), + ) + .await; + assert_eq!(s, StatusCode::ACCEPTED, "{v}"); + } + #[tokio::test] async fn precheck_detects_copy_without_queuing() { let st = state(); diff --git a/crates/prism-lium/src/client.rs b/crates/prism-lium/src/client.rs index 3d77785a0..7ba275848 100644 --- a/crates/prism-lium/src/client.rs +++ b/crates/prism-lium/src/client.rs @@ -16,18 +16,10 @@ use crate::ssh::{ use crate::types::{GpuPreference, Instance, InstanceSpec, LiumSshConfig, Offer, RemoteExecResult}; use crate::{EvalJobBackend, HARNESS_LOG_RETAIN_BYTES, LIUM_API_BASE_URL, MIN_LIFETIME_HOURS}; -/// Pod image: Lium-owned DinD variant pulses its own dockerd init and never -/// Only `daturaai/*-dind` pods deliver a reachable sshd on this marketplace -/// (vanilla pytorch has none; cu12.8-dinD needs `service ssh start`, whose -/// process-exit kills sshd when the startup job ends — metachar rejection -/// makes keep-alive chains impossible). The cuda**13.0.2**-dinD tag runs sshd -/// from its own init: sleeps keep ssh alive across verify + exec phases. +/// `daturaai/*-dind` cuda13.0.2 — sshd from image init (empty startup). const RECIPES_TEMPLATE_IMAGE: &str = "daturaai/pytorch"; const RECIPES_TEMPLATE_TAG: &str = "2.12.0-py3.12-cuda13.0.2-devel-ubuntu24.04-dind"; -/// Recipe template ns (v9: the cu13 DinD shape that ssh-verifies with an -/// EMPTY startup script; proven ssh-stable ≥7 min on a 5070 Ti pod). const RECIPES_TEMPLATE_NAME: &str = "prism-recipe-v9"; -/// Boot: nothing — sshd comes up by itself in the cu13 dinD image. const RECIPES_TEMPLATE_STARTUP: &str = ""; const RUNNING_STATUSES: &[&str] = &["RUNNING", "RUNNING_SSH", "READY"]; @@ -39,12 +31,14 @@ const TERMINAL_FAIL_STATUSES: &[&str] = &[ "DELETED", "STOPPED", ]; +const RATE_LIMIT_RETRIES: u32 = 5; +const RATE_LIMIT_BASE_MS: u64 = 2_000; +const DEPS_INSTALL_TIMEOUT_SECS: u64 = 1_200; -/// Async Lium REST client. API key only in `X-API-Key` header. +/// Async Lium REST client (`X-API-Key`). pub struct LiumClient { http: reqwest::Client, base_url: String, - /// Stored but never Debug/Display'd. api_key: String, ssh: LiumSshConfig, } @@ -60,15 +54,13 @@ impl std::fmt::Debug for LiumClient { } impl LiumClient { - /// Build a client. `api_key` must be non-empty. - /// /// # Errors /// Empty key or HTTP client build failure. pub fn new(api_key: impl Into) -> Result { Self::with_base_url(api_key, LIUM_API_BASE_URL) } - /// Build with custom base URL (tests / wiremock). + /// Custom base URL (tests / wiremock). /// /// # Errors /// Empty key or HTTP client build failure. @@ -79,8 +71,6 @@ impl LiumClient { Self::with_config(api_key, base_url, LiumSshConfig::default_live()) } - /// Build with SSH config for live eval. - /// /// # Errors /// Empty key or HTTP client build failure. pub fn with_config( @@ -97,7 +87,6 @@ impl LiumClient { .map_err(|e| LiumError::Api(format!("invalid api key header: {e}")))?; hv.set_sensitive(true); headers.insert("X-API-Key", hv); - // Lium edge WAF returns 403 for empty/missing User-Agent. headers.insert( reqwest::header::USER_AGENT, HeaderValue::from_static("prism-lium/0.1 (base; +https://lium.io)"), @@ -119,13 +108,11 @@ impl LiumClient { }) } - /// Never expose key. #[must_use] pub fn base_url(&self) -> &str { &self.base_url } - /// Override private key path after construction. pub fn set_ssh_private_key_path(&mut self, path: PathBuf) { self.ssh.private_key_path = Some(path); } @@ -143,63 +130,56 @@ impl LiumClient { Ok(()) } - async fn request_json(&self, method: reqwest::Method, path: &str) -> Result { - let url = format!("{}{path}", self.base_url); - let resp = self - .http - .request(method.clone(), &url) - .send() - .await - .map_err(|e| LiumError::Transport(sanitize_err(&e.to_string(), &self.api_key)))?; - let status = resp.status(); - let text = resp - .text() - .await - .map_err(|e| LiumError::Transport(sanitize_err(&e.to_string(), &self.api_key)))?; - if !status.is_success() { - return Err(LiumError::Api(format!( - "{method} {path} -> {status}: {}", - truncate(&sanitize_err(&text, &self.api_key), 200) - ))); - } - if text.trim().is_empty() { - return Ok(Value::Null); - } - serde_json::from_str(&text).map_err(|e| LiumError::Api(format!("json: {e}"))) - } - - async fn request_json_body( + async fn request( &self, method: reqwest::Method, path: &str, - body: &Value, + body: Option<&Value>, ) -> Result { let url = format!("{}{path}", self.base_url); - let resp = self - .http - .request(method.clone(), &url) - .json(body) - .send() - .await - .map_err(|e| LiumError::Transport(sanitize_err(&e.to_string(), &self.api_key)))?; - let status = resp.status(); - let text = resp - .text() - .await - .map_err(|e| LiumError::Transport(sanitize_err(&e.to_string(), &self.api_key)))?; - if !status.is_success() { - return Err(LiumError::Api(format!( - "{method} {path} -> {status}: {}", - truncate(&sanitize_err(&text, &self.api_key), 200) - ))); - } - if text.trim().is_empty() { - return Ok(Value::Null); + let mut attempt = 0u32; + loop { + let mut builder = self.http.request(method.clone(), &url); + if let Some(b) = body { + builder = builder.json(b); + } + let resp = builder + .send() + .await + .map_err(|e| LiumError::Transport(sanitize_err(&e.to_string(), &self.api_key)))?; + let status = resp.status(); + let retry_after = resp + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.parse::().ok()); + let text = resp + .text() + .await + .map_err(|e| LiumError::Transport(sanitize_err(&e.to_string(), &self.api_key)))?; + if status.as_u16() == 429 && attempt < RATE_LIMIT_RETRIES { + attempt = attempt.saturating_add(1); + let wait_ms = retry_after.map_or( + RATE_LIMIT_BASE_MS << (attempt - 1).min(3), + |s| s.saturating_mul(1000).max(50), + ); + warn!(%path, attempt, wait_ms, "lium 429; backing off"); + sleep(Duration::from_millis(wait_ms)).await; + continue; + } + if !status.is_success() { + return Err(LiumError::Api(format!( + "{method} {path} -> {status}: {}", + truncate(&sanitize_err(&text, &self.api_key), 200) + ))); + } + if text.trim().is_empty() { + return Ok(Value::Null); + } + return serde_json::from_str(&text).map_err(|e| LiumError::Api(format!("json: {e}"))); } - serde_json::from_str(&text).map_err(|e| LiumError::Api(format!("json: {e}"))) } - /// Parse executors list into offers. fn parse_offers(v: &Value) -> Vec { let items = v .as_array() @@ -217,33 +197,32 @@ impl LiumClient { } async fn list_pods_raw(&self) -> Result, LiumError> { - let v = self.request_json(reqwest::Method::GET, "/pods").await?; + let v = self.request(reqwest::Method::GET, "/pods", None).await?; Ok(v.as_array() .cloned() .or_else(|| v.get("pods").and_then(|x| x.as_array().cloned())) .unwrap_or_default()) } - /// GET /pods/{id} raw JSON. pub async fn get_pod_raw(&self, instance_id: &str) -> Result { - self.request_json(reqwest::Method::GET, &format!("/pods/{instance_id}")) + self.request(reqwest::Method::GET, &format!("/pods/{instance_id}"), None) .await } - /// Parsed instance status. pub async fn status(&self, instance_id: &str) -> Result { let v = self.get_pod_raw(instance_id).await?; Ok(parse_instance(&v, instance_id)) } - /// Ensure SSH public key is registered with Lium (idempotent). pub async fn ensure_ssh_key( &self, public_key: &str, name: Option<&str>, ) -> Result { let normalized = public_key.trim(); - let v = self.request_json(reqwest::Method::GET, "/ssh-keys").await?; + let v = self + .request(reqwest::Method::GET, "/ssh-keys", None) + .await?; let keys = v .as_array() .cloned() @@ -263,11 +242,10 @@ impl LiumClient { if let Some(n) = name { body["name"] = Value::String(n.to_owned()); } - self.request_json_body(reqwest::Method::POST, "/ssh-keys", &body) + self.request(reqwest::Method::POST, "/ssh-keys", Some(&body)) .await } - /// Ensure a named template exists; return its id (idempotent). pub async fn ensure_template( &self, name: &str, @@ -276,7 +254,7 @@ impl LiumClient { startup_commands: Option<&str>, ) -> Result { let v = self - .request_json(reqwest::Method::GET, "/templates") + .request(reqwest::Method::GET, "/templates", None) .await?; let templates = v .as_array() @@ -301,12 +279,11 @@ impl LiumClient { if let Some(tag) = docker_image_tag { body["docker_image_tag"] = serde_json::Value::String(tag.to_owned()); } - // Metachar-free keep-alive; Lium rejects shell metachar startup chains. if let Some(cmd) = startup_commands { body["startup_commands"] = serde_json::Value::String(cmd.to_owned()); } let created = self - .request_json_body(reqwest::Method::POST, "/templates", &body) + .request(reqwest::Method::POST, "/templates", Some(&body)) .await?; created .get("id") @@ -315,7 +292,6 @@ impl LiumClient { .ok_or_else(|| LiumError::Api("template create missing id".into())) } - /// Resolve template id from spec (explicit id, name, or default e2e template). async fn resolve_template_id(&self, spec: &InstanceSpec) -> Result { if let Some(id) = &spec.template_id { if !id.is_empty() { @@ -338,7 +314,9 @@ impl LiumClient { /// Account balance (USD) when available. pub async fn balance(&self) -> Result { - let v = self.request_json(reqwest::Method::GET, "/users/me").await?; + let v = self + .request(reqwest::Method::GET, "/users/me", None) + .await?; v.get("balance") .and_then(|x| x.as_f64()) .or_else(|| { @@ -349,7 +327,6 @@ impl LiumClient { .ok_or_else(|| LiumError::Api("users/me missing balance".into())) } - /// Poll until RUNNING (or fail). pub async fn wait_until_running(&self, instance_id: &str) -> Result { let timeout = Duration::from_secs(self.ssh.running_timeout_secs.max(30)); let start = Instant::now(); @@ -401,11 +378,6 @@ impl LiumClient { }) } - /// Live recipe eval: wait RUNNING → SSH nvidia-smi → upload harness + - /// miner sources → run [`prism_recipe`] harness → parse `METRICS_JSON`. - /// - /// The harness trains the miner's code on the pinned fineweb-edu shard and - /// scores the frozen val cut; no metric comes from hashing sources. async fn exec_eval_live( &self, instance_id: &str, @@ -415,8 +387,8 @@ impl LiumClient { let _running = self.wait_until_running(instance_id).await?; let target = self.resolve_ssh_target(instance_id).await?; let key = resolve_private_key(self.ssh.private_key_path.as_deref())?; - let gpu_type = self.gpu_smoke(&target, &key).await?; + self.ensure_python_deps(&target, &key).await?; let arch_b64 = base64_encode(architecture_py.as_bytes()); let train_b64 = base64_encode(training_py.as_bytes()); @@ -424,19 +396,11 @@ impl LiumClient { let train_cap_secs = (self.ssh.train_hours_cap * 3600.0) as u64; let remote = format!( "set -e -command -v pip >/dev/null 2>&1 || apt-get update -q -command -v pip >/dev/null 2>&1 || DEBIAN_FRONTEND=noninteractive apt-get install -y -q python3-pip -python3 -c 'import torch' 2>/dev/null || echo 'torch stopping' -python3 -c 'import transformers' 2>/dev/null || pip install \ - --break-system-packages --root-user-action=ignore \ - 'transformers==4.44.2' 'datasets==3.0.2' 'pyarrow==17.0.0' mkdir -p /tmp/prism_eval echo '{harness_b64}' | base64 -d > /tmp/prism_eval/prism_harness.py echo '{arch_b64}' | base64 -d > /tmp/prism_eval/architecture.py echo '{train_b64}' | base64 -d > /tmp/prism_eval/training.py cd /tmp/prism_eval -# Persist full harness output on-pod so timeout / stuck-sweep can harvest the -# fatal tail even when the long-lived SSH session is killed without pipes. set +e PRISM_DATASET_URL='{dataset_url}' \ PRISM_DATASET_SHA256='{dataset_sha}' \ @@ -447,7 +411,6 @@ PRISM_GPU_TYPE='{gpu_type}' \ > /tmp/prism_eval/harness.log 2>&1 ec=$? set -e -# Surface the log tail on the SSH channel for the happy path + failure parse. tail -c 524288 /tmp/prism_eval/harness.log || true exit $ec\n", harness_b64 = harness_b64, @@ -474,7 +437,6 @@ exit $ec\n", { Ok(o) => o, Err(e) => { - // Session timed out / dropped — second SSH pulls the on-pod log. let harvested = self .harvest_logs_inner(instance_id) .await @@ -519,7 +481,33 @@ exit $ec\n", Ok(v) } - /// SSH-fetch the on-pod harness log tail (empty when missing / unreachable). + async fn ensure_python_deps(&self, target: &SshTarget, key: &Path) -> Result<(), LiumError> { + const VERIFY: &str = + "python3 -c 'import transformers, datasets, pyarrow; print(\"DEPS_OK\")'"; + const INSTALL: &str = "set -e\ncommand -v pip >/dev/null 2>&1 || { apt-get update -q; DEBIAN_FRONTEND=noninteractive apt-get install -y -q python3-pip; }\npython3 -c 'import transformers, datasets, pyarrow' 2>/dev/null || pip install --break-system-packages --root-user-action=ignore 'transformers==4.44.2' 'datasets==3.0.2' 'pyarrow==17.0.0'\npython3 -c 'import transformers, datasets, pyarrow; print(\"DEPS_OK\")'\n"; + let (a, r) = (self.ssh.ssh_attempts, self.ssh.ssh_retry_secs); + let out = + ssh_exec_allow_fail(target, key, INSTALL, a, r, DEPS_INSTALL_TIMEOUT_SECS).await?; + if out.stdout.contains("DEPS_OK") { + return Ok(()); + } + let v = ssh_exec_allow_fail(target, key, VERIFY, a.max(3), r, 120).await?; + if v.stdout.contains("DEPS_OK") { + return Ok(()); + } + Err(LiumError::Exec(format!( + "deps install failed (code {}): {}", + out.returncode, + truncate_tail( + &format!( + "{}\n{}\n---\n{}\n{}", + out.stdout, out.stderr, v.stdout, v.stderr + ), + HARNESS_LOG_RETAIN_BYTES + ) + ))) + } + async fn harvest_logs_inner(&self, instance_id: &str) -> Result { let target = self.resolve_ssh_target(instance_id).await?; let key = resolve_private_key(self.ssh.private_key_path.as_deref())?; @@ -556,12 +544,7 @@ exit $ec\n", } } -/// Optional test-mode env lines for the pod harness (`KEY='v' \\\n` pairs). -/// -/// Forwards `PRISM_TEST_TRAIN_MINUTES` / `PRISM_TEST_MAX_PARAMS` from the -/// master process env so staging can run short/tiny evals. Values are -/// forwarded only when they parse as plain numerics — the remote string is -/// shell, so anything else would be an injection vector. +/// Forward numeric-only `PRISM_TEST_*` env into the remote harness shell. fn test_mode_env() -> String { use std::fmt::Write as _; let mut out = String::new(); @@ -608,7 +591,6 @@ fn parse_one_offer(item: &Value) -> Option { .or_else(|| item.get("executor_id")) .and_then(|x| x.as_str())? .to_owned(); - // Live API uses machine_name + price_per_gpu (not gpu_type / price_per_hour). let gpu_type = item .get("gpu_type") .or_else(|| item.get("gpu_name")) @@ -703,7 +685,7 @@ fn extract_pod_id(v: &Value) -> Option { impl EvalJobBackend for LiumClient { async fn list_offers(&self, max_price_per_hour: Option) -> Result, LiumError> { let v = self - .request_json(reqwest::Method::GET, "/executors") + .request(reqwest::Method::GET, "/executors", None) .await?; let mut offers = Self::parse_offers(&v); if let Some(max) = max_price_per_hour { @@ -730,9 +712,6 @@ impl EvalJobBackend for LiumClient { } let mut offers = self.list_offers(Some(spec.max_price_per_hour)).await?; - // GPU pin first (RTX 5090 → ordered fallback), price only breaks ties - // within the same rank: cheapest-first would silently drift evals - // onto weaker GPUs whenever the pinned SKU is not the cheapest offer. let pref = GpuPreference::default_prism(); offers.sort_by(|a, b| { pref.rank(&a.gpu_type) @@ -783,19 +762,18 @@ impl EvalJobBackend for LiumClient { %template_id, "lium rent" ); - // Try the requested split first; on "no GPU splitting allowed" - // retry the whole node (per-GPU price is unchanged). let mut split_choices = vec![spec.gpu_count]; if selected.gpu_count != spec.gpu_count { split_choices.push(selected.gpu_count); } let mut rented: Result = Err(LiumError::Api("unrented".into())); for gcount in split_choices { + let body = make_body(gcount); rented = self - .request_json_body( + .request( reqwest::Method::POST, &format!("/executors/{}/rent", selected.id), - &make_body(gcount), + Some(&body), ) .await; if let Err(e) = &rented { @@ -831,8 +809,6 @@ impl EvalJobBackend for LiumClient { self.cleanup_after_rent(None).await; continue; }; - // Wait for RUNNING here so a CREATION_FAILED offer falls - // through to the next candidate instead of poisoning exec. match self.wait_until_running(&id).await { Ok(inst) => return Ok(inst), Err(e) => { @@ -843,6 +819,9 @@ impl EvalJobBackend for LiumClient { } Err(e) => { last_err = e.to_string(); + if last_err.contains("429") { + sleep(Duration::from_millis(RATE_LIMIT_BASE_MS)).await; + } self.cleanup_after_rent(pod_id.as_deref()).await; } } @@ -962,6 +941,33 @@ mod tests { assert_eq!(offers[0].id, "a"); } + #[tokio::test] + async fn request_retries_on_429_then_succeeds() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/executors")) + .respond_with( + ResponseTemplate::new(429) + .insert_header("Retry-After", "0") + .set_body_string("Too many requests"), + ) + .up_to_n_times(1) + .expect(1..) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/executors")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!([ + {"id": "a", "gpu_type": "NVIDIA A100", "gpu_count": 1, "price_per_hour": 0.5} + ]))) + .mount(&server) + .await; + let c = LiumClient::with_base_url("test-key", server.uri()).unwrap(); + let offers = c.list_offers(None).await.unwrap(); + assert_eq!(offers.len(), 1); + assert_eq!(offers[0].id, "a"); + } + fn provision_spec() -> InstanceSpec { InstanceSpec { name: "x".into(), diff --git a/crates/prism-lium/src/ssh.rs b/crates/prism-lium/src/ssh.rs index bb548f464..060d36418 100644 --- a/crates/prism-lium/src/ssh.rs +++ b/crates/prism-lium/src/ssh.rs @@ -164,9 +164,9 @@ pub async fn ssh_exec( .arg("-o") .arg("ConnectTimeout=15") .arg("-o") - .arg("ServerAliveInterval=10") + .arg("ServerAliveInterval=30") .arg("-o") - .arg("ServerAliveCountMax=60") + .arg("ServerAliveCountMax=120") .arg("-o") .arg("TCPKeepAlive=yes") .arg("-o") @@ -242,9 +242,9 @@ pub async fn ssh_exec_allow_fail( .arg("-o") .arg("ConnectTimeout=15") .arg("-o") - .arg("ServerAliveInterval=10") + .arg("ServerAliveInterval=30") .arg("-o") - .arg("ServerAliveCountMax=60") + .arg("ServerAliveCountMax=120") .arg("-o") .arg("TCPKeepAlive=yes") .arg("-o") diff --git a/crates/submission-gating/src/lib.rs b/crates/submission-gating/src/lib.rs index 2cf285276..e82ae0926 100644 --- a/crates/submission-gating/src/lib.rs +++ b/crates/submission-gating/src/lib.rs @@ -8,11 +8,14 @@ //! ```text //! open ──intake accept──▶ registered ──terminal retry-exhausted──▶ blocked //! └─cheat / copy gate─────────────────────▶ rejected +//! blocked (infra) ──miner resubmit ≤30m──▶ registered (new intake) //! blocked|rejected ──watcher: hotkey gone from metagraph──▶ open //! ``` //! -//! Non-`open` rows make intake fail with an explicit 409; only the watcher -//! (or an out-of-band operator) returns a row to `open`. +//! Non-`open` rows make intake fail with an explicit 409, except infra +//! `blocked` within [`INFRA_RESUBMIT_WINDOW_MS`] (install / AST / LLM). +//! Cheat `rejected` never soft-reopens; the watcher (or an operator) +//! returns other rows to `open`. #![forbid(unsafe_code)] #![allow(clippy::missing_errors_doc)] @@ -62,6 +65,24 @@ impl GatingState { } } +/// Miner may POST a new submission after infra `blocked` for this long. +pub const INFRA_RESUBMIT_WINDOW_MS: u64 = 30 * 60 * 1000; + +/// Infra auto-retry / `ChallengeInternal` classes (not cheat). +#[must_use] +pub fn is_infra_error_class(class: Option<&str>) -> bool { + matches!(class, Some("install" | "ast_infra" | "llm_infra")) +} + +/// `blocked` + infra class + still inside the post-install resubmit window. +#[must_use] +pub fn infra_resubmit_allowed(row: &GatingRow, now_ms: u64) -> bool { + row.state == GatingState::Blocked + && is_infra_error_class(row.last_error_class.as_deref()) + && row.updated_at_ms > 0 + && now_ms.saturating_sub(row.updated_at_ms) <= INFRA_RESUBMIT_WINDOW_MS +} + /// One `submission_gating` row. #[derive(Debug, Clone, PartialEq, Eq)] pub struct GatingRow { @@ -142,6 +163,18 @@ impl MemoryGatingStore { pub fn new() -> Self { Self::default() } + + /// Test helper: backdate / set `updated_at_ms` for window checks. + pub fn set_updated_at_ms(&self, challenge: &str, hotkey: &str, updated_at_ms: u64) -> bool { + let Ok(mut m) = self.rows.lock() else { + return false; + }; + let Some(row) = m.get_mut(&(challenge.to_owned(), hotkey.to_owned())) else { + return false; + }; + row.updated_at_ms = updated_at_ms; + true + } } #[async_trait] @@ -652,6 +685,32 @@ mod tests { ); } + #[test] + fn infra_resubmit_window_helpers() { + let now = 10_000_000u64; + let mut row = GatingRow { + challenge: "prism".into(), + hotkey: hk(1), + uid: Some(0), + state: GatingState::Blocked, + attempt_count: 3, + last_error_class: Some("install".into()), + created_at_ms: now, + updated_at_ms: now - 60_000, + }; + assert!(infra_resubmit_allowed(&row, now)); + row.updated_at_ms = now.saturating_sub(INFRA_RESUBMIT_WINDOW_MS + 1); + assert!(!infra_resubmit_allowed(&row, now)); + row.updated_at_ms = now; + row.last_error_class = Some("miner".into()); + assert!(!infra_resubmit_allowed(&row, now)); + row.last_error_class = Some("install".into()); + row.state = GatingState::Rejected; + assert!(!infra_resubmit_allowed(&row, now)); + assert!(is_infra_error_class(Some("ast_infra"))); + assert!(!is_infra_error_class(Some("cheat"))); + } + #[tokio::test] async fn watch_once_updates_cache_and_reconciles() { let chain = chain::FakeChain::new(chain::FakeChainConfig::default()); diff --git a/docs/PRISM.md b/docs/PRISM.md index eb3b8e7d3..eeb7657db 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -79,9 +79,11 @@ enforces **one accepted submission per `(prism, hotkey)`** unknown hotkey → `403 hotkey_not_in_metagraph`. Infra-class failures (`install` = Lium/pod, `ast_infra` = similarity, `llm_infra` = review/agentic) **auto-retry up to 3 times** before a terminal `blocked`; cheat / suspicious -verdicts are terminal `rejected` (no retry). A metagraph **watcher** reopens -eligibility when the hotkey leaves the metagraph (uid deregistered or hotkey -replaced). Manual `POST /v1/submissions/{id}/retry` is unchanged. +verdicts are terminal `rejected` (no retry). After an infra `blocked`, the +miner may **resubmit for up to 30 minutes** (new `POST /v1/submissions` or +`POST /v1/submissions/{id}/retry` for `ChallengeInternal`); after the window +the slot stays blocked until the metagraph watcher reopens it (hotkey left / +replaced). **Training-only entries** gate separately under the composite challenge key `prism:train:`: one accepted entry per `(hotkey, arch_id)`, with diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index 12547152e..23f8ab5d8 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -79,17 +79,19 @@ quota — trust `/v1/recipe`, not the chart meta line. → `403 hotkey_not_in_metagraph`; a fresh registration may lag the snapshot (`503 metagraph_unavailable` → retry shortly). - **One accepted architecture submission per hotkey.** While yours is - `registered` / `blocked` / `rejected`, a *different* architecture submission - gets `409 submission_gated`. Re-POSTing the **identical** sources is always - safe (idempotent `200 already-queued`). + `registered` / `rejected`, or `blocked` **outside** the infra window, a + *different* architecture submission gets `409 submission_gated`. Re-POSTing + the **identical** sources is always safe (idempotent `200 already-queued`). - **Training-only entries are separate**: one accepted entry per `(hotkey, arch_id)`, same retry rules — you may train on many published archs, one script per arch. - If your hotkey **leaves the metagraph**, the watcher reopens your slot(s) automatically — resubmit under your new uid. - Infra failures (Lium pod, review/similarity/LLM infra) **auto-retry up to 3 - times**; cheat / rejected verdicts are terminal. Manual retry: - `POST /v1/submissions/{id}/retry`. + times**; cheat / rejected verdicts are terminal. After an infra failure + (`ChallengeInternal`), you may **resubmit within 30 minutes** (new POST or + `POST /v1/submissions/{id}/retry`). After 30 minutes the slot stays blocked + until your hotkey leaves the metagraph. ## Anti-copy rule (architecture-only)