diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index cbd41f8fc..4f79efb4f 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -1,5 +1,8 @@ -//! Lium job orchestrator: claim→screen→pod→review→score; epoch emitter via +//! Lium job orchestrator: claim→screen→review→pod→score; epoch emitter via //! [`Orchestrator::run_emitter`]. State in store; API is a projection. +//! +//! Pre-pod order is fail-closed: copy/static/similarity + LLM quality + +//! agentic (sources) must pass before any Lium rent. use std::sync::Arc; use std::time::Duration; @@ -444,9 +447,10 @@ impl Orchestrator { let _active = self.active.enter(&id); info!(submission_id = %id, miner = %row.miner_hotkey, "prism eval start"); - let Some(similarity) = self.pre_pod_screens(&id, &row).await else { + let Some(pre) = self.pre_pod_screens(&id, &row).await else { return Ok(()); }; + let (similarity, review, _pre_agentic) = pre; let (measured, fresh) = match resume_measurement(&row) { Some(mr) => (Ok(mr), false), @@ -472,10 +476,8 @@ impl Orchestrator { } let bpb = metrics.as_ref().map(|m| m.bpb); - let Some(review) = self.review_step(&id, &row).await else { - return Ok(()); - }; - + // Metrics-aware agentic pass (inconsistent_metrics / eval forge). + // Structural cheats already failed closed pre-pod — this never rents. let Some(agentic) = self .agentic_step(&id, &row, metrics.as_ref(), receipt.as_ref()) .await @@ -544,10 +546,19 @@ impl Orchestrator { Ok(()) } - /// Pre-pod screens: copy gate → static cheat → AST similarity. - /// Returns `Some(similarity)` when the row may proceed to Lium rent; - /// `None` when already finalized (rejected / failed / retrying). - async fn pre_pod_screens(&self, id: &str, row: &SubmissionState) -> Option { + /// Pre-pod screens: copy → static → similarity → LLM quality → agentic. + /// Returns gates when the row may proceed to Lium rent; `None` when + /// already finalized (rejected / failed / retrying). OpenRouter / agentic + /// infra errors fail closed here — they must never rent a pod. + async fn pre_pod_screens( + &self, + id: &str, + row: &SubmissionState, + ) -> Option<( + SimilarityVerdict, + prism_review::ReviewVerdict, + AgenticVerdict, + )> { if self.copy_gate_step(row).await { return None; } @@ -580,7 +591,26 @@ impl Orchestrator { .await; return None; } - Some(similarity) + let review = self.review_step(id, row).await?; + let agentic = self.agentic_step(id, row, None, None).await?; + if matches!( + agentic.verdict, + VerdictKind::Cheat | VerdictKind::Suspicious + ) { + let detail = format!("pre-pod agentic: {:?}", agentic.verdict); + self.reject_pre_pod( + row, + Some(similarity), + Some(serde_json::json!({ + "gate": "agentic_pre_pod", + "agentic": serde_json::to_value(&agentic).unwrap_or_default(), + })), + detail, + ) + .await; + return None; + } + Some((similarity, review, agentic)) } /// Pre-LLM copy gate on `architecture.py`. Returns `true` when the row was diff --git a/crates/prism-challenge/tests/agentic_review_retry.rs b/crates/prism-challenge/tests/agentic_review_retry.rs index 0139b4b39..1c4212d5f 100644 --- a/crates/prism-challenge/tests/agentic_review_retry.rs +++ b/crates/prism-challenge/tests/agentic_review_retry.rs @@ -1,9 +1,7 @@ -//! Post-run review-stage failures must never re-run the pod job (E12): -//! the completed measurement is persisted at measure time and survives the -//! retry reset, so an `llm_infra` auto-retry resumes at the review stages. -//! An exhausted retry budget finalizes `NoScore(ChallengeInternal)` -//! (fail-closed, per PRISM.md §4 "missing / unparseable") — never a -//! miner-zero, never an infinite retrain loop. +//! Review-stage failures must never burn GPU: +//! - Pre-pod agentic/LLM infra fails closed **before** Lium rent. +//! - A metrics-aware agentic failure after a completed measurement must +//! resume without re-provisioning (E12). #![forbid(unsafe_code)] #![allow(clippy::expect_used, clippy::unwrap_used)] @@ -14,7 +12,7 @@ use std::sync::{Arc, Mutex}; use async_trait::async_trait; use chain::{AxonInfo, ChainError, Metagraph, WeightsTlockPayload}; use chain::{ChainClient, FakeChain, FakeChainConfig}; -use challenge_agentic::{AgenticBackend, AgenticError, AgenticVerdict, ReviewRequest}; +use challenge_agentic::{AgenticBackend, AgenticError, AgenticVerdict, ReviewRequest, VerdictKind}; use crypto::KEY_LEN; use prism_challenge::{ FinalScore, GatewayClient, GatewayClientConfig, MemoryPrismStore, Orchestrator, @@ -133,8 +131,7 @@ impl EvalJobBackend for CountingBackend { } } -/// Agentic backend that always dies the way the live `transformer_pp` -/// verification run did (budget exhausted mid-review, no verdict). +/// Always dies the way a live `OpenRouter` budget exhaustion does. struct BudgetDeadAgent; #[async_trait] @@ -146,8 +143,43 @@ impl AgenticBackend for BudgetDeadAgent { } } +/// Pre-pod (no metrics) succeeds; metrics-aware pass fails with infra error. +struct MetricsPassDeadAgent; + +#[async_trait] +impl AgenticBackend for MetricsPassDeadAgent { + async fn review(&self, req: &ReviewRequest) -> Result { + if req.metrics_relpath.is_none() { + return Ok(AgenticVerdict { + verdict: VerdictKind::Clean, + cheat_codes: vec![], + nearest_id: None, + similarity_bps: 0, + rationale: "pre-pod structural clean".into(), + }); + } + Err(AgenticError::NoVerdict( + "token budget exhausted (39869)".into(), + )) + } +} + +fn training_with_hooks() -> &'static str { + concat!( + "import prism_telemetry\n", + "def train(model, ctx):\n", + " prism_telemetry.report(loss=1.0, step=1)\n", + " prism_telemetry.finish_evaluation()\n", + " return {'loss': 1.0}\n", + ) +} + +fn architecture_py() -> &'static str { + "import torch\ndef build_model(ctx):\n return torch.nn.Linear(8, 8)\n" +} + #[tokio::test] -async fn agentic_infra_retry_resumes_without_remeasure() { +async fn agentic_infra_pre_pod_never_provisions() { let store = Arc::new(MemoryPrismStore::new()); let chain = Arc::new(LockedFake(Mutex::new(fake_chain()))); let gateway = Arc::new( @@ -177,16 +209,91 @@ async fn agentic_infra_retry_resumes_without_remeasure() { sk, ); - let architecture_py = "import torch\ndef build_model(ctx):\n return torch.nn.Linear(8, 8)\n"; - // Telemetry hooks keep the pre-pod static screen out of the way: this test - // is about the review-stage retry, not the contract. - let training_py = concat!( - "import prism_telemetry\n", - "def train(model, ctx):\n", - " prism_telemetry.report(loss=1.0, step=1)\n", - " prism_telemetry.finish_evaluation()\n", - " return {'loss': 1.0}\n", + let id = "agentic-pre-pod-no-rent".to_owned(); + store + .insert_queued(&SubmissionState { + id: id.clone(), + miner_hotkey: "11".repeat(32), + miner_coldkey: None, + epoch: 7, + netuid: 541, + status: Stage::Queued, + architecture_py: architecture_py().into(), + training_py: training_with_hooks().into(), + tree_blob: None, + label: Some("agentic-pre-pod".into()), + pod_id: None, + pod_provider: None, + receipt: None, + metrics_json: None, + bpb: None, + arch_id: None, + review: None, + similarity: None, + final_score: None, + retry_count: 0, + error_detail: None, + created_at_ms: 1, + updated_at_ms: 1, + }) + .await + .unwrap(); + + assert!(orch.cycle_once().await.unwrap()); + let row = store.get(&id).await.unwrap().expect("row"); + assert_eq!(row.status, Stage::Queued, "auto-retried: {row:?}"); + assert_eq!(row.retry_count, 1); + assert!(row.receipt.is_none() && row.metrics_json.is_none()); + assert_eq!( + backend.provisions.load(Ordering::SeqCst), + 0, + "pre-pod agentic infra must not rent a pod" ); + + assert!(orch.cycle_once().await.unwrap()); + assert_eq!(backend.provisions.load(Ordering::SeqCst), 0); + assert_eq!(backend.exec_calls.load(Ordering::SeqCst), 0); + let row = store.get(&id).await.unwrap().expect("row"); + assert_eq!(row.status, Stage::Failed, "{row:?}"); + assert_eq!( + row.final_score, + Some(FinalScore::NoScore(6)), + "review-inconclusive terminal = NoScore(ChallengeInternal), got {:?}", + row.final_score + ); +} + +#[tokio::test] +async fn agentic_infra_retry_resumes_without_remeasure() { + let store = Arc::new(MemoryPrismStore::new()); + let chain = Arc::new(LockedFake(Mutex::new(fake_chain()))); + let gateway = Arc::new( + GatewayClient::new(GatewayClientConfig { + base_url: "dry-run".into(), + max_attempts: 1, + backoff: std::time::Duration::from_millis(1), + }) + .unwrap(), + ); + let mut sk = [7u8; KEY_LEN]; + sk[0] = 0x42; + let backend = Arc::new(CountingBackend::new()); + let orch = Orchestrator::new( + OrchestratorConfig { + netuid: 541, + auto_retry_max: 1, + claim_poll: std::time::Duration::from_millis(10), + ..Default::default() + }, + Arc::clone(&store) as Arc, + Arc::clone(&backend) as Arc, + Arc::new(SimReviewer::new()), + Arc::new(MetricsPassDeadAgent), + &gateway, + chain, + sk, + ); + let id = "agentic-retry-resume".to_owned(); store .insert_queued(&SubmissionState { @@ -196,8 +303,8 @@ async fn agentic_infra_retry_resumes_without_remeasure() { epoch: 7, netuid: 541, status: Stage::Queued, - architecture_py: architecture_py.into(), - training_py: training_py.into(), + architecture_py: architecture_py().into(), + training_py: training_with_hooks().into(), tree_blob: None, label: Some("agentic-retry".into()), pod_id: None, @@ -217,8 +324,7 @@ async fn agentic_infra_retry_resumes_without_remeasure() { .await .unwrap(); - // Cycle 1: the pod job runs once, then the review-stage failure - // auto-retries. The measurement must survive the retry reset. + // Cycle 1: pre-pod agentic clean → pod once → metrics agentic fails → retry. assert!(orch.cycle_once().await.unwrap()); let row = store.get(&id).await.unwrap().expect("row"); assert_eq!(row.status, Stage::Queued, "auto-retried: {row:?}"); @@ -228,18 +334,17 @@ async fn agentic_infra_retry_resumes_without_remeasure() { "measurement must survive the post-run retry reset: {row:?}" ); - // Cycle 2: resumes at the review stages — no fresh pod, no re-measure — - // then the exhausted budget finalizes fail-closed. + // Cycle 2: resumes measurement — no fresh pod — then fail-closed. assert!(orch.cycle_once().await.unwrap()); assert_eq!( backend.provisions.load(Ordering::SeqCst), 1, - "a review-stage retry must not provision a second pod" + "a metrics-review retry must not provision a second pod" ); assert_eq!( backend.exec_calls.load(Ordering::SeqCst), 1, - "a review-stage retry must not re-measure" + "a metrics-review retry must not re-measure" ); let row = store.get(&id).await.unwrap().expect("row"); assert_eq!(row.status, Stage::Failed, "{row:?}"); diff --git a/crates/prism-recipe/harness/main.py b/crates/prism-recipe/harness/main.py index 3097c76f7..a29809f4e 100644 --- a/crates/prism-recipe/harness/main.py +++ b/crates/prism-recipe/harness/main.py @@ -156,19 +156,17 @@ def _eval_battery_status(): def _detect_flow(): - """v1 (legacy single invocation) vs v3 (two-phase train/eval). + """v1 (legacy single invocation) vs v3 (two-phase train/eval + G1–G8). - Explicit `PRISM_FLOW=v1|v3` wins. Otherwise the flow stays - v1-compatible until the operator stages private assets or a secret - seed — the battery then runs in the v3 child with the public dev - family when assets are absent (`eval_tier: "public_dev"`). + Explicit `PRISM_FLOW=v1|v3` wins. Default is **v3** so scored runs always + execute the public battery (full pack when `PRISM_EVAL_ASSETS_DIR` is + staged; otherwise `eval_tier=public_dev` fixtures — never silent BPB-only). + Set `PRISM_FLOW=v1` only for legacy single-shot compatibility. """ f = os.environ.get("PRISM_FLOW", "").strip().lower() if f in ("v1", "v3"): return f - if os.environ.get("PRISM_EVAL_ASSETS_DIR") or os.environ.get("PRISM_EVAL_SECRET_SEED"): - return "v3" - return "v1" + return "v3" def _cheatguard(): diff --git a/crates/prism-registry/src/hf.rs b/crates/prism-registry/src/hf.rs new file mode 100644 index 000000000..8331897c8 --- /dev/null +++ b/crates/prism-registry/src/hf.rs @@ -0,0 +1,306 @@ +//! HuggingFace Hub top-model publisher. +//! +//! When a submission becomes the new global-best bpb, source files +//! (`architecture.py`, `training.py`, `METRICS.json`, `README.md`) are +//! committed to `PRISM_TOPMODEL_HF_REPO` (default +//! `BaseIntelligence/prism-top-model`) via the Hub ndjson commit API. +//! +//! Token discipline mirrors GitHub: read from +//! `PRISM_TOPMODEL_HF_TOKEN_FILE` (e.g. `deploy/secrets/huggingface/token`), +//! never from env text. Absent/empty → graceful no-op. + +use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; +use tracing::info; + +use crate::publish::{PublishError, TopModelRequest}; + +const DEFAULT_API_BASE: &str = "https://huggingface.co"; +const DEFAULT_REPO: &str = "BaseIntelligence/prism-top-model"; +const DEFAULT_REVISION: &str = "main"; + +/// HuggingFace Hub publisher (token never `Debug`/`Display`'d). +pub struct HfTopModelPublisher { + http: reqwest::Client, + api_base: String, + repo: String, + revision: String, +} + +impl std::fmt::Debug for HfTopModelPublisher { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HfTopModelPublisher") + .field("api_base", &self.api_base) + .field("repo", &self.repo) + .field("revision", &self.revision) + .field("http", &"") + .finish_non_exhaustive() + } +} + +impl HfTopModelPublisher { + /// Configured Hub repo id (`org/name`). + #[must_use] + pub fn repo_id(&self) -> &str { + &self.repo + } + + /// `None` when the token file env is unset/empty. + #[must_use] + pub fn from_env() -> Option { + let path = std::env::var("PRISM_TOPMODEL_HF_TOKEN_FILE").ok()?; + let token = std::fs::read_to_string(path).ok()?.trim().to_owned(); + if token.len() < 8 { + return None; + } + let repo = std::env::var("PRISM_TOPMODEL_HF_REPO").unwrap_or_else(|_| DEFAULT_REPO.into()); + let revision = + std::env::var("PRISM_TOPMODEL_HF_REVISION").unwrap_or_else(|_| DEFAULT_REVISION.into()); + Self::with_config(token, DEFAULT_API_BASE, repo, revision).ok() + } + + /// Explicit config (tests / wiremock). + pub fn with_config( + token: impl Into, + api_base: impl Into, + repo: impl Into, + revision: impl Into, + ) -> Result { + let token = token.into(); + if token.trim().is_empty() { + return Err(PublishError::Transport("empty hf token".into())); + } + let mut headers = reqwest::header::HeaderMap::new(); + let mut hv = reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")) + .map_err(|e| PublishError::Transport(e.to_string()))?; + hv.set_sensitive(true); + headers.insert(reqwest::header::AUTHORIZATION, hv); + headers.insert( + reqwest::header::USER_AGENT, + reqwest::header::HeaderValue::from_static("base-prism-topmodel-hf/0.1"), + ); + let http = reqwest::Client::builder() + .default_headers(headers) + .timeout(std::time::Duration::from_mins(10)) + .build() + .map_err(|e| PublishError::Transport(e.to_string()))?; + Ok(Self { + http, + api_base: api_base.into().trim_end_matches('/').to_owned(), + repo: repo.into(), + revision: revision.into(), + }) + } + + /// Ensure the model repo exists, then commit top-model sources. + /// + /// # Errors + /// Transport / Hub API failures. + pub async fn publish(&self, req: &TopModelRequest) -> Result { + self.ensure_repo().await?; + let arch = req.arch_id.as_deref().unwrap_or("arch-unregistered"); + let readme = format!( + "# PRISM top model (HuggingFace)\n\n\ + Global-best bpb champion published by the Base master.\n\n\ + | field | value |\n|---|---|\n\ + | arch_id | `{arch}` |\n\ + | bpb | `{:.6}` |\n\ + | submission | `{}` |\n\ + | owner_hotkey | `{}…` |\n\n\ + Companion GitHub publish (when configured) lives under\n\ + `BaseIntelligence/prism` `top-model/`.\n", + req.bpb, + req.submission_id, + req.owner_hotkey.chars().take(12).collect::(), + ); + let metrics = serde_json::to_string_pretty(&serde_json::json!({ + "submission_id": req.submission_id, + "arch_id": req.arch_id, + "owner_hotkey": req.owner_hotkey, + "bpb": req.bpb, + "n_params": req.metrics_json.as_ref().and_then(|m| m.get("n_params")), + "tokens_seen": req.metrics_json.as_ref().and_then(|m| m.get("tokens_seen")), + "wall_clock_seconds": req.metrics_json.as_ref().and_then(|m| m.get("wall_clock_seconds")), + "battery": req.metrics_json.as_ref().and_then(|m| m.get("battery")), + "eval_tier": req.metrics_json.as_ref().and_then(|m| m.get("eval_tier")), + "flow": req.metrics_json.as_ref().and_then(|m| m.get("flow")), + })) + .unwrap_or_else(|_| "{}".into()); + let files: [(&str, &[u8]); 4] = [ + ("architecture.py", req.architecture_py.as_bytes()), + ("training.py", req.training_py.as_bytes()), + ("METRICS.json", metrics.as_bytes()), + ("README.md", readme.as_bytes()), + ]; + let oid = self + .commit_files(&format!("top-model: {arch} bpb={:.4}", req.bpb), &files) + .await?; + info!( + submission_id = %req.submission_id, + repo = %self.repo, + commit = %oid, + "top model published to HuggingFace" + ); + Ok(oid) + } + + async fn ensure_repo(&self) -> Result<(), PublishError> { + let (org, name) = split_repo(&self.repo)?; + let url = format!("{}/api/repos/create", self.api_base); + let body = serde_json::json!({ + "name": name, + "organization": org, + "private": false, + "type": "model", + }); + let resp = self + .http + .post(&url) + .json(&body) + .send() + .await + .map_err(|e| PublishError::Transport(e.to_string()))?; + let status = resp.status(); + // 409 / already exists → ok; 200/201 → created. + if status.is_success() || status.as_u16() == 409 { + return Ok(()); + } + let text = resp.text().await.unwrap_or_default(); + // Hub sometimes returns 400 "You already created this repository". + if text.to_ascii_lowercase().contains("already") { + return Ok(()); + } + Err(PublishError::Api(format!( + "hf create repo {status}: {text}" + ))) + } + + async fn commit_files( + &self, + summary: &str, + files: &[(&str, &[u8])], + ) -> Result { + let url = format!( + "{}/api/models/{}/commit/{}", + self.api_base, self.repo, self.revision + ); + let mut ndjson = String::new(); + ndjson.push_str( + &serde_json::json!({ + "key": "header", + "value": {"summary": summary, "description": ""} + }) + .to_string(), + ); + ndjson.push('\n'); + for (path, bytes) in files { + let line = serde_json::json!({ + "key": "file", + "value": { + "content": B64.encode(bytes), + "path": path, + "encoding": "base64", + } + }); + ndjson.push_str(&line.to_string()); + ndjson.push('\n'); + } + let resp = self + .http + .post(&url) + .header(reqwest::header::CONTENT_TYPE, "application/x-ndjson") + .body(ndjson) + .send() + .await + .map_err(|e| PublishError::Transport(e.to_string()))?; + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + if !status.is_success() { + return Err(PublishError::Api(format!("hf commit {status}: {body}"))); + } + let v: serde_json::Value = + serde_json::from_str(&body).map_err(|e| PublishError::Api(e.to_string()))?; + Ok(v.get("commitOid") + .and_then(|x| x.as_str()) + .unwrap_or("ok") + .to_owned()) + } +} + +fn split_repo(repo: &str) -> Result<(&str, &str), PublishError> { + let mut parts = repo.splitn(2, '/'); + let org = parts + .next() + .filter(|s| !s.is_empty()) + .ok_or_else(|| PublishError::Transport("hf repo missing org".into()))?; + let name = parts + .next() + .filter(|s| !s.is_empty()) + .ok_or_else(|| PublishError::Transport("hf repo missing name".into()))?; + Ok((org, name)) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn req() -> TopModelRequest { + TopModelRequest { + submission_id: "subm-hf".into(), + arch_id: Some("arch_hf".into()), + owner_hotkey: "cd".repeat(32), + bpb: 1.1, + architecture_py: "def build_model(ctx):\n pass\n".into(), + training_py: "def train(model, ctx):\n return {}\n".into(), + metrics_json: Some(serde_json::json!({ + "n_params": 1, + "battery": {"g1": {"status": "ok"}}, + "flow": "v3", + "eval_tier": "public", + })), + checkpoint_path: None, + } + } + + #[tokio::test] + async fn commits_sources_via_ndjson() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/repos/create")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({}))) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/api/models/BaseIntelligence/prism-top-model/commit/main")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "commitOid": "hfoid123", + "commitUrl": "https://huggingface.co/BaseIntelligence/prism-top-model/commit/hfoid123" + }))) + .mount(&server) + .await; + let p = HfTopModelPublisher::with_config( + "hf_tok_test", + server.uri(), + "BaseIntelligence/prism-top-model", + "main", + ) + .unwrap(); + let oid = p.publish(&req()).await.unwrap(); + assert_eq!(oid, "hfoid123"); + } + + #[test] + fn from_env_graceful_without_file() { + assert!(HfTopModelPublisher::from_env().is_none()); + } + + #[test] + fn split_repo_ok() { + assert_eq!( + split_repo("BaseIntelligence/prism-top-model").unwrap(), + ("BaseIntelligence", "prism-top-model") + ); + } +} diff --git a/crates/prism-registry/src/hooks.rs b/crates/prism-registry/src/hooks.rs index 06174077a..11689ec9e 100644 --- a/crates/prism-registry/src/hooks.rs +++ b/crates/prism-registry/src/hooks.rs @@ -83,9 +83,9 @@ pub async fn post_score_hooks( } } - // (3) Top-model publish on a new global best — only after secure receive - // (RECEIPT.json + sha256 match). Missing/bad receipt ⇒ no weight publish. - let Some(publisher) = publisher else { return }; + // (3) Top-model publish on a new global best — GitHub (optional) + HF + // (optional). GitHub weights require secure receive (RECEIPT.json); + // HF publishes sources regardless so the champion card stays current. let last = store.last_publication_bpb().await.unwrap_or(None); let global = store.best_scored_bpb().await.unwrap_or(None); let is_global_best = global.is_some_and(|g| bpb <= g); @@ -123,23 +123,54 @@ pub async fn post_score_hooks( metrics_json: row.metrics_json.clone(), checkpoint_path: ckpt, }; - match publisher.publish(&req).await { - Ok(sha) => { - info!(submission_id = %row.id, bpb, commit = %sha, "top model published to GitHub"); - let rec = TopModelPublication { - submission_id: row.id.clone(), - arch_id, - owner_hotkey: row.miner_hotkey.clone(), - bpb, - repo_path: TOPMODEL_REPO_PATH.to_owned(), - commit_sha: Some(sha), - }; - if let Err(e) = store.record_publication(&rec).await { - warn!(submission_id = %row.id, error = %e, "publication journal failed"); + let mut journaled = false; + if let Some(publisher) = publisher { + match publisher.publish(&req).await { + Ok(sha) => { + info!(submission_id = %row.id, bpb, commit = %sha, "top model published to GitHub"); + let rec = TopModelPublication { + submission_id: row.id.clone(), + arch_id: arch_id.clone(), + owner_hotkey: row.miner_hotkey.clone(), + bpb, + repo_path: TOPMODEL_REPO_PATH.to_owned(), + commit_sha: Some(sha), + }; + if let Err(e) = store.record_publication(&rec).await { + warn!(submission_id = %row.id, error = %e, "publication journal failed"); + } else { + journaled = true; + } + } + Err(e) => { + warn!(submission_id = %row.id, error = %e, "top-model publish failed (will retry on next best)"); } } - Err(e) => { - warn!(submission_id = %row.id, error = %e, "top-model publish failed (will retry on next best)"); + } + if let Some(hf) = crate::hf::HfTopModelPublisher::from_env() { + match hf.publish(&req).await { + Ok(oid) => { + if !journaled { + let rec = TopModelPublication { + submission_id: row.id.clone(), + arch_id, + owner_hotkey: row.miner_hotkey.clone(), + bpb, + repo_path: format!("hf:{}", hf.repo_id()), + commit_sha: Some(oid), + }; + if let Err(e) = store.record_publication(&rec).await { + warn!(submission_id = %row.id, error = %e, "hf publication journal failed"); + } + } + } + Err(e) => { + warn!( + submission_id = %row.id, + error = %e, + "top-model HuggingFace publish failed (will retry on next best)" + ); + } } } } diff --git a/crates/prism-registry/src/lib.rs b/crates/prism-registry/src/lib.rs index ef5bf0486..71302e03a 100644 --- a/crates/prism-registry/src/lib.rs +++ b/crates/prism-registry/src/lib.rs @@ -9,6 +9,9 @@ //! public `BaseIntelligence/prism` GitHub repo under `top-model/`, via a //! token read from a deploy secret file (`PRISM_TOPMODEL_GITHUB_TOKEN_FILE`; //! graceful no-op when absent). +//! - [`HfTopModelPublisher`] — same trigger, commits sources to HuggingFace +//! (`PRISM_TOPMODEL_HF_TOKEN_FILE`; default repo +//! `BaseIntelligence/prism-top-model`). #![forbid(unsafe_code)] #![allow(clippy::cast_precision_loss)] @@ -17,11 +20,13 @@ #![allow(clippy::module_name_repetitions)] mod competition; +mod hf; mod hooks; mod publish; mod weights; pub use competition::{apply_wta, competition_scores, OWNER_ARCH_CREDIT_ENABLED}; +pub use hf::HfTopModelPublisher; pub use hooks::post_score_hooks; pub use publish::{TopModelPublisher, TopModelRequest, TOPMODEL_REPO_PATH}; pub use weights::{CheckpointMeta, TOPMODEL_RELEASE_TAG}; diff --git a/deploy/compose/env-prod.yml b/deploy/compose/env-prod.yml index fccbd319c..254426998 100644 --- a/deploy/compose/env-prod.yml +++ b/deploy/compose/env-prod.yml @@ -56,6 +56,13 @@ services: prism-challenge: environment: PRISM_FORCE_SIM: "false" + # G1–G8 battery (two-phase train/eval). Default harness is v3; pin here + # so pods never silently fall back to BPB-only even on older images. + PRISM_FLOW: "v3" + # Full public pack (not tiny caps). Host path staged by + # deploy/scripts/prism-overnight-battery.sh / build_private_pack. + PRISM_EVAL_ASSETS_DIR: "/var/lib/prism/eval-assets" + PRISM_TEST_EVAL_CAPS: "0" # Recipe 2.0 AutoModel pin (stage with deploy/scripts/stage-automodel-pin.sh). # Fail-closed intake when unset/unmounted — miners see code=pin. PRISM_AUTOMODEL_PIN_DIR: "/var/lib/prism/automodel-pin" @@ -66,10 +73,15 @@ services: PRISM_MAX_CONCURRENT_EVALS: "8" # Emitter/gating/epoch-feed chain reads share the gateway failover list. BASE_CHAIN_ENDPOINTS: "wss://bittensor-finney.api.onfinality.io/public-ws,wss://entrypoint-finney.opentensor.ai:443" + # Top-model HuggingFace publish (optional; no-op without token file). + PRISM_TOPMODEL_HF_TOKEN_FILE: "/run/base/huggingface/token" + PRISM_TOPMODEL_HF_REPO: "BaseIntelligence/prism-top-model" volumes: - /var/lib/prism/automodel-pin:/var/lib/prism/automodel-pin:ro - /var/lib/prism/payer-vault:/var/lib/prism/payer-vault + - /var/lib/prism/eval-assets:/var/lib/prism/eval-assets:ro - ./deploy/secrets/prism/payer_vault_key:/run/secrets/prism_payer_vault_key:ro + - ./deploy/secrets/huggingface:/run/base/huggingface:ro design-challenge: environment: # Docker-only on prod. Never set BASE_ALLOW_HOST_SIM / DESIGN_FORCE_SIM here. diff --git a/deploy/compose/env-staging.yml b/deploy/compose/env-staging.yml index 25f660a71..2b13dc261 100644 --- a/deploy/compose/env-staging.yml +++ b/deploy/compose/env-staging.yml @@ -60,16 +60,21 @@ services: PRISM_FORCE_SIM: "true" PRISM_TEST_TRAIN_MINUTES: "15" PRISM_TEST_MAX_PARAMS: "2000000" + # v3 battery even under short-train knobs (tiny grids via default + # PRISM_TEST_* → tiny_caps; set CAPS=0 for full G1–G8 on real Lium). + PRISM_FLOW: "v3" # Same pin path as prod when testing live AutoModel intake on staging # (stage with deploy/scripts/stage-automodel-pin.sh). Sim/fixture pins # still work when miners submit automodel@fixture-v1. PRISM_AUTOMODEL_PIN_DIR: "/var/lib/prism/automodel-pin" PRISM_PAYER_VAULT_DIR: "/var/lib/prism/payer-vault" PRISM_PAYER_VAULT_KEY_FILE: "/run/secrets/prism_payer_vault_key" + PRISM_TOPMODEL_HF_TOKEN_FILE: "/run/base/huggingface/token" volumes: - /var/lib/prism/automodel-pin:/var/lib/prism/automodel-pin:ro - /var/lib/prism/payer-vault:/var/lib/prism/payer-vault - ./deploy/secrets/prism/payer_vault_key:/run/secrets/prism_payer_vault_key:ro + - ./deploy/secrets/huggingface:/run/base/huggingface:ro design-challenge: environment: BASE_CHAIN_ENDPOINT: "wss://test.chain.opentensor.ai:443" diff --git a/deploy/env/prism-challenge.env.example b/deploy/env/prism-challenge.env.example index fef9f86ad..b02e54180 100644 --- a/deploy/env/prism-challenge.env.example +++ b/deploy/env/prism-challenge.env.example @@ -29,3 +29,14 @@ BASE_NETUID=541 # publish + playground verify before use. # PRISM_ARTIFACT_DIR=/var/lib/prism/artifacts # PRISM_TOPMODEL_REQUIRE_WEIGHTS=1 + +# G1–G8 battery (harness defaults to v3). Stage the public pack under +# PRISM_EVAL_ASSETS_DIR (see deploy/scripts/prism-overnight-battery.sh). +# PRISM_FLOW=v3 +# PRISM_EVAL_ASSETS_DIR=/var/lib/prism/eval-assets +# PRISM_TEST_EVAL_CAPS=0 + +# Top-model HuggingFace publish (optional). Token file only — never paste +# the token into this env file. Missing/empty → HF publish no-ops. +# PRISM_TOPMODEL_HF_TOKEN_FILE=/run/base/huggingface/token +# PRISM_TOPMODEL_HF_REPO=BaseIntelligence/prism-top-model diff --git a/deploy/scripts/prism-overnight-battery.sh b/deploy/scripts/prism-overnight-battery.sh index d66feb182..3de616f8c 100755 --- a/deploy/scripts/prism-overnight-battery.sh +++ b/deploy/scripts/prism-overnight-battery.sh @@ -5,7 +5,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" EVIDENCE="${EVIDENCE:-/tmp/prism-overnight-$(date -u +%Y%m%dT%H%M%SZ)}" -PACK_DIR="${PRISM_EVAL_ASSETS_DIR:-/tmp/prism-eval-assets}" +PACK_DIR="${PRISM_EVAL_ASSETS_DIR:-/var/lib/prism/eval-assets}" ARTIFACT_DIR="${PRISM_ARTIFACT_DIR:-/tmp/prism-artifacts}" OVERRIDE="${PRISM_OVERNIGHT_OVERRIDE:-/tmp/prism-overnight-compose.override.yml}" PRISM_URL="${PRISM_URL:-http://127.0.0.1:28092}" diff --git a/deploy/scripts/remote-deploy.sh b/deploy/scripts/remote-deploy.sh index 5d5db7888..775aa4fb8 100755 --- a/deploy/scripts/remote-deploy.sh +++ b/deploy/scripts/remote-deploy.sh @@ -191,6 +191,7 @@ rsync -az --delete \ # absent; if a directory already poisoned the path, replace it with a file. ssh_h "mkdir -p '$REMOTE_DIR/deploy/env' '$REMOTE_DIR/deploy/secrets/lium' \ '$REMOTE_DIR/deploy/secrets/openrouter' '$REMOTE_DIR/deploy/secrets/design' \ + '$REMOTE_DIR/deploy/secrets/github' '$REMOTE_DIR/deploy/secrets/huggingface' \ '$REMOTE_DIR/deploy/secrets/wallets' \ && chmod 700 '$REMOTE_DIR/deploy/secrets' '$REMOTE_DIR/deploy/secrets/lium' \ && for f in api_key ssh_ed25519 ssh_ed25519.pub; do \ @@ -198,6 +199,8 @@ ssh_h "mkdir -p '$REMOTE_DIR/deploy/env' '$REMOTE_DIR/deploy/secrets/lium' \ done \ && [ -e '$REMOTE_DIR/deploy/secrets/openrouter/api_key' ] || : > '$REMOTE_DIR/deploy/secrets/openrouter/api_key' \ && [ -e '$REMOTE_DIR/deploy/secrets/design/annotator_tokens' ] || : > '$REMOTE_DIR/deploy/secrets/design/annotator_tokens' \ + && [ -e '$REMOTE_DIR/deploy/secrets/github/token' ] || : > '$REMOTE_DIR/deploy/secrets/github/token' \ + && [ -e '$REMOTE_DIR/deploy/secrets/huggingface/token' ] || : > '$REMOTE_DIR/deploy/secrets/huggingface/token' \ && for sk in prism_sk design_sk; do \ p='$REMOTE_DIR/deploy/secrets/'\$sk; \ if [ -d \"\$p\" ]; then rm -rf \"\$p\"; fi; \ @@ -207,9 +210,13 @@ ssh_h "mkdir -p '$REMOTE_DIR/deploy/env' '$REMOTE_DIR/deploy/secrets/lium' \ && chmod 400 '$REMOTE_DIR/deploy/secrets/lium/'* \ '$REMOTE_DIR/deploy/secrets/openrouter/api_key' \ '$REMOTE_DIR/deploy/secrets/design/annotator_tokens' \ + '$REMOTE_DIR/deploy/secrets/github/token' \ + '$REMOTE_DIR/deploy/secrets/huggingface/token' \ && chown -R 65532:65532 '$REMOTE_DIR/deploy/secrets/lium' \ '$REMOTE_DIR/deploy/secrets/openrouter' \ '$REMOTE_DIR/deploy/secrets/design' \ + '$REMOTE_DIR/deploy/secrets/github' \ + '$REMOTE_DIR/deploy/secrets/huggingface' \ && chmod -R a-w '$REMOTE_DIR/deploy/secrets/wallets' 2>/dev/null; \ chown -R 65532:65532 '$REMOTE_DIR/deploy/secrets/wallets' 2>/dev/null; true" diff --git a/deploy/secrets/README.md b/deploy/secrets/README.md index 2e01416b1..897232522 100644 --- a/deploy/secrets/README.md +++ b/deploy/secrets/README.md @@ -53,6 +53,19 @@ chmod 0400 deploy/secrets/design/annotator_tokens deploy/secrets/openrouter/api_ `PRISM_TOPMODEL_GITHUB_TOKEN_FILE` (`/run/base/github/token`); missing or empty file = top-model publish silently disabled. Mode **0400**, uid **65532** — never commit it. +- `huggingface/token` — prism-challenge HuggingFace top-model publisher: Hub + **write** token for `BaseIntelligence/prism-top-model` (override + `PRISM_TOPMODEL_HF_REPO`). Read via `PRISM_TOPMODEL_HF_TOKEN_FILE` + (`/run/base/huggingface/token`); missing or empty = HF publish no-ops. + Mode **0400**, uid **65532** — never commit it. + +```bash +mkdir -p deploy/secrets/huggingface +touch deploy/secrets/huggingface/token +chown 65532:65532 deploy/secrets/huggingface/token +chmod 0400 deploy/secrets/huggingface/token +``` + - `prism/admin_tokens` — one operator bearer per line for Prism `/v1/submissions/{id}/retry`, `POST /v1/admin/playground/complete`, `POST /v1/admin/gating/{hotkey}/reset`, and diff --git a/docker-compose.yml b/docker-compose.yml index ecb29da92..e5edf3f02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -213,10 +213,17 @@ services: # Top-model GitHub publish (BaseIntelligence/prism top-model/): no-op # when the token file is absent/empty. PRISM_TOPMODEL_GITHUB_TOKEN_FILE: /run/base/github/token + # Top-model HuggingFace publish (BaseIntelligence/prism-top-model): no-op + # when the token file is absent/empty. + PRISM_TOPMODEL_HF_TOKEN_FILE: /run/base/huggingface/token + PRISM_TOPMODEL_HF_REPO: "${PRISM_TOPMODEL_HF_REPO:-BaseIntelligence/prism-top-model}" # Require harvested checkpoint for top-model journal (set 0 for source-only). PRISM_TOPMODEL_REQUIRE_WEIGHTS: "${PRISM_TOPMODEL_REQUIRE_WEIGHTS:-1}" # Parked checkpoints harvested from Lium pods (master-local). PRISM_ARTIFACT_DIR: /var/lib/prism/artifacts + # G1–G8 eval assets pack (optional; harness falls back to public_dev). + PRISM_EVAL_ASSETS_DIR: "${PRISM_EVAL_ASSETS_DIR:-}" + PRISM_FLOW: "${PRISM_FLOW:-v3}" # Recipe 2.0 AutoModel pin checkout (deploy/scripts/stage-automodel-pin.sh). # Required for live AutoModel intake; unset → pin unavailable (fail-closed). PRISM_AUTOMODEL_PIN_DIR: "${PRISM_AUTOMODEL_PIN_DIR:-}" @@ -234,6 +241,7 @@ services: - ./deploy/secrets/lium:/run/base/lium:ro - ./deploy/secrets/openrouter:/run/base/openrouter:ro - ./deploy/secrets/github:/run/base/github:ro + - ./deploy/secrets/huggingface:/run/base/huggingface:ro - ./deploy/secrets/prism:/run/base/prism:ro - prism-artifacts:/var/lib/prism/artifacts expose: diff --git a/docs/PRISM.md b/docs/PRISM.md index dfe61d22a..6d4352095 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -56,7 +56,7 @@ hypertraining B300 tournament code. stateDiagram-v2 [*] --> Queued: POST /v1/submissions Queued --> Rejected: pre-pod screens (copy gate / static cheat / similarity) - Queued --> Provisioning: worker claims + pre-pod screens pass + Queued --> Provisioning: worker claims + pre-pod screens + LLM/agentic pass Provisioning --> Running: pod SSH + harness up Running --> Reviewing: METRICS_JSON collected Reviewing --> AgenticReview: quality + post-pod agentic @@ -229,8 +229,10 @@ publishes `architecture.py` + `training.py` + `METRICS.json` + `ARTIFACT.json` + a `README.md` block to the public [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) repo under `top-model/` via the GitHub contents API; large checkpoints upload as -a mutable Release tag `prism-top-model`. The publication is journaled -(`prism_topmodel_publication`). Token: +a mutable Release tag `prism-top-model`. The same trigger also commits +sources to HuggingFace (`PRISM_TOPMODEL_HF_TOKEN_FILE`, default repo +`BaseIntelligence/prism-top-model`) when that token file is present. The +publication is journaled (`prism_topmodel_publication`). GitHub token: `PRISM_TOPMODEL_GITHUB_TOKEN_FILE` (`deploy/secrets/github/token`); absent/empty → publish no-op. With `PRISM_TOPMODEL_REQUIRE_WEIGHTS=1` (default), a missing/invalid receipt fails the publish (no journal). @@ -373,7 +375,8 @@ still recorded on every v3 run (it is a G1 input and the shadow score). ## Agentic anti-cheat + AST + metrics gate Before any pod rent, **pre-pod screens** (no GPU, no private eval assets) run -in order and terminal-reject with `Score(0)` on hit: +in order and terminal-reject with `Score(0)` on hit (OpenRouter / agentic +infra errors also fail closed here — they must **never** rent a pod): 1. **Pre-LLM copy gate** — candidate `architecture.py` vs **champions** (current top + historical Score>0 ex-tops) from **other miners** (byte hash @@ -393,13 +396,18 @@ in order and terminal-reject with `Score(0)` on hit: Below-threshold `Suspicious` (e.g. 0.7) does not wipe. Parsers coerce verdicts whose evidence is only standard LM components (RMSNorm / RoPE / SwiGLU / LayerNorm / gated or parallel residual, …). - -After measure, the LLM quality review and the shared `challenge-agentic` loop -inspect sources + metrics/receipt with read-only tools (`list_dir`, -`read_file`, `ast_summary`, `ast_diff_nearest`, `read_metrics`) against an -**architecture-only** corpus of baseline + champions. Final judge is the -mandatory `submit_verdict` function-call. Agentic must not treat generic -modern-LM components as plagiarism; AST bands (`≥8500` suspicious / +4. **LLM quality review** (`prism-review`) — audit-only for the bpb score; + infra failure fails closed (no rent). +5. **Agentic anti-cheat (sources)** — shared `challenge-agentic` loop on + architecture / training / tree only (`cheat` / `suspicious` → `Score(0)`, + no rent). + +After measure, a second **metrics-aware** agentic pass inspects sources + +metrics/receipt with read-only tools (`list_dir`, `read_file`, `ast_summary`, +`ast_diff_nearest`, `read_metrics`) against an **architecture-only** corpus +of baseline + champions (catches `inconsistent_metrics` / eval forge). Final +judge is the mandatory `submit_verdict` function-call. Agentic must not treat +generic modern-LM components as plagiarism; AST bands (`≥8500` suspicious / `≥9500` cheat) remain the structural copy thresholds. | Verdict | Leaf effect | diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index 072e96b30..f510f200d 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -3,7 +3,7 @@ # Prism challenge — HTTP AutoModel patch submit **challenge_id:** `prism` -**scoring_version:** `2` live (bpb-only; LLM review is an anti-cheat gate, not a grader). **v3 (opt-in, shadow-by-default):** composite scoring runs alongside — your run is also measured on the G1–G8 battery; see *v3 scoring* below. +**scoring_version:** `2` live (bpb leaf; LLM review is an anti-cheat gate, not a grader). **v3 harness (default):** every scored run executes the **G1–G8 battery** (leaf score stays bpb while `PRISM_SCORING_MODE=shadow`); see *v3 scoring* below. **recipe_version:** `2.0.0` (pinned [NeMo AutoModel](https://github.com/NVIDIA-NeMo/Automodel) base + miner unified diff; legacy 1.x layouts rejected on live) **Path:** HTTP only — **no Phala/CVM** @@ -208,7 +208,8 @@ a better valid score supersedes them (WTA still collapses to one leaf winner). The global-best model (sources + `ARTIFACT.json` / checkpoint release) is published to [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) -`top-model/`. See [`PRISM.md`](../PRISM.md). +`top-model/` and (when configured) a HuggingFace model repo +`BaseIntelligence/prism-top-model`. See [`PRISM.md`](../PRISM.md). ## v3 scoring (shadow-by-default)