diff --git a/Cargo.lock b/Cargo.lock index af9453d74..875fedc54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3896,6 +3896,7 @@ dependencies = [ "prism-challenge-task", "prism-registry", "prism-store", + "serde_json", "thiserror 2.0.19", "tokio", "tracing", diff --git a/crates/prism-challenge/tests/arch_competition.rs b/crates/prism-challenge/tests/arch_competition.rs index 77551e65b..dbafe4406 100644 --- a/crates/prism-challenge/tests/arch_competition.rs +++ b/crates/prism-challenge/tests/arch_competition.rs @@ -119,7 +119,7 @@ fn row( pod_id: None, pod_provider: None, receipt: None, - metrics_json: None, + metrics_json: Some(serde_json::json!({"recipe": "2.0.0"})), bpb: None, arch_id: arch_id.map(str::to_owned), review: None, diff --git a/crates/prism-challenge/tests/e2e_orchestrate_sim.rs b/crates/prism-challenge/tests/e2e_orchestrate_sim.rs index aaa420142..4a00c1d55 100644 --- a/crates/prism-challenge/tests/e2e_orchestrate_sim.rs +++ b/crates/prism-challenge/tests/e2e_orchestrate_sim.rs @@ -290,7 +290,7 @@ async fn emit_and_submit_covers_expected_set() { pod_id: None, pod_provider: None, receipt: None, - metrics_json: None, + metrics_json: Some(serde_json::json!({"recipe": "2.0.0"})), bpb: Some(2.0), arch_id: None, review: None, diff --git a/crates/prism-emit/Cargo.toml b/crates/prism-emit/Cargo.toml index ce614d7bb..92323c527 100644 --- a/crates/prism-emit/Cargo.toml +++ b/crates/prism-emit/Cargo.toml @@ -20,6 +20,7 @@ thiserror = "2" tracing = "0.1" [dev-dependencies] +serde_json = "1" tokio = { version = "1", features = ["macros", "rt-multi-thread"] } [lints] diff --git a/crates/prism-emit/src/lib.rs b/crates/prism-emit/src/lib.rs index 95e35af9c..8a08e685b 100644 --- a/crates/prism-emit/src/lib.rs +++ b/crates/prism-emit/src/lib.rs @@ -318,6 +318,7 @@ mod tests { miner_hotkey: hex::encode(hk), arch_id: arch.map(str::to_owned), final_score: FinalScore::Score(score), + weight_eligible: true, } } diff --git a/crates/prism-emit/tests/epoch_semantics.rs b/crates/prism-emit/tests/epoch_semantics.rs index a3fb9bf48..7a4e8528e 100644 --- a/crates/prism-emit/tests/epoch_semantics.rs +++ b/crates/prism-emit/tests/epoch_semantics.rs @@ -75,7 +75,8 @@ fn scored_row(id: &str, hotkey: &str, accept_epoch: u64, score: FinalScore) -> S pod_id: None, pod_provider: None, receipt: None, - metrics_json: None, + // Emission tests exercise AutoModel / recipe 2.0 weight eligibility. + metrics_json: Some(serde_json::json!({"recipe": "2.0.0"})), bpb: Some(2.0), arch_id: None, review: None, diff --git a/crates/prism-registry/src/competition.rs b/crates/prism-registry/src/competition.rs index 618ae7466..94301689b 100644 --- a/crates/prism-registry/src/competition.rs +++ b/crates/prism-registry/src/competition.rs @@ -15,6 +15,10 @@ //! - **WTA leaf emission**: [`apply_wta`] keeps a single positive `Score` //! (argmax; lexicographically smallest hotkey on ties). Prism's emission //! share goes to that submitter. +//! - **Recipe 2.0 / AutoModel only**: rows with `weight_eligible == false` +//! (legacy 1.x) contribute `Score(0)` only — never win WTA. If every +//! positive score is ineligible, emission fail-closes to an all-zero / +//! burn projection (no 1.x winner). use std::collections::BTreeMap; @@ -35,6 +39,8 @@ pub const OWNER_ARCH_CREDIT_ENABLED: bool = false; /// Compute per-hotkey emission for one epoch. /// /// `arch_owners` is ignored while [`OWNER_ARCH_CREDIT_ENABLED`] is `false`. +/// Legacy (non-[`EpochScoreRow::weight_eligible`]) positive scores are +/// treated as `Score(0)` so they cannot win WTA or carry emission. #[must_use] pub fn competition_scores( rows: &[EpochScoreRow], @@ -50,12 +56,14 @@ pub fn competition_scores( for r in rows { match &r.final_score { FinalScore::Score(v) => { + // Fail-closed: legacy 1.x never receives emission credit. + let v = if r.weight_eligible { *v } else { 0 }; let e = own.entry(r.miner_hotkey.clone()).or_insert(0); - *e = (*e).max(*v); - if OWNER_ARCH_CREDIT_ENABLED { + *e = (*e).max(v); + if OWNER_ARCH_CREDIT_ENABLED && r.weight_eligible { if let Some(a) = r.arch_id.as_deref() { let e = arch_best.entry(a).or_insert(0); - *e = (*e).max(*v); + *e = (*e).max(v); } } } @@ -138,6 +146,16 @@ mod tests { miner_hotkey: hk.into(), arch_id: arch.map(str::to_owned), final_score: FinalScore::Score(score), + weight_eligible: true, + } + } + + fn legacy_row(hk: &str, score: u64) -> EpochScoreRow { + EpochScoreRow { + miner_hotkey: hk.into(), + arch_id: None, + final_score: FinalScore::Score(score), + weight_eligible: false, } } @@ -205,6 +223,7 @@ mod tests { miner_hotkey: "cc".into(), arch_id: None, final_score: FinalScore::NoScore(6), + weight_eligible: true, }, row("aa", None, 100_000), ]; @@ -246,4 +265,28 @@ mod tests { assert_eq!(wta.get("aa"), Some(&FinalScore::Score(900_000))); assert_eq!(wta.get("bb"), Some(&FinalScore::Score(0))); } + + #[test] + fn legacy_positive_scores_cannot_win_wta() { + // Legacy 1.x tops the lattice but is weight-ineligible; AutoModel + // runner with a lower score still wins. Fail-closed vs 1.x emission. + let rows = vec![legacy_row("legacy", 900_000), row("auto", None, 100_000)]; + let out = competition_scores(&rows, &BTreeMap::new()); + assert_eq!(out.get("legacy"), Some(&FinalScore::Score(0))); + assert_eq!(out.get("auto"), Some(&FinalScore::Score(100_000))); + let wta = apply_wta(out); + assert_eq!(wta.get("auto"), Some(&FinalScore::Score(100_000))); + assert_eq!(wta.get("legacy"), Some(&FinalScore::Score(0))); + } + + #[test] + fn only_legacy_tops_fail_closed_to_burn() { + // No AutoModel-eligible positive → no WTA winner (burn / hold). + let rows = vec![legacy_row("aa", 900_000), legacy_row("bb", 800_000)]; + let out = competition_scores(&rows, &BTreeMap::new()); + assert_eq!(out.get("aa"), Some(&FinalScore::Score(0))); + assert_eq!(out.get("bb"), Some(&FinalScore::Score(0))); + let wta = apply_wta(out); + assert!(wta.values().all(|s| matches!(s, FinalScore::Score(0)))); + } } diff --git a/crates/prism-registry/src/hooks.rs b/crates/prism-registry/src/hooks.rs index 5e32cf13e..48d09411e 100644 --- a/crates/prism-registry/src/hooks.rs +++ b/crates/prism-registry/src/hooks.rs @@ -86,6 +86,15 @@ pub async fn post_score_hooks( // (3) Top-model publish on a new global best — GitHub (optional) + HF // (optional). GitHub weights require secure receive (RECEIPT.json); // HF publishes sources regardless so the champion card stays current. + // Recipe 2.0 / AutoModel only — legacy 1.x never becomes the published + // top-model champion (historical FE rows stay in Postgres). + if !row.weight_eligible() { + info!( + submission_id = %row.id, + "top-model: skip legacy / non-AutoModel row (weight-ineligible)" + ); + return; + } let last = store.last_publication_bpb().await.unwrap_or(None); let global = store.best_scored_bpb().await.unwrap_or(None); let is_global_best = global.is_some_and(|g| bpb <= g); diff --git a/crates/prism-store-types/src/lib.rs b/crates/prism-store-types/src/lib.rs index c645efe3d..86bf0bea8 100644 --- a/crates/prism-store-types/src/lib.rs +++ b/crates/prism-store-types/src/lib.rs @@ -13,6 +13,6 @@ mod types; pub use types::{ - ArchitectureRecord, EpochScoreRow, FinalScore, PublishArchOutcome, Stage, StageEvent, - StatePatch, StoreError, SubmissionId, SubmissionState, TopModelPublication, + submission_weight_eligible, ArchitectureRecord, EpochScoreRow, FinalScore, PublishArchOutcome, + Stage, StageEvent, StatePatch, StoreError, SubmissionId, SubmissionState, TopModelPublication, }; diff --git a/crates/prism-store-types/src/types.rs b/crates/prism-store-types/src/types.rs index 76a9e6584..b838abf9e 100644 --- a/crates/prism-store-types/src/types.rs +++ b/crates/prism-store-types/src/types.rs @@ -225,6 +225,81 @@ pub enum PublishArchOutcome { Duplicate(String), } +/// Packed-tree / USTAR path marker for recipe 2.0 `AutoModel` patch artifacts. +const AUTOMODEL_PATCH_MARKER: &[u8] = b".prism/automodel.patch"; + +/// Fail-closed emission eligibility: recipe **2.0 / `AutoModel`** only. +/// +/// Legacy 1.x (`architecture.py` / `training.py` era) may remain visible on +/// the site FE and in the DB, but must not win WTA leaf emission or displace +/// an `AutoModel` champion via score carry. Unknown / missing signals are +/// **not** eligible. +#[must_use] +pub fn submission_weight_eligible( + metrics_json: Option<&serde_json::Value>, + tree_blob: Option<&[u8]>, +) -> bool { + if let Some(m) = metrics_json { + if recipe_major_ge2(m) || automodel_pin_signal(m) { + return true; + } + } + tree_blob.is_some_and(tree_blob_has_automodel_patch) +} + +fn recipe_major_ge2(m: &serde_json::Value) -> bool { + for path in ["recipe", "/pod_manifest/recipe"] { + let raw = if path.starts_with('/') { + m.pointer(path).and_then(|v| v.as_str()) + } else { + m.get(path).and_then(|v| v.as_str()) + }; + if let Some(s) = raw { + if let Some(maj) = s + .trim() + .split('.') + .next() + .and_then(|p| p.parse::().ok()) + { + if maj >= 2 { + return true; + } + } + } + } + false +} + +fn automodel_pin_signal(m: &serde_json::Value) -> bool { + for path in [ + "/pod_manifest/automodel_base", + "/pod_manifest/pin_id", + "/pin_id", + "/automodel_base", + ] { + if m.pointer(path) + .and_then(|v| v.as_str()) + .is_some_and(|s| s.trim().starts_with("automodel@")) + { + return true; + } + } + false +} + +fn tree_blob_has_automodel_patch(blob: &[u8]) -> bool { + blob.windows(AUTOMODEL_PATCH_MARKER.len()) + .any(|w| w == AUTOMODEL_PATCH_MARKER) +} + +impl SubmissionState { + /// Whether this row may receive on-chain Prism weight (`AutoModel` 2.0). + #[must_use] + pub fn weight_eligible(&self) -> bool { + submission_weight_eligible(self.metrics_json.as_ref(), self.tree_blob.as_deref()) + } +} + /// One scored row inside an emission batch (competition scoring input). /// Batches are epoch-close assignments (see [`PrismStore::assign_emit_batch`]), /// not acceptance-epoch lookups: a row's acceptance `epoch` is intake metadata. @@ -236,6 +311,52 @@ pub struct EpochScoreRow { pub arch_id: Option, /// Final lattice score (or absence). pub final_score: FinalScore, + /// Recipe 2.0 / `AutoModel` only — legacy positive scores must not win WTA. + pub weight_eligible: bool, +} + +#[cfg(test)] +mod weight_eligible_tests { + #![allow(clippy::unwrap_used)] + use super::*; + use serde_json::json; + + #[test] + fn recipe_2_metrics_eligible() { + assert!(submission_weight_eligible( + Some(&json!({"recipe": "2.0.0"})), + None + )); + } + + #[test] + fn automodel_pin_eligible() { + assert!(submission_weight_eligible( + Some(&json!({"pod_manifest": {"automodel_base": "automodel@v0.5.0"}})), + None + )); + } + + #[test] + fn tree_patch_marker_eligible() { + let mut blob = b"ustar....".to_vec(); + blob.extend_from_slice(b".prism/automodel.patch"); + blob.extend_from_slice(b"....tail"); + assert!(submission_weight_eligible(None, Some(&blob))); + } + + #[test] + fn legacy_and_unknown_fail_closed() { + assert!(!submission_weight_eligible( + Some(&json!({"recipe": "1.4.0"})), + None + )); + assert!(!submission_weight_eligible(None, None)); + assert!(!submission_weight_eligible( + Some(&json!({"bpb": 4.2})), + Some(b"no-patch-here") + )); + } } /// Top-model publication journal row (migration 0010). diff --git a/crates/prism-store/src/arch.rs b/crates/prism-store/src/arch.rs index 4712bd278..6c746983f 100644 --- a/crates/prism-store/src/arch.rs +++ b/crates/prism-store/src/arch.rs @@ -221,9 +221,12 @@ pub(crate) async fn last_publication( } pub(crate) async fn best_scored_bpb(pool: &PgPool) -> Result, StoreError> { - let row: (Option,) = sqlx::query_as( - "SELECT MIN(bpb) FROM prism_submission WHERE kind = 'score' AND score > 0 AND bpb IS NOT NULL", - ) + let row: (Option,) = sqlx::query_as(&format!( + "SELECT MIN(bpb) FROM prism_submission \ + WHERE kind = 'score' AND score > 0 AND bpb IS NOT NULL \ + AND {}", + crate::emit::WEIGHT_ELIGIBLE_SQL + )) .fetch_one(pool) .await .map_err(backend)?; diff --git a/crates/prism-store/src/emit.rs b/crates/prism-store/src/emit.rs index bffcd8a97..17b000ac3 100644 --- a/crates/prism-store/src/emit.rs +++ b/crates/prism-store/src/emit.rs @@ -23,11 +23,26 @@ fn backend(e: sqlx::Error) -> StoreError { StoreError::Backend(e.to_string()) } -type EmitSqlRow = (String, Option, String, Option, Option); +/// SQL predicate: recipe 2.0 / `AutoModel` pin / `.prism/automodel.patch` in tree. +pub(crate) const WEIGHT_ELIGIBLE_SQL: &str = "(\ +COALESCE(metrics_json->>'recipe','') LIKE '2.%' \ +OR COALESCE(metrics_json#>>'{pod_manifest,automodel_base}','') LIKE 'automodel@%' \ +OR COALESCE(metrics_json#>>'{pod_manifest,pin_id}','') LIKE 'automodel@%' \ +OR (tree_blob IS NOT NULL AND position('\\x2e707269736d2f6175746f6d6f64656c2e7061746368'::bytea in tree_blob) > 0)\ +)"; + +type EmitSqlRow = ( + String, + Option, + String, + Option, + Option, + bool, +); fn rows_to_epoch(rows: Vec) -> Vec { rows.into_iter() - .filter_map(|(hk, arch_id, kind, score, absence)| { + .filter_map(|(hk, arch_id, kind, score, absence, weight_eligible)| { let final_score = match kind.as_str() { "score" => score.map(|s| FinalScore::Score(s.cast_unsigned())), "no_score" => absence.map(|a| FinalScore::NoScore(u8::try_from(a).unwrap_or(0))), @@ -37,6 +52,7 @@ fn rows_to_epoch(rows: Vec) -> Vec { miner_hotkey: hk, arch_id, final_score, + weight_eligible, }) }) .collect() @@ -50,11 +66,12 @@ pub(crate) async fn assign_emit_batch( netuid: i32, epoch: i64, ) -> Result, StoreError> { - let rows: Vec = sqlx::query_as( + let rows: Vec = sqlx::query_as(&format!( "UPDATE prism_submission SET emitted_epoch = $2, updated_at = now() \ WHERE netuid = $1 AND kind IS NOT NULL AND emitted_epoch IS NULL \ - RETURNING miner_hotkey, arch_id, kind, score, absence_reason", - ) + RETURNING miner_hotkey, arch_id, kind, score, absence_reason, \ + {WEIGHT_ELIGIBLE_SQL} AS weight_eligible" + )) .bind(netuid) .bind(epoch) .fetch_all(pool) @@ -69,11 +86,12 @@ pub(crate) async fn emit_batch( netuid: i32, epoch: i64, ) -> Result, StoreError> { - let rows: Vec = sqlx::query_as( - "SELECT miner_hotkey, arch_id, kind, score, absence_reason \ + let rows: Vec = sqlx::query_as(&format!( + "SELECT miner_hotkey, arch_id, kind, score, absence_reason, \ + {WEIGHT_ELIGIBLE_SQL} AS weight_eligible \ FROM prism_submission \ - WHERE netuid = $1 AND emitted_epoch = $2 AND kind IS NOT NULL", - ) + WHERE netuid = $1 AND emitted_epoch = $2 AND kind IS NOT NULL" + )) .bind(netuid) .bind(epoch) .fetch_all(pool) @@ -84,19 +102,20 @@ pub(crate) async fn emit_batch( /// Positive lattice scores still eligible for epoch-close competition carry. /// -/// `Score(0)` rejects and `NoScore` absences are excluded — they must not -/// displace a prior valid winner when an epoch's fresh outbox is empty or -/// burn-only. Competition aggregation takes `max` over the union of the -/// fresh batch and this set, so a better later score supersedes naturally. +/// `Score(0)` rejects, `NoScore` absences, and legacy 1.x positives are +/// excluded. Competition aggregation takes `max` over the union of the fresh +/// batch and this set, so a better later score supersedes naturally. pub(crate) async fn active_score_rows( pool: &PgPool, netuid: i32, ) -> Result, StoreError> { - let rows: Vec = sqlx::query_as( - "SELECT miner_hotkey, arch_id, kind, score, absence_reason \ + let rows: Vec = sqlx::query_as(&format!( + "SELECT miner_hotkey, arch_id, kind, score, absence_reason, \ + {WEIGHT_ELIGIBLE_SQL} AS weight_eligible \ FROM prism_submission \ - WHERE netuid = $1 AND kind = 'score' AND score > 0", - ) + WHERE netuid = $1 AND kind = 'score' AND score > 0 \ + AND {WEIGHT_ELIGIBLE_SQL}" + )) .bind(netuid) .fetch_all(pool) .await diff --git a/crates/prism-store/src/lib.rs b/crates/prism-store/src/lib.rs index 355ad434e..e00313da9 100644 --- a/crates/prism-store/src/lib.rs +++ b/crates/prism-store/src/lib.rs @@ -22,6 +22,6 @@ pub use store::{MemoryPrismStore, PrismStore}; // The data contract lives in `prism-store-types` (per-crate LOC cap); it is // re-exported wholesale so `prism_store::…` stays the single import path. pub use prism_store_types::{ - ArchitectureRecord, EpochScoreRow, FinalScore, PublishArchOutcome, Stage, StageEvent, - StatePatch, StoreError, SubmissionId, SubmissionState, TopModelPublication, + submission_weight_eligible, ArchitectureRecord, EpochScoreRow, FinalScore, PublishArchOutcome, + Stage, StageEvent, StatePatch, StoreError, SubmissionId, SubmissionState, TopModelPublication, }; diff --git a/crates/prism-store/src/store.rs b/crates/prism-store/src/store.rs index 535a8fd2c..b2007cc13 100644 --- a/crates/prism-store/src/store.rs +++ b/crates/prism-store/src/store.rs @@ -424,6 +424,7 @@ impl PrismStore for MemoryPrismStore { miner_hotkey: r.miner_hotkey.clone(), arch_id: r.arch_id.clone(), final_score: r.final_score.clone().unwrap_or(FinalScore::Score(0)), + weight_eligible: r.weight_eligible(), }); } } @@ -448,6 +449,7 @@ impl PrismStore for MemoryPrismStore { miner_hotkey: r.miner_hotkey.clone(), arch_id: r.arch_id.clone(), final_score: r.final_score.clone().unwrap_or(FinalScore::Score(0)), + weight_eligible: r.weight_eligible(), }) .collect()) } @@ -459,12 +461,13 @@ impl PrismStore for MemoryPrismStore { .map_err(|_| StoreError::Backend("poison".into()))?; Ok(rows .iter() - .filter(|r| r.netuid == netuid) + .filter(|r| r.netuid == netuid && r.weight_eligible()) .filter_map(|r| match &r.final_score { Some(FinalScore::Score(v)) if *v > 0 => Some(EpochScoreRow { miner_hotkey: r.miner_hotkey.clone(), arch_id: r.arch_id.clone(), final_score: FinalScore::Score(*v), + weight_eligible: true, }), _ => None, }) @@ -608,6 +611,7 @@ impl PrismStore for MemoryPrismStore { .lock() .map_err(|_| StoreError::Backend("poison".into()))? .iter() + .filter(|r| r.weight_eligible()) .filter(|r| matches!(r.final_score, Some(FinalScore::Score(v)) if v > 0)) .filter_map(|r| r.bpb) .min_by(f64::total_cmp)) @@ -677,7 +681,8 @@ mod tests { pod_id: None, pod_provider: None, receipt: None, - metrics_json: None, + // Default test rows are recipe 2.0 eligible (emission carry / WTA). + metrics_json: Some(serde_json::json!({"recipe": "2.0.0"})), bpb: None, arch_id: None, review: None, diff --git a/docs/PRISM.md b/docs/PRISM.md index e7c33e145..7bf275981 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -194,9 +194,16 @@ lands in, not the leaf format or the math).** Per emitted epoch set: - *WTA emission*: argmax over positive per-hotkey credits → one Score leaf; Prism's emission share (50% of the subnet) goes entirely to that **submitter** (best BPB → that submission's miner UID). +- *Recipe 2.0 / AutoModel weight eligibility* (fail-closed): only submissions + with recipe major ≥ 2, an `automodel@…` pin signal, or a packed tree + containing `.prism/automodel.patch` may carry into the competition set or + win WTA. Legacy 1.x positives remain in Postgres / site FE history but are + treated as `Score(0)` for emission; if every positive score is ineligible, + the epoch projects all-zero (burn / hold) — never emit Prism share to 1.x. **Top-model publish + secure receive.** The master tracks the global best -bpb across all scored submissions. After a successful Lium eval it +bpb across **weight-eligible** (recipe 2.0 / AutoModel) scored submissions. +After a successful Lium eval it **pulls** `checkpoint.pt` from the pod over SSH (master-initiated; the pod never pushes) and stages it through the secure receive hook into `$PRISM_ARTIFACT_DIR//` **before** terminate. Staging