diff --git a/Cargo.lock b/Cargo.lock index da7e9895c..e5c7452aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5203,6 +5203,7 @@ dependencies = [ "serde", "serde_json", "site-data", + "site-prism", "site-types", "tokio", "tower", @@ -5228,6 +5229,15 @@ dependencies = [ "wiremock", ] +[[package]] +name = "site-prism" +version = "0.1.0" +dependencies = [ + "serde_json", + "site-data", + "site-types", +] + [[package]] name = "site-types" version = "0.1.0" diff --git a/crates/site-api/Cargo.toml b/crates/site-api/Cargo.toml index 6aba11db2..9582b6af0 100644 --- a/crates/site-api/Cargo.toml +++ b/crates/site-api/Cargo.toml @@ -18,6 +18,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls" serde = { version = "1", features = ["derive"] } serde_json = "1" site-data = { path = "../site-data" } +site-prism = { path = "../site-prism" } site-types = { path = "../site-types" } tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } futures = { version = "0.3", default-features = false, features = ["std"] } diff --git a/crates/site-api/src/handlers.rs b/crates/site-api/src/handlers.rs index 7a24d4194..278bb823f 100644 --- a/crates/site-api/src/handlers.rs +++ b/crates/site-api/src/handlers.rs @@ -11,11 +11,6 @@ use futures::future::join_all; use serde::Deserialize; use serde_json::{json, Value}; -use crate::prism_enrich::{ - enrich_leaderboard_row_from_detail_with_zone, enrich_submission_from_detail_with_zone, - map_benchmarks, pin_id_from_payload, prism_reference_baselines, - prism_submission_detail_with_zone, -}; use crate::state::SiteState; use crate::upstream::{self, DESIGN, PRISM}; use site_data::map::{ @@ -25,6 +20,11 @@ use site_data::map::{ prism_bpb_leaderboard, prism_submission, prism_telemetry, prism_window, submission_matches_query, uid_index_from_hotkeys, }; +use site_prism::{ + enrich_leaderboard_row_from_detail_with_zone, enrich_submission_from_detail_with_zone, + infer_recipe_era_with_live, map_benchmarks, payload_is_v21_contest, pin_id_from_payload, + prism_reference_baselines, prism_submission_detail_with_zone, +}; use site_types::coding_arena; use site_types::page_slice; use site_types::{ @@ -454,10 +454,21 @@ async fn prism_leaderboard_json( .collect(); ids.truncate(PRISM_CHAMPION_DETAIL_FANOUT); let details = fetch_prism_details(st, &ids).await; + let live_recipe = fetch_prism_recipe(st) + .await + .as_ref() + .and_then(|r| r.get("version")) + .and_then(Value::as_str) + .map(str::to_owned); + let live = live_recipe.as_deref(); for row in &mut board { if let Some(id) = row.submission_id.as_ref() { if let Some(fan) = details.get(id) { enrich_leaderboard_row_from_detail_with_zone(row, &fan.detail, fan.zone_a.as_ref()); + row.recipe_era = Some(infer_recipe_era_with_live(&fan.detail, live)); + if row.recipe_era == Some(RecipeEra::V21) { + row.run.weight_eligible = Some(true); + } } } if row.recipe_era.is_none() { @@ -465,6 +476,8 @@ async fn prism_leaderboard_json( row.recipe_era = Some(RecipeEra::Legacy); } } + // Public board is the live v2.1 contest only. Empty + burn is honest. + board.retain(|r| r.recipe_era == Some(RecipeEra::V21)); decorate_leaderboard(st, &mut board); if let Some(needle) = q.filter(|s| !s.trim().is_empty()) { board.retain(|r| leaderboard_matches_query(r, needle)); @@ -478,6 +491,10 @@ async fn prism_leaderboard_json( "pageCount": page_out.page_count, "epoch": epoch, "metric": "score", + "competitionId": "prism-v2.1", + "scoringGeneration": 21, + "recipeVersion": live_recipe.as_deref().unwrap_or("2.1.0"), + "waitingForFirst": page_out.total == 0, "updatedAt": now_iso(), }) } @@ -597,9 +614,20 @@ async fn get_submissions( let mut ids: Vec = items.iter().map(|s| s.id.clone()).collect(); ids.truncate(PRISM_CHAMPION_DETAIL_FANOUT); let details = fetch_prism_details(&st, &ids).await; + let live_recipe = fetch_prism_recipe(&st) + .await + .as_ref() + .and_then(|r| r.get("version")) + .and_then(Value::as_str) + .map(str::to_owned); + let live = live_recipe.as_deref(); for item in &mut items { if let Some(fan) = details.get(&item.id) { enrich_submission_from_detail_with_zone(item, &fan.detail, fan.zone_a.as_ref()); + item.recipe_era = Some(infer_recipe_era_with_live(&fan.detail, live)); + if item.recipe_era == Some(RecipeEra::V21) { + item.run.weight_eligible = Some(true); + } } else if item.recipe_era.is_none() { // Pre-2.0 / unknown → legacy so era tabs are not empty. item.recipe_era = Some(RecipeEra::Legacy); @@ -739,19 +767,39 @@ async fn get_prism_window(State(st): State) -> impl IntoResponse { ids.sort_by(|a, b| a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)); ids.truncate(PRISM_WINDOW_TELEMETRY_FANOUT); let mut telemetry = HashMap::new(); + let mut v21_ids = std::collections::HashSet::new(); + for row in &rows { + if payload_is_v21_contest(row) { + if let Some(id) = row.get("id").and_then(Value::as_str) { + v21_ids.insert(id.to_owned()); + } + } + } for (_, id) in ids { if let Some(detail) = upstream::get_json_opt(&st, PRISM, &format!("/v1/submissions/{id}")).await { + if payload_is_v21_contest(&detail) { + v21_ids.insert(id.clone()); + } if let Some(t) = prism_telemetry(&detail) { telemetry.insert(id, t); } } } + let v21_rows: Vec = rows + .into_iter() + .filter(|r| { + r.get("id") + .and_then(Value::as_str) + .is_some_and(|id| v21_ids.contains(id)) + || payload_is_v21_contest(r) + }) + .collect(); Json(prism_window( recipe.as_ref(), status.as_ref(), - &rows, + &v21_rows, &telemetry, )) } @@ -1033,6 +1081,9 @@ mod tests { mount_design_site_mocks(design).await; mount_prism_list_mocks(prism).await; mount_prism_detail_mock(prism).await; + // Re-mount the list after per-id stubs so later GETs (`?limit=`) still + // hit the collection (wiremock first-match can steal `/v1/submissions*`). + mount_prism_list_collection(prism).await; } async fn mount_design_site_mocks(design: &MockServer) { @@ -1097,6 +1148,22 @@ mod tests { }))) .mount(prism) .await; + Mock::given(method("GET")) + .and(path("/v1/recipe")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "version": "2.1.0", + "competition_id": "prism-v2.1", + "scoring_generation": 21, + "dataset_ref": "HuggingFaceFW/fineweb-edu@sample/10BT", + "max_params": 1_000_000_000_u64, + "train_hours_cap": 4.0 + }))) + .mount(prism) + .await; + mount_prism_list_collection(prism).await; + } + + async fn mount_prism_list_collection(prism: &MockServer) { Mock::given(method("GET")) .and(path("/v1/submissions")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ @@ -1111,6 +1178,17 @@ mod tests { "score": {"kind":"score","value": 900}, "created_at_ms": 1_700_000_000_000_u64, "updated_at_ms": 1_700_000_000_000_u64 + }, { + "id": "sub-old20", + "miner_hotkey": "dd".repeat(32), + "epoch": 2, + "status": "terminated", + "label": "old-2.0", + "bpb": 0.9, + "n_params": 350_000_000_u64, + "score": {"kind":"score","value": 800}, + "created_at_ms": 1_700_000_000_000_u64, + "updated_at_ms": 1_700_000_000_000_u64 }, { "id": "sub-running", "miner_hotkey": "cc".repeat(32), @@ -1157,9 +1235,15 @@ mod tests { "n_params": 12_000_000_u64, "score": {"kind":"score","value": 900}, "metrics": { - "recipe": "2.0.0", + "recipe": "2.1.0", + "competition_id": "prism-v2.1", + "scoring_generation": 21, "bpb": 1.25, "tokens_seen": 2048, + "org.g7.throughput_toks_s": {"value": 1200.0}, + "org.diag.mfu_achieved": {"value": 0.23}, + "org.diag.flops_attested": {"value": 1.5e18}, + "bits_per_byte": {"value": 0.88}, "wall_clock_seconds": 12.0, "gpu_type": "SIM", "n_params": 12_000_000_u64, @@ -1205,6 +1289,29 @@ mod tests { }))) .mount(prism) .await; + Mock::given(method("GET")) + .and(path("/v1/submissions/sub-old20")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "submission": { + "id": "sub-old20", + "miner_hotkey": "dd".repeat(32), + "epoch": 2, + "status": "terminated", + "label": "old-2.0", + "bpb": 0.9, + "n_params": 350_000_000_u64, + "score": {"kind":"score","value": 800}, + "created_at_ms": 1_700_000_000_000_u64, + "metrics": { + "recipe": "2.0.0", + "bpb": 0.9, + "org.g2.hellaswag_acc": {"value": 0.99} + } + }, + "events": [] + }))) + .mount(prism) + .await; } #[tokio::test] @@ -1328,14 +1435,25 @@ mod tests { let (s, v) = call(app.clone(), "/v1/site/arenas/prism/submissions").await; assert_eq!(s, StatusCode::OK, "{v}"); - assert_eq!(v["total"], 2, "default scope=all includes in-flight: {v}"); - assert_eq!(v["items"][0]["recipeEra"], "automodel"); + assert_eq!( + v["total"], 3, + "default scope=all includes in-flight + closed 2.0: {v}" + ); + assert_eq!(v["items"][0]["recipeEra"], "v21"); assert_eq!(v["items"][0]["pinId"], "automodel@v0.5.0"); + assert_eq!(v["items"][0]["weightEligible"], true); + assert_eq!(v["items"][0]["competitionId"], "prism-v2.1"); + assert_eq!(v["items"][0]["tokens"], 2048.0); + assert_eq!(v["items"][0]["tokensPerSec"], 1200.0); + assert_eq!(v["items"][0]["mfu"], 0.23); assert_eq!(v["items"][0]["benchmarks"]["hellaswag"], 0.31); assert_eq!(v["items"][0]["evalGroups"].as_array().unwrap().len(), 2); - assert_eq!(v["items"][1]["id"], "sub-running"); - assert_eq!(v["items"][1]["status"], "pending"); - assert_eq!(v["items"][1]["recipeEra"], "legacy"); + assert_eq!(v["items"][1]["id"], "sub-old20"); + assert_eq!(v["items"][1]["recipeEra"], "automodel"); + assert_eq!(v["items"][1]["weightEligible"], false); + assert_eq!(v["items"][2]["id"], "sub-running"); + assert_eq!(v["items"][2]["status"], "pending"); + assert_eq!(v["items"][2]["recipeEra"], "legacy"); // scope=champions keeps Score>0 gallery; default scope=all includes in-flight. let (s, v) = call( @@ -1344,18 +1462,26 @@ mod tests { ) .await; assert_eq!(s, StatusCode::OK, "{v}"); - assert_eq!(v["total"], 1, "{v}"); + assert_eq!( + v["total"], 2, + "scope=champions should keep Score>0 rows (v21 + closed 2.0): {v}" + ); let (s, v) = call(app.clone(), "/v1/site/arenas/prism/leaderboard").await; assert_eq!(s, StatusCode::OK, "{v}"); + assert_eq!(v["competitionId"], "prism-v2.1"); + assert_eq!(v["scoringGeneration"], 21); + assert_eq!(v["waitingForFirst"], false); + assert_eq!(v["total"], 1, "closed 2.0 champion must not rank: {v}"); assert_eq!(v["items"][0]["submissionId"], "sub1"); - assert_eq!(v["items"][0]["recipeEra"], "automodel"); + assert_eq!(v["items"][0]["recipeEra"], "v21"); + assert_eq!(v["items"][0]["weightEligible"], true); assert_eq!(v["items"][0]["benchmarks"]["piqa"], 0.62); let (s, v) = call(app.clone(), "/v1/site/arenas/prism/submissions/sub1").await; assert_eq!(s, StatusCode::OK, "{v}"); assert_eq!(v["id"], "sub1"); - assert_eq!(v["recipeEra"], "automodel"); + assert_eq!(v["recipeEra"], "v21"); assert_eq!(v["pinId"], "automodel@v0.5.0"); assert_eq!(v["eval"]["status"], "scored"); assert_eq!(v["eval"]["groups"].as_array().unwrap().len(), 2); @@ -1523,7 +1649,7 @@ mod tests { Mock::given(method("GET")) .and(path("/v1/recipe")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "version": "1.0.1", + "version": "2.1.0", "dataset_ref": "ds@pin", "pin_hex": "abc" }))) @@ -1540,8 +1666,8 @@ mod tests { .and(path("/v1/submissions")) .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "submissions": [ - {"id":"x1","status":"terminated","bpb":2.0,"label":"a"}, - {"id":"x2","status":"terminated","bpb":1.0,"label":"b"} + {"id":"x1","status":"terminated","bpb":2.0,"label":"a","metrics":{"recipe":"2.1.0"}}, + {"id":"x2","status":"terminated","bpb":1.0,"label":"b","metrics":{"recipe":"2.1.0"}} ] }))) .mount(&prism) @@ -1553,6 +1679,8 @@ mod tests { "id": "x2", "bpb": 1.0, "metrics": { + "recipe": "2.1.0", + "competition_id": "prism-v2.1", "bpb": 1.0, "n_params": 12_000_000_u64, "telemetry": { diff --git a/crates/site-api/src/lib.rs b/crates/site-api/src/lib.rs index 171753c77..9ef9bd656 100644 --- a/crates/site-api/src/lib.rs +++ b/crates/site-api/src/lib.rs @@ -13,7 +13,6 @@ #![allow(clippy::doc_markdown)] mod handlers; -mod prism_enrich; mod state; mod upstream; diff --git a/crates/site-api/src/upstream.rs b/crates/site-api/src/upstream.rs index 1e2cbe17c..8aaa9bf32 100644 --- a/crates/site-api/src/upstream.rs +++ b/crates/site-api/src/upstream.rs @@ -53,7 +53,11 @@ pub async fn get_json( .map_err(|e| UpstreamError::Transport(e.to_string()))?; let status = resp.status(); if !status.is_success() { - st.registry.record_failure(backend.id); + // Optional fan-out (diff / zone-A) often 404s; do not mark the + // challenge backend unhealthy or later list/leaderboard reads go dark. + if status.as_u16() != 404 { + st.registry.record_failure(backend.id); + } return Err(UpstreamError::Transport(format!( "upstream {challenge_id} {path} → {status}" ))); diff --git a/crates/site-data/src/map.rs b/crates/site-data/src/map.rs index 91514bdbe..b53d93084 100644 --- a/crates/site-data/src/map.rs +++ b/crates/site-data/src/map.rs @@ -397,6 +397,7 @@ pub fn design_leaderboard( pin_id: None, eval_groups: None, benchmarks: None, + run: site_types::PrismRunStats::default(), }, ) .collect() @@ -534,6 +535,7 @@ pub fn design_submission( pin_id: None, eval_groups: None, benchmarks: None, + run: site_types::PrismRunStats::default(), }) } @@ -628,6 +630,7 @@ pub fn prism_submission(row: &Value) -> Option { pin_id: None, eval_groups: None, benchmarks: None, + run: site_types::PrismRunStats::default(), }) } @@ -728,6 +731,7 @@ pub fn prism_bpb_leaderboard(subs: &[Value], epoch: u64) -> Vec pin_id: None, eval_groups: None, benchmarks: None, + run: site_types::PrismRunStats::default(), }, ) .collect() diff --git a/crates/site-prism/Cargo.toml b/crates/site-prism/Cargo.toml new file mode 100644 index 000000000..1e6e5ad87 --- /dev/null +++ b/crates/site-prism/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "site-prism" +description = "Prism list/detail enrichment for the marketing site API" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +publish = false + +[dependencies] +serde_json = "1" +site-data = { path = "../site-data" } +site-types = { path = "../site-types" } + +[lints] +workspace = true diff --git a/crates/site-api/src/prism_enrich.rs b/crates/site-prism/src/lib.rs similarity index 76% rename from crates/site-api/src/prism_enrich.rs rename to crates/site-prism/src/lib.rs index 0334b72fd..c95343d19 100644 --- a/crates/site-api/src/prism_enrich.rs +++ b/crates/site-prism/src/lib.rs @@ -1,11 +1,17 @@ //! Prism list/detail enrichment: recipe era, eval groups, G2 benches, GPT-2 refs. +#![allow(clippy::cast_precision_loss)] +#![allow(clippy::cast_possible_truncation)] +#![allow(clippy::cast_sign_loss)] +#![allow(clippy::cast_possible_wrap)] +#![allow(clippy::doc_markdown)] + use serde_json::Value; use site_types::{ EvalGroupScore, PrismBenchmarks, PrismEvalSummary, PrismGateSummary, PrismPublicReview, - PrismPublicSimilarity, PrismReferenceBaseline, PrismSubmissionDetail, PrismTelemetry, - RecipeEra, Submission, + PrismPublicSimilarity, PrismReferenceBaseline, PrismRunStats, PrismSubmissionDetail, + PrismTelemetry, RecipeEra, Submission, }; use site_data::map::{prism_submission, prism_telemetry}; @@ -95,31 +101,133 @@ pub fn prism_reference_baselines() -> Vec { ] } -/// Infer AutoModel vs legacy from a detail or list-shaped payload. +/// Infer contest era from a detail or list-shaped payload. /// -/// Signals (first match wins as automodel): explicit `pin_id`, AutoModel pin -/// id string, recipe major ≥ 2 in metrics / pod manifest, or `.prism` -/// automodel artifact paths. Otherwise legacy (incl. unknown / pre-2.0). +/// Fail-closed for **v2.1**: recipe `2.1.x`, `competition_id=prism-v2.1`, or +/// `scoring_generation=21`. Pin id alone is **not** v2.1 (2.0 used the same +/// AutoModel pin). Recipe `2.0.x` / pin / automodel paths → closed Automodel +/// contest. Otherwise legacy. #[must_use] pub fn infer_recipe_era(payload: &Value) -> RecipeEra { + infer_recipe_era_with_live(payload, None) +} + +/// Same as [`infer_recipe_era`], but a live recipe `2.1.x` lets pin-only +/// in-flight rows (no harvest metrics yet) count as v2.1. +#[must_use] +pub fn infer_recipe_era_with_live(payload: &Value, live_recipe: Option<&str>) -> RecipeEra { + if payload_is_v21_contest(payload) { + return RecipeEra::V21; + } let sub = payload.get("submission").unwrap_or(payload); + if recipe_is_major_minor(sub, 2, 0) { + return RecipeEra::Automodel; + } if pin_id_from_payload(payload).is_some() { + if live_recipe.is_some_and(recipe_semver_is_v21) { + return RecipeEra::V21; + } return RecipeEra::Automodel; } if recipe_major(sub).is_some_and(|m| m >= 2) { return RecipeEra::Automodel; } - // Diff / tree hints when present on a fan-in blob. if payload .pointer("/diffstat/files") .and_then(Value::as_array) .is_some_and(|a| !a.is_empty()) { + if live_recipe.is_some_and(recipe_semver_is_v21) { + return RecipeEra::V21; + } return RecipeEra::Automodel; } RecipeEra::Legacy } +/// True when the payload is the live Prism v2.1 contest (fail-closed). +#[must_use] +pub fn payload_is_v21_contest(payload: &Value) -> bool { + let root = payload.get("submission").unwrap_or(payload); + let metrics = root.get("metrics").unwrap_or(root); + contest_id_is_v21(metrics) + || scoring_generation_is_21(metrics) + || recipe_is_major_minor(root, 2, 1) + || recipe_is_major_minor(metrics, 2, 1) +} + +fn contest_id_is_v21(metrics: &Value) -> bool { + for path in [ + "/competition_id", + "/pod_manifest/competition_id", + "/metrics/competition_id", + "/metrics/pod_manifest/competition_id", + ] { + if metrics + .pointer(path) + .and_then(Value::as_str) + .is_some_and(|s| s.trim() == "prism-v2.1") + { + return true; + } + if metrics + .get("competition_id") + .and_then(Value::as_str) + .is_some_and(|s| s.trim() == "prism-v2.1") + { + return true; + } + } + false +} + +fn scoring_generation_is_21(metrics: &Value) -> bool { + for node in [ + metrics.get("scoring_generation"), + metrics.pointer("/pod_manifest/scoring_generation"), + metrics.pointer("/metrics/scoring_generation"), + ] { + let Some(v) = node else { continue }; + if v.as_u64() == Some(21) { + return true; + } + if v.as_str() + .and_then(|s| s.trim().parse::().ok()) + .is_some_and(|n| n == 21) + { + return true; + } + } + false +} + +fn recipe_semver_is_v21(raw: &str) -> bool { + let mut parts = raw.trim().split('.'); + parts.next() == Some("2") && parts.next() == Some("1") +} + +fn recipe_is_major_minor(sub: &Value, major: u32, minor: u32) -> bool { + recipe_version_str(sub).is_some_and(|s| { + let mut parts = s.trim().split('.'); + parts.next().and_then(|p| p.parse::().ok()) == Some(major) + && parts.next().and_then(|p| p.parse::().ok()) == Some(minor) + }) +} + +fn recipe_version_str(sub: &Value) -> Option<&str> { + sub.pointer("/metrics/recipe") + .and_then(Value::as_str) + .or_else(|| { + sub.pointer("/metrics/pod_manifest/recipe_version") + .and_then(Value::as_str) + }) + .or_else(|| { + sub.pointer("/metrics/pod_manifest/recipe") + .and_then(Value::as_str) + }) + .or_else(|| sub.get("recipe").and_then(Value::as_str)) +} + /// AutoModel pin id from detail, `/diff`, or metrics pod manifest. #[must_use] pub fn pin_id_from_payload(payload: &Value) -> Option { @@ -287,7 +395,7 @@ pub fn enrich_leaderboard_row_from_detail_with_zone( let root = detail.get("submission").unwrap_or(detail); let era = infer_recipe_era(detail); row.recipe_era = Some(era); - if era == RecipeEra::Automodel { + if era == RecipeEra::Automodel || era == RecipeEra::V21 { row.pin_id = pin_id_from_payload(detail); } if let Some(groups) = map_eval_groups(root.get("eval").filter(|e| !e.is_null())) { @@ -297,6 +405,7 @@ pub fn enrich_leaderboard_row_from_detail_with_zone( if !benches.is_empty() { row.benchmarks = Some(benches); } + row.run = map_run_stats(detail, era); } /// Apply detail fan-out fields onto a list [`Submission`]. @@ -309,7 +418,7 @@ pub fn enrich_submission_from_detail_with_zone( let era = infer_recipe_era(detail); sub.recipe_era = Some(era); let pin = pin_id_from_payload(detail); - if era == RecipeEra::Automodel { + if era == RecipeEra::Automodel || era == RecipeEra::V21 { sub.pin_id = pin; } let eval = root.get("eval").filter(|e| !e.is_null()); @@ -339,6 +448,83 @@ pub fn enrich_submission_from_detail_with_zone( }) .map(|p| p as f64 / 1e6); } + sub.run = map_run_stats(detail, era); +} + +/// Map documented harness keys onto public run stats. Missing keys stay `None`. +fn map_run_stats(detail: &Value, era: RecipeEra) -> PrismRunStats { + let root = detail.get("submission").unwrap_or(detail); + let metrics = root.get("metrics").filter(|m| !m.is_null()); + let recipe = recipe_version_str(root).map(str::to_owned); + let competition_id = metrics + .and_then(|m| { + m.get("competition_id").and_then(Value::as_str).or_else(|| { + m.pointer("/pod_manifest/competition_id") + .and_then(Value::as_str) + }) + }) + .map(str::to_owned); + let scoring_generation = metrics.and_then(|m| { + m.get("scoring_generation") + .and_then(Value::as_u64) + .or_else(|| { + m.get("scoring_generation") + .and_then(Value::as_str) + .and_then(|s| s.trim().parse::().ok()) + }) + .or_else(|| { + m.pointer("/pod_manifest/scoring_generation") + .and_then(Value::as_u64) + }) + .and_then(|n| u16::try_from(n).ok()) + }); + let eval = root.get("eval").filter(|e| !e.is_null()); + PrismRunStats { + competition_id, + scoring_generation, + recipe_version: recipe, + weight_eligible: Some(era == RecipeEra::V21), + bits_per_byte: metrics.and_then(|m| { + metric_f64( + m, + &[ + "bits_per_byte", + "org.g1.bits_per_byte_val", + "g1.bits_per_byte.val", + ], + ) + }), + tokens: metrics + .and_then(|m| metric_f64(m, &["tokens_seen", "org.diag.tokens_seen", "tokens"])), + tokens_per_sec: metrics.and_then(|m| { + metric_f64( + m, + &[ + "org.g7.throughput_toks_s", + "g7.throughput_toks_s", + "org.g7.throughput_toks_s_per_gparam", + ], + ) + }), + mfu: metrics.and_then(|m| metric_f64(m, &["org.diag.mfu_achieved", "mfu"])), + flops: metrics.and_then(|m| { + metric_f64( + m, + &[ + "org.diag.flops_attested", + "flops_spent", + "org.diag.flops_spent", + ], + ) + }), + gpu_type: metrics + .and_then(|m| m.get("gpu_type")) + .and_then(Value::as_str) + .map(str::to_owned), + gates_complete: eval + .and_then(|e| e.pointer("/gates/complete")) + .and_then(Value::as_bool), + } } /// Map challenge `GET /v1/submissions/{id}` (+ optional Zone-A) → public detail. @@ -423,14 +609,7 @@ fn map_eval_summary(eval: &Value) -> PrismEvalSummary { } fn recipe_major(sub: &Value) -> Option { - let raw = sub - .pointer("/metrics/recipe") - .and_then(Value::as_str) - .or_else(|| { - sub.pointer("/metrics/pod_manifest/recipe_version") - .and_then(Value::as_str) - })?; - raw.split('.').next()?.parse().ok() + recipe_version_str(sub)?.split('.').next()?.parse().ok() } fn metric_f64(metrics: &Value, keys: &[&str]) -> Option { @@ -481,8 +660,16 @@ mod tests { assert_eq!(infer_recipe_era(&legacy), RecipeEra::Legacy); let auto = json!({"metrics": {"recipe": "2.0.0"}}); assert_eq!(infer_recipe_era(&auto), RecipeEra::Automodel); + let v21 = json!({"metrics": {"recipe": "2.1.0", "competition_id": "prism-v2.1"}}); + assert_eq!(infer_recipe_era(&v21), RecipeEra::V21); + let gen = json!({"metrics": {"scoring_generation": 21}}); + assert_eq!(infer_recipe_era(&gen), RecipeEra::V21); let pin = json!({"pin_id": "automodel@v0.5.0"}); assert_eq!(infer_recipe_era(&pin), RecipeEra::Automodel); + assert_eq!( + infer_recipe_era_with_live(&pin, Some("2.1.0")), + RecipeEra::V21 + ); assert_eq!( pin_id_from_payload(&pin).as_deref(), Some("automodel@v0.5.0") diff --git a/crates/site-types/src/frames.rs b/crates/site-types/src/frames.rs index cf5b31ec6..bd98efbc9 100644 --- a/crates/site-types/src/frames.rs +++ b/crates/site-types/src/frames.rs @@ -72,18 +72,18 @@ pub fn prism_frame() -> Arena { Arena { slug: ArenaSlug::Prism, name: "Prism".into(), - tagline: "Agents propose neural architectures and train them on a sealed data window — score is final validation loss.".into(), - description: "Every miner trains inside the operator-owned recipe on the same pinned shard, seed, and caps. Rankings use real BPB / lattice scores from terminated runs — never invented curves.".into(), + tagline: "Prism v2.1 — AutoModel pin+patch, 4h train on 1× B200, dense 1B reference (850M–1B). Public board is G2 lattice; 2.0 harvests cannot win.".into(), + description: "New competition (prism-v2.1, scoring generation 21, recipe 2.1.0). Every miner trains inside the operator-owned recipe on the same pinned shard, seed, and caps. The public board lists only v2.1 harvests. Until the first eligible 2.1 run finishes, subnet weights stay burn (uid 0). Rankings use measured G2 / G1–G8 fields — never invented curves.".into(), status: "live".into(), scoring: ScoringMethod::SpectralFusion, mechanism: vec![ - "Pinned recipe + dataset".into(), - "Lium (or sim) train + val BPB".into(), - "Agentic / similarity gate → leaf".into(), + "Recipe 2.1.0 · prism-v2.1 · 4h / 1× B200".into(), + "G2 lattice (0–100) + transparent G1–G8 stats".into(), + "Agentic / similarity gate → leaf (WTA)".into(), ], agents: 0, best_score: "—".into(), - best_score_label: "BPB".into(), + best_score_label: "BEST G2".into(), emission_share: 0.0, weight: 0.0, rewards_per_day: 0.0, diff --git a/crates/site-types/src/types.rs b/crates/site-types/src/types.rs index 92cc33e06..68468344f 100644 --- a/crates/site-types/src/types.rs +++ b/crates/site-types/src/types.rs @@ -134,16 +134,59 @@ pub struct Agent { pub joined_epoch: u64, } -/// Prism recipe era (`AutoModel` 2.0 vs legacy 1.x two-script / tree). +/// Prism recipe era (`v2.1` contest vs closed `2.0` AutoModel vs legacy 1.x). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum RecipeEra { - /// Recipe ≥ 2.0 AutoModel pin + patch. + /// Live contest: recipe `2.1.x` / `competition_id=prism-v2.1` / gen `21`. + V21, + /// Closed AutoModel 2.0 contest (pin+patch). Not weight-eligible. Automodel, /// Pre-2.0 architecture/training (or tree) layout. Legacy, } +/// Optional measured Prism run stats (only fields the upstream payload has). +/// +/// Flattened onto leaderboard / submission rows. Never invent values. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PrismRunStats { + /// `prism-v2.1` when the harvest carries the live contest id. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub competition_id: Option, + /// `21` when the harvest carries live `scoring_generation`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub scoring_generation: Option, + /// Recipe semver (`2.1.0`) when present on metrics / manifest. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub recipe_version: Option, + /// Fail-closed weight eligibility (recipe 2.1 / gen 21 only). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub weight_eligible: Option, + /// Tokenizer-neutral bits/byte when the harness emitted it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bits_per_byte: Option, + /// Tokens seen / accounted when present. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tokens: Option, + /// `org.g7.throughput_toks_s` when present. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tokens_per_sec: Option, + /// `org.diag.mfu_achieved` when present (0..1). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mfu: Option, + /// Attested / spent FLOPs when present. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub flops: Option, + /// GPU SKU string when the harness recorded one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gpu_type: Option, + /// Eval gate `complete` when the composite outcome is present. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gates_complete: Option, +} + /// One G1–G8 composite group score for marketing tables. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -274,6 +317,9 @@ pub struct LeaderboardRow { /// Public G2 benchmark subset when measured. #[serde(default, skip_serializing_if = "Option::is_none")] pub benchmarks: Option, + /// Measured contest / efficiency fields (absent when unknown). + #[serde(flatten, default)] + pub run: PrismRunStats, } /// Submission status. @@ -345,6 +391,9 @@ pub struct Submission { /// Public G2 benchmark subset when measured. #[serde(default, skip_serializing_if = "Option::is_none")] pub benchmarks: Option, + /// Measured contest / efficiency fields (absent when unknown). + #[serde(flatten, default)] + pub run: PrismRunStats, } /// Lexicographic gate flags from a Prism composite outcome (public subset). diff --git a/docs/SITE_API.md b/docs/SITE_API.md index 25dbaa911..d80083db6 100644 --- a/docs/SITE_API.md +++ b/docs/SITE_API.md @@ -62,7 +62,7 @@ Additional Prism routes: | `GET /v1/site/arenas/prism/references` | `PrismReferenceBaseline[]` — frozen **Prism-protocol** GPT-2 references (Large 774M **and** Small 124M): measured val **`bpb`** + G2 benches from 1×RTX 5090 eval-only runs on the public pack (`gpt2-large` + `openai-community/gpt2`). Includes `sourceUrl` / `disclaimer`. | | `GET /v1/site/arenas/prism/submissions/{id}/telemetry` | Existing loss-curve payload (also embedded on detail). | -GPT-2 Large + Small constants live in `crates/site-api` (`prism_enrich`) so API and FE stay aligned; they are **measured Prism-protocol** numbers (eval-only, public pack), not Eleuther literature tables. List/leaderboard row shells still map in `crates/site-data`. +GPT-2 Large + Small constants live in `crates/site-prism` so API and FE stay aligned; they are **measured Prism-protocol** numbers (eval-only, public pack), not Eleuther literature tables. List/leaderboard row shells still map in `crates/site-data`. `GET /v1/site/arenas/{slug}/submissions` and `/leaderboard` accept optional `?q=` — case-insensitive substring over miner hotkey (SS58 or hex), handle,