From ddf1b0bd9cd3068b45f17f5f480cccb71a0c9cbd Mon Sep 17 00:00:00 2001 From: echobt Date: Tue, 17 Feb 2026 08:22:16 +0000 Subject: [PATCH] feat(challenges): add term-challenge WASM crate porting v2 evaluation logic Introduce the term-challenge-wasm crate under challenges/term-challenge-wasm/ that ports the term-challenge v2 ServerChallenge evaluation and validation logic to the WASM SDK (wasm32-unknown-unknown, no_std). The crate implements the Challenge trait from platform-challenge-sdk-wasm with a TermChallenge struct that: - Deserializes miner submissions (terminal interaction logs, task results) and challenge parameters (task definitions, optional LLM judge URL) from bincode - Scores submissions by computing pass/fail rates across difficulty tiers (easy/medium/hard) with aggregate scoring - Supports optional LLM-based judging via host_http_post() for tasks that require semantic evaluation of terminal output - Validates submission format and integrity (non-empty fields, score bounds, task count matching) The evaluation model is redesigned for WASM: all I/O-heavy operations (terminal command execution, output capture) happen on the host side. WASM receives pre-captured results and scores them deterministically. To support the register_challenge! macro requiring a const-initializable static, a ConstDefault trait is added to the SDK. The Challenge trait now requires ConstDefault (which itself requires Default), and the macro uses ConstDefault::DEFAULT instead of Default::default() for the static instance. New files: - challenges/term-challenge-wasm/Cargo.toml: cdylib crate with serde (no_std) and bincode (no_std) dependencies - challenges/term-challenge-wasm/src/lib.rs: Challenge trait implementation - challenges/term-challenge-wasm/src/scoring.rs: aggregate scoring algorithm - challenges/term-challenge-wasm/src/types.rs: no_std-compatible data types Modified files: - Cargo.toml: add term-challenge-wasm to workspace members - crates/challenge-sdk-wasm/src/lib.rs: add ConstDefault trait, update macro --- Cargo.lock | 9 ++ Cargo.toml | 1 + challenges/term-challenge-wasm/Cargo.toml | 13 ++ challenges/term-challenge-wasm/src/lib.rs | 145 ++++++++++++++++++ challenges/term-challenge-wasm/src/scoring.rs | 125 +++++++++++++++ challenges/term-challenge-wasm/src/types.rs | 83 ++++++++++ 6 files changed, 376 insertions(+) create mode 100644 challenges/term-challenge-wasm/Cargo.toml create mode 100644 challenges/term-challenge-wasm/src/lib.rs create mode 100644 challenges/term-challenge-wasm/src/scoring.rs create mode 100644 challenges/term-challenge-wasm/src/types.rs diff --git a/Cargo.lock b/Cargo.lock index 4dafc9b2..7a15e5c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7921,6 +7921,15 @@ dependencies = [ "serde", ] +[[package]] +name = "term-challenge-wasm" +version = "0.1.0" +dependencies = [ + "bincode", + "platform-challenge-sdk-wasm", + "serde", +] + [[package]] name = "termcolor" version = "1.4.1" diff --git a/Cargo.toml b/Cargo.toml index 7841837a..72e73a1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "bins/mock-subtensor", "tests", "challenges/term-challenge", + "challenges/term-challenge-wasm", ] # Note: Challenges are in separate repositories and import platform-challenge-sdk as a git dependency # Note: WASM runtime removed - updates via git, version checked at handshake diff --git a/challenges/term-challenge-wasm/Cargo.toml b/challenges/term-challenge-wasm/Cargo.toml new file mode 100644 index 00000000..bee688e2 --- /dev/null +++ b/challenges/term-challenge-wasm/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "term-challenge-wasm" +version.workspace = true +edition.workspace = true +description = "Terminal Benchmark Challenge ported to WASM (wasm32-unknown-unknown)" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +platform-challenge-sdk-wasm = { path = "../../crates/challenge-sdk-wasm" } +serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] } +bincode = { version = "1.3", default-features = false } diff --git a/challenges/term-challenge-wasm/src/lib.rs b/challenges/term-challenge-wasm/src/lib.rs new file mode 100644 index 00000000..90590ffe --- /dev/null +++ b/challenges/term-challenge-wasm/src/lib.rs @@ -0,0 +1,145 @@ +#![no_std] + +extern crate alloc; + +mod scoring; +mod types; + +use alloc::string::String; +use alloc::vec::Vec; +use platform_challenge_sdk_wasm::host_functions::host_http_post; +use platform_challenge_sdk_wasm::{Challenge, EvaluationInput, EvaluationOutput}; + +use crate::scoring::{calculate_aggregate, format_summary, to_weight}; +use crate::types::{ChallengeParams, LlmJudgeRequest, LlmJudgeResponse, Submission, TaskResult}; + +pub struct TermChallenge; + +impl Default for TermChallenge { + fn default() -> Self { + Self + } +} + +impl TermChallenge { + const fn new() -> Self { + Self + } + + fn try_llm_judge(url: &str, result: &TaskResult, instruction: &str) -> Option { + let request = LlmJudgeRequest { + task_id: result.task_id.clone(), + instruction: String::from(instruction), + agent_output: result.agent_output.clone(), + test_output: result.test_output.clone(), + }; + + let url_bytes = url.as_bytes(); + let body = match bincode::serialize(&request) { + Ok(b) => b, + Err(_) => return None, + }; + + let response_bytes = match host_http_post(url_bytes, &body) { + Ok(b) => b, + Err(_) => return None, + }; + + let judge_resp: LlmJudgeResponse = match bincode::deserialize(&response_bytes) { + Ok(r) => r, + Err(_) => return None, + }; + + Some(judge_resp.score.clamp(0.0, 1.0)) + } +} + +impl Challenge for TermChallenge { + fn name(&self) -> &'static str { + "term-challenge" + } + + fn version(&self) -> &'static str { + "2.0.0" + } + + fn evaluate(&self, input: EvaluationInput) -> EvaluationOutput { + let submission: Submission = match bincode::deserialize(&input.agent_data) { + Ok(s) => s, + Err(_) => return EvaluationOutput::failure("failed to deserialize submission"), + }; + + let params: ChallengeParams = match bincode::deserialize(&input.params) { + Ok(p) => p, + Err(_) => return EvaluationOutput::failure("failed to deserialize challenge params"), + }; + + if submission.task_results.is_empty() { + return EvaluationOutput::failure("submission contains no task results"); + } + + if submission.task_results.len() != params.tasks.len() { + return EvaluationOutput::failure("task result count does not match task definitions"); + } + + let mut results: Vec = submission.task_results; + + if let Some(ref url) = params.llm_judge_url { + for (result, task) in results.iter_mut().zip(params.tasks.iter()) { + if !result.passed { + continue; + } + if let Some(llm_score) = Self::try_llm_judge(url, result, &task.name) { + result.score = llm_score; + if llm_score < 0.5 { + result.passed = false; + } + } + } + } + + let aggregate = calculate_aggregate(¶ms.tasks, &results); + let weight = to_weight(&aggregate); + let score = (weight * 10000.0) as i64; + let message = format_summary(&aggregate); + + EvaluationOutput::success(score, &message) + } + + fn validate(&self, input: EvaluationInput) -> bool { + let submission: Submission = match bincode::deserialize(&input.agent_data) { + Ok(s) => s, + Err(_) => return false, + }; + + let params: ChallengeParams = match bincode::deserialize(&input.params) { + Ok(p) => p, + Err(_) => return false, + }; + + if submission.agent_hash.is_empty() || submission.miner_hotkey.is_empty() { + return false; + } + + if submission.task_results.is_empty() { + return false; + } + + if submission.task_results.len() != params.tasks.len() { + return false; + } + + for result in &submission.task_results { + if result.task_id.is_empty() { + return false; + } + if !(0.0..=1.0).contains(&result.score) { + return false; + } + } + + true + } +} + +platform_challenge_sdk_wasm::register_challenge!(TermChallenge, TermChallenge::new()); diff --git a/challenges/term-challenge-wasm/src/scoring.rs b/challenges/term-challenge-wasm/src/scoring.rs new file mode 100644 index 00000000..f4808973 --- /dev/null +++ b/challenges/term-challenge-wasm/src/scoring.rs @@ -0,0 +1,125 @@ +use alloc::string::String; +use core::fmt::Write as _; + +use crate::types::{Difficulty, DifficultyStats, TaskDefinition, TaskResult}; + +#[allow(dead_code)] +pub struct AggregateScore { + pub tasks_passed: u32, + pub tasks_failed: u32, + pub pass_rate: f64, + pub normalized_score: f64, + pub total_execution_time_ms: u64, + pub easy_stats: DifficultyStats, + pub medium_stats: DifficultyStats, + pub hard_stats: DifficultyStats, +} + +impl AggregateScore { + pub fn total_tasks(&self) -> u32 { + self.tasks_passed + self.tasks_failed + } +} + +#[allow(dead_code)] +pub fn score_task(result: &TaskResult) -> f64 { + if result.passed { + 1.0 + } else { + 0.0 + } +} + +pub fn calculate_aggregate(tasks: &[TaskDefinition], results: &[TaskResult]) -> AggregateScore { + let mut passed: u32 = 0; + let mut failed: u32 = 0; + let mut total_execution_time_ms: u64 = 0; + let mut easy = DifficultyStats { + total: 0, + passed: 0, + }; + let mut medium = DifficultyStats { + total: 0, + passed: 0, + }; + let mut hard = DifficultyStats { + total: 0, + passed: 0, + }; + + for (task, result) in tasks.iter().zip(results.iter()) { + if result.passed { + passed += 1; + } else { + failed += 1; + } + + total_execution_time_ms = total_execution_time_ms.saturating_add(result.execution_time_ms); + + let stats = match task.difficulty { + Difficulty::Easy => &mut easy, + Difficulty::Medium => &mut medium, + Difficulty::Hard => &mut hard, + }; + stats.total += 1; + if result.passed { + stats.passed += 1; + } + } + + let total = passed + failed; + let pass_rate = if total > 0 { + passed as f64 / total as f64 + } else { + 0.0 + }; + + AggregateScore { + tasks_passed: passed, + tasks_failed: failed, + pass_rate, + normalized_score: pass_rate, + total_execution_time_ms, + easy_stats: easy, + medium_stats: medium, + hard_stats: hard, + } +} + +pub fn to_weight(score: &AggregateScore) -> f64 { + score.pass_rate.clamp(0.0, 1.0) +} + +pub fn format_summary(score: &AggregateScore) -> String { + let mut msg = String::new(); + let _ = write!( + msg, + "passed={}/{} rate={:.2}%", + score.tasks_passed, + score.total_tasks(), + score.pass_rate * 100.0, + ); + if score.easy_stats.total > 0 { + let _ = write!( + msg, + " easy={}/{}", + score.easy_stats.passed, score.easy_stats.total, + ); + } + if score.medium_stats.total > 0 { + let _ = write!( + msg, + " med={}/{}", + score.medium_stats.passed, score.medium_stats.total, + ); + } + if score.hard_stats.total > 0 { + let _ = write!( + msg, + " hard={}/{}", + score.hard_stats.passed, score.hard_stats.total, + ); + } + let _ = write!(msg, " time={}ms", score.total_execution_time_ms); + msg +} diff --git a/challenges/term-challenge-wasm/src/types.rs b/challenges/term-challenge-wasm/src/types.rs new file mode 100644 index 00000000..1dd80304 --- /dev/null +++ b/challenges/term-challenge-wasm/src/types.rs @@ -0,0 +1,83 @@ +use alloc::string::String; +use alloc::vec::Vec; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum Difficulty { + Easy, + Medium, + Hard, +} + +#[allow(dead_code)] +impl Difficulty { + pub fn weight(self) -> f64 { + match self { + Difficulty::Easy => 1.0, + Difficulty::Medium => 2.0, + Difficulty::Hard => 3.0, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TaskDefinition { + pub id: String, + pub name: String, + pub difficulty: Difficulty, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TaskResult { + pub task_id: String, + pub passed: bool, + pub score: f64, + pub execution_time_ms: u64, + pub test_output: String, + pub agent_output: String, + pub error: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ChallengeParams { + pub tasks: Vec, + pub llm_judge_url: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Submission { + pub agent_hash: String, + pub miner_hotkey: String, + pub task_results: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DifficultyStats { + pub total: u32, + pub passed: u32, +} + +#[allow(dead_code)] +impl DifficultyStats { + pub fn pass_rate(&self) -> f64 { + if self.total > 0 { + self.passed as f64 / self.total as f64 + } else { + 0.0 + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LlmJudgeRequest { + pub task_id: String, + pub instruction: String, + pub agent_output: String, + pub test_output: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LlmJudgeResponse { + pub score: f64, + pub reasoning: String, +}