From 5735aea11e643084245b3c3ee970c3635476822d Mon Sep 17 00:00:00 2001 From: echobt Date: Tue, 17 Feb 2026 07:47:29 +0000 Subject: [PATCH] feat(challenges): add term-challenge WASM module crate Introduce the term-challenge-wasm crate at challenges/term-challenge-wasm/, implementing the Challenge trait from platform-challenge-sdk-wasm as a no_std-compatible WASM module targeting wasm32-unknown-unknown (cdylib). The crate contains: - lib.rs: TermChallenge unit struct implementing Challenge with name() returning "term-challenge", version() returning "0.2.3", evaluate() deserializing EvalParams via bincode and computing aggregate scores, and validate() for input validation. Registered via register_challenge!. - evaluation.rs: Core evaluation logic with EvalParams deserialization, agent data size validation (1MB limit), score calculation delegation, and no_std-compatible numeric formatting for result messages. - scoring.rs: ScoreCalculator with aggregate scoring (pass/fail counting, pass rate, normalized scores), DifficultyStats, and AggregateScore types with score-to-i64 conversion (0-10000 range). - tasks.rs: TaskDefinition and TaskResult types with Difficulty enum (Easy/Medium/Hard with weights), serde support, and convenience constructors for success/failure results. Also updates the register_challenge! macro in challenge-sdk-wasm to accept an explicit initializer expression instead of requiring Default, enabling unit structs to be used directly without a Default impl. The macro signature changes from register_challenge!(Type) to register_challenge!(Type, Expr). The crate is added to the workspace Cargo.toml members list and Cargo.lock is updated accordingly. --- Cargo.lock | 9 ++ Cargo.toml | 5 +- challenges/term-challenge-wasm/Cargo.toml | 12 +++ .../term-challenge-wasm/src/evaluation.rs | 100 ++++++++++++++++++ challenges/term-challenge-wasm/src/lib.rs | 32 ++++++ challenges/term-challenge-wasm/src/scoring.rs | 98 +++++++++++++++++ challenges/term-challenge-wasm/src/tasks.rs | 83 +++++++++++++++ crates/challenge-sdk-wasm/src/lib.rs | 9 +- 8 files changed, 341 insertions(+), 7 deletions(-) create mode 100644 challenges/term-challenge-wasm/Cargo.toml create mode 100644 challenges/term-challenge-wasm/src/evaluation.rs create mode 100644 challenges/term-challenge-wasm/src/lib.rs create mode 100644 challenges/term-challenge-wasm/src/scoring.rs create mode 100644 challenges/term-challenge-wasm/src/tasks.rs diff --git a/Cargo.lock b/Cargo.lock index a51aab8c..6488ad14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7912,6 +7912,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "term-challenge-wasm" +version = "0.2.3" +dependencies = [ + "bincode", + "platform-challenge-sdk-wasm", + "serde", +] + [[package]] name = "termcolor" version = "1.4.1" diff --git a/Cargo.toml b/Cargo.toml index 4612dc01..d5a5cda3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,15 +18,12 @@ members = [ "bins/utils", "bins/mock-subtensor", "tests", + "challenges/term-challenge-wasm", ] # Note: Challenges are in separate repositories and import platform-challenge-sdk as a git dependency # Note: WASM runtime removed - updates via git, version checked at handshake # Note: P2P-only architecture - no centralized platform-server -# Challenge crates can be added here or as optional path/git dependencies -# Example: -# "challenges/example-challenge", - [workspace.package] version = "0.1.0" edition = "2021" diff --git a/challenges/term-challenge-wasm/Cargo.toml b/challenges/term-challenge-wasm/Cargo.toml new file mode 100644 index 00000000..b50eb9a5 --- /dev/null +++ b/challenges/term-challenge-wasm/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "term-challenge-wasm" +version = "0.2.3" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +platform-challenge-sdk-wasm = { path = "../../crates/challenge-sdk-wasm" } +serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] } +bincode = { version = "1.3", default-features = false } diff --git a/challenges/term-challenge-wasm/src/evaluation.rs b/challenges/term-challenge-wasm/src/evaluation.rs new file mode 100644 index 00000000..e34f1e82 --- /dev/null +++ b/challenges/term-challenge-wasm/src/evaluation.rs @@ -0,0 +1,100 @@ +use alloc::string::String; +use alloc::vec::Vec; +use platform_challenge_sdk_wasm::EvaluationOutput; +use serde::{Deserialize, Serialize}; + +use crate::scoring::ScoreCalculator; +use crate::tasks::{TaskDefinition, TaskResult}; + +const MAX_AGENT_SIZE: usize = 1024 * 1024; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct EvalParams { + pub tasks: Vec, + pub results: Vec, +} + +pub fn evaluate(agent_data: &[u8], params: &[u8]) -> EvaluationOutput { + if agent_data.is_empty() { + return EvaluationOutput::failure("no agent data provided"); + } + + if agent_data.len() > MAX_AGENT_SIZE { + return EvaluationOutput::failure("agent data exceeds 1MB size limit"); + } + + let eval_params: EvalParams = match bincode::deserialize(params) { + Ok(p) => p, + Err(_) => return EvaluationOutput::failure("failed to deserialize evaluation params"), + }; + + if eval_params.tasks.is_empty() { + return EvaluationOutput::failure("no tasks provided"); + } + + if eval_params.results.is_empty() { + return EvaluationOutput::failure("no task results provided"); + } + + let calculator = ScoreCalculator; + let aggregate = calculator.calculate_aggregate(&eval_params.tasks, &eval_params.results); + let score = calculator.to_score_i64(&aggregate); + + let mut msg = String::new(); + msg.push_str("passed="); + push_usize(&mut msg, aggregate.tasks_passed); + msg.push_str(" failed="); + push_usize(&mut msg, aggregate.tasks_failed); + msg.push_str(" rate="); + push_f64_pct(&mut msg, aggregate.pass_rate); + + EvaluationOutput::success(score, &msg) +} + +pub fn validate(agent_data: &[u8], params: &[u8]) -> bool { + if agent_data.is_empty() || agent_data.len() > MAX_AGENT_SIZE { + return false; + } + + let eval_params: EvalParams = match bincode::deserialize(params) { + Ok(p) => p, + Err(_) => return false, + }; + + !eval_params.tasks.is_empty() +} + +fn push_usize(s: &mut String, v: usize) { + let mut buf = [0u8; 20]; + let n = fmt_usize(v, &mut buf); + if let Ok(part) = core::str::from_utf8(&buf[20 - n..]) { + s.push_str(part); + } +} + +fn fmt_usize(mut v: usize, buf: &mut [u8; 20]) -> usize { + if v == 0 { + buf[19] = b'0'; + return 1; + } + let mut i = 20; + while v > 0 { + i -= 1; + buf[i] = b'0' + (v % 10) as u8; + v /= 10; + } + 20 - i +} + +fn push_f64_pct(s: &mut String, v: f64) { + let pct = (v * 10000.0) as u64; + let whole = pct / 100; + let frac = pct % 100; + push_usize(s, whole as usize); + s.push('.'); + if frac < 10 { + s.push('0'); + } + push_usize(s, frac as usize); + s.push('%'); +} diff --git a/challenges/term-challenge-wasm/src/lib.rs b/challenges/term-challenge-wasm/src/lib.rs new file mode 100644 index 00000000..e1817f66 --- /dev/null +++ b/challenges/term-challenge-wasm/src/lib.rs @@ -0,0 +1,32 @@ +#![no_std] +#![allow(dead_code)] + +extern crate alloc; + +mod evaluation; +mod scoring; +mod tasks; + +use platform_challenge_sdk_wasm::{Challenge, EvaluationInput, EvaluationOutput}; + +pub struct TermChallenge; + +impl Challenge for TermChallenge { + fn name(&self) -> &'static str { + "term-challenge" + } + + fn version(&self) -> &'static str { + "0.2.3" + } + + fn evaluate(&self, input: EvaluationInput) -> EvaluationOutput { + evaluation::evaluate(&input.agent_data, &input.params) + } + + fn validate(&self, input: EvaluationInput) -> bool { + evaluation::validate(&input.agent_data, &input.params) + } +} + +platform_challenge_sdk_wasm::register_challenge!(TermChallenge, TermChallenge); diff --git a/challenges/term-challenge-wasm/src/scoring.rs b/challenges/term-challenge-wasm/src/scoring.rs new file mode 100644 index 00000000..445c0666 --- /dev/null +++ b/challenges/term-challenge-wasm/src/scoring.rs @@ -0,0 +1,98 @@ +use serde::{Deserialize, Serialize}; + +use crate::tasks::{TaskDefinition, TaskResult}; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct DifficultyStats { + pub total: usize, + pub passed: usize, + pub total_score: f64, +} + +impl DifficultyStats { + pub fn pass_rate(&self) -> f64 { + if self.total > 0 { + self.passed as f64 / self.total as f64 + } else { + 0.0 + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AggregateScore { + pub total_score: f64, + pub normalized_score: f64, + pub max_possible: f64, + pub tasks_passed: usize, + pub tasks_failed: usize, + pub pass_rate: f64, + pub total_execution_time_ms: Option, +} + +impl AggregateScore { + pub fn total_tasks(&self) -> usize { + self.tasks_passed + self.tasks_failed + } + + pub fn percentage(&self) -> f64 { + self.normalized_score * 100.0 + } +} + +pub struct ScoreCalculator; + +impl ScoreCalculator { + pub fn score_task(&self, result: &TaskResult) -> f64 { + if result.passed { + 1.0 + } else { + 0.0 + } + } + + pub fn calculate_aggregate( + &self, + tasks: &[TaskDefinition], + results: &[TaskResult], + ) -> AggregateScore { + let mut passed: usize = 0; + let mut failed: usize = 0; + let mut total_execution_time_ms: u64 = 0; + + for result in results.iter().take(tasks.len()) { + if result.passed { + passed += 1; + } else { + failed += 1; + } + total_execution_time_ms = + total_execution_time_ms.saturating_add(result.execution_time_ms); + } + + let total = passed + failed; + let pass_rate = if total > 0 { + passed as f64 / total as f64 + } else { + 0.0 + }; + + AggregateScore { + total_score: passed as f64, + normalized_score: pass_rate, + max_possible: total as f64, + tasks_passed: passed, + tasks_failed: failed, + pass_rate, + total_execution_time_ms: Some(total_execution_time_ms), + } + } + + pub fn to_weight(&self, score: &AggregateScore) -> f64 { + score.pass_rate.clamp(0.0, 1.0) + } + + pub fn to_score_i64(&self, score: &AggregateScore) -> i64 { + (score.normalized_score.clamp(0.0, 1.0) * 10000.0) as i64 + } +} diff --git a/challenges/term-challenge-wasm/src/tasks.rs b/challenges/term-challenge-wasm/src/tasks.rs new file mode 100644 index 00000000..9a6ae9a4 --- /dev/null +++ b/challenges/term-challenge-wasm/src/tasks.rs @@ -0,0 +1,83 @@ +use alloc::string::String; +use alloc::vec::Vec; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum Difficulty { + Easy, + #[default] + Medium, + Hard, +} + +impl Difficulty { + pub fn weight(self) -> f64 { + match self { + Difficulty::Easy => 1.0, + Difficulty::Medium => 2.0, + Difficulty::Hard => 3.0, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TaskDefinition { + pub id: String, + pub name: String, + pub difficulty: Difficulty, + #[serde(default)] + pub tags: Vec, + #[serde(default = "default_timeout")] + pub timeout_secs: f64, +} + +fn default_timeout() -> f64 { + 180.0 +} + +impl Default for TaskDefinition { + fn default() -> Self { + Self { + id: String::new(), + name: String::new(), + difficulty: Difficulty::default(), + tags: Vec::new(), + timeout_secs: default_timeout(), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TaskResult { + pub task_id: String, + pub passed: bool, + #[serde(default)] + pub score: f64, + #[serde(default)] + pub execution_time_ms: u64, + #[serde(default)] + pub error: Option, +} + +impl TaskResult { + pub fn success(task_id: String, execution_time_ms: u64) -> Self { + Self { + task_id, + passed: true, + score: 1.0, + execution_time_ms, + error: None, + } + } + + pub fn failure(task_id: String, execution_time_ms: u64, error: String) -> Self { + Self { + task_id, + passed: false, + score: 0.0, + execution_time_ms, + error: Some(error), + } + } +} diff --git a/crates/challenge-sdk-wasm/src/lib.rs b/crates/challenge-sdk-wasm/src/lib.rs index e2ed5021..52be5b7c 100644 --- a/crates/challenge-sdk-wasm/src/lib.rs +++ b/crates/challenge-sdk-wasm/src/lib.rs @@ -27,6 +27,9 @@ pub fn pack_ptr_len(ptr: i32, len: i32) -> i64 { /// Register a [`Challenge`] implementation and export the required WASM ABI /// functions (`evaluate`, `validate`, `get_name`, `get_version`, and `alloc`). /// +/// Pass the type and a const initializer expression. For unit structs the +/// expression is simply the struct name. +/// /// # Usage /// /// ```ignore @@ -41,12 +44,12 @@ pub fn pack_ptr_len(ptr: i32, len: i32) -> i64 { /// fn validate(&self, input: EvaluationInput) -> bool { true } /// } /// -/// platform_challenge_sdk_wasm::register_challenge!(MyChallenge); +/// platform_challenge_sdk_wasm::register_challenge!(MyChallenge, MyChallenge); /// ``` #[macro_export] macro_rules! register_challenge { - ($ty:ty) => { - static _CHALLENGE: $ty = <$ty as Default>::default(); + ($ty:ty, $init:expr) => { + static _CHALLENGE: $ty = $init; #[no_mangle] pub extern "C" fn evaluate(agent_ptr: i32, agent_len: i32) -> i64 {