feat(challenges): add term-challenge WASM crate with evaluation logic

Cargo.toml

@@ -19,6 +19,7 @@ members = [
     "bins/mock-subtensor",
     "tests",
     "challenges/term-challenge",
+    "challenges/term-challenge-wasm",
 ]
 # Note: Challenges are in separate repositories and import platform-challenge-sdk as a git dependency
 # Note: WASM runtime removed - updates via git, version checked at handshake

challenges/term-challenge-wasm/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "term-challenge-wasm"
+version.workspace = true
+edition.workspace = true
+description = "Terminal Benchmark Challenge ported to WASM (wasm32-unknown-unknown)"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+platform-challenge-sdk-wasm = { path = "../../crates/challenge-sdk-wasm" }
+serde = { version = "1.0", default-features = false, features = ["derive", "alloc"] }
+bincode = { version = "1.3", default-features = false }

challenges/term-challenge-wasm/src/lib.rs

@@ -0,0 +1,145 @@
+#![no_std]
+
+extern crate alloc;
+
+mod scoring;
+mod types;
+
+use alloc::string::String;
+use alloc::vec::Vec;
+use platform_challenge_sdk_wasm::host_functions::host_http_post;
+use platform_challenge_sdk_wasm::{Challenge, EvaluationInput, EvaluationOutput};
+
+use crate::scoring::{calculate_aggregate, format_summary, to_weight};
+use crate::types::{ChallengeParams, LlmJudgeRequest, LlmJudgeResponse, Submission, TaskResult};
+
+pub struct TermChallenge;
+
+impl Default for TermChallenge {
+    fn default() -> Self {
+        Self
+    }
+}
+
+impl TermChallenge {
+    const fn new() -> Self {
+        Self
+    }
+
+    fn try_llm_judge(url: &str, result: &TaskResult, instruction: &str) -> Option<f64> {
+        let request = LlmJudgeRequest {
+            task_id: result.task_id.clone(),
+            instruction: String::from(instruction),
+            agent_output: result.agent_output.clone(),
+            test_output: result.test_output.clone(),
+        };
+
+        let url_bytes = url.as_bytes();
+        let body = match bincode::serialize(&request) {
+            Ok(b) => b,
+            Err(_) => return None,
+        };
+
+        let response_bytes = match host_http_post(url_bytes, &body) {
+            Ok(b) => b,
+            Err(_) => return None,
+        };
+
+        let judge_resp: LlmJudgeResponse = match bincode::deserialize(&response_bytes) {
+            Ok(r) => r,
+            Err(_) => return None,
+        };
+
+        Some(judge_resp.score.clamp(0.0, 1.0))
+    }
+}
+
+impl Challenge for TermChallenge {
+    fn name(&self) -> &'static str {
+        "term-challenge"
+    }
+
+    fn version(&self) -> &'static str {
+        "2.0.0"
+    }
+
+    fn evaluate(&self, input: EvaluationInput) -> EvaluationOutput {
+        let submission: Submission = match bincode::deserialize(&input.agent_data) {
+            Ok(s) => s,
+            Err(_) => return EvaluationOutput::failure("failed to deserialize submission"),
+        };
+
+        let params: ChallengeParams = match bincode::deserialize(&input.params) {
+            Ok(p) => p,
+            Err(_) => return EvaluationOutput::failure("failed to deserialize challenge params"),
+        };
+
+        if submission.task_results.is_empty() {
+            return EvaluationOutput::failure("submission contains no task results");
+        }
+
+        if submission.task_results.len() != params.tasks.len() {
+            return EvaluationOutput::failure("task result count does not match task definitions");
+        }
+
+        let mut results: Vec<TaskResult> = submission.task_results;
+
+        if let Some(ref url) = params.llm_judge_url {
+            for (result, task) in results.iter_mut().zip(params.tasks.iter()) {
+                if !result.passed {
+                    continue;
+                }
+                if let Some(llm_score) = Self::try_llm_judge(url, result, &task.name) {
+                    result.score = llm_score;
+                    if llm_score < 0.5 {
+                        result.passed = false;
+                    }
+                }
+            }
+        }
+
+        let aggregate = calculate_aggregate(&params.tasks, &results);
+        let weight = to_weight(&aggregate);
+        let score = (weight * 10000.0) as i64;
+        let message = format_summary(&aggregate);
+
+        EvaluationOutput::success(score, &message)
+    }
+
+    fn validate(&self, input: EvaluationInput) -> bool {
+        let submission: Submission = match bincode::deserialize(&input.agent_data) {
+            Ok(s) => s,
+            Err(_) => return false,
+        };
+
+        let params: ChallengeParams = match bincode::deserialize(&input.params) {
+            Ok(p) => p,
+            Err(_) => return false,
+        };
+
+        if submission.agent_hash.is_empty() || submission.miner_hotkey.is_empty() {
+            return false;
+        }
+
+        if submission.task_results.is_empty() {
+            return false;
+        }
+
+        if submission.task_results.len() != params.tasks.len() {
+            return false;
+        }
+
+        for result in &submission.task_results {
+            if result.task_id.is_empty() {
+                return false;
+            }
+            if !(0.0..=1.0).contains(&result.score) {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+platform_challenge_sdk_wasm::register_challenge!(TermChallenge, TermChallenge::new());

challenges/term-challenge-wasm/src/scoring.rs

@@ -0,0 +1,125 @@
+use alloc::string::String;
+use core::fmt::Write as _;
+
+use crate::types::{Difficulty, DifficultyStats, TaskDefinition, TaskResult};
+
+#[allow(dead_code)]
+pub struct AggregateScore {
+    pub tasks_passed: u32,
+    pub tasks_failed: u32,
+    pub pass_rate: f64,
+    pub normalized_score: f64,
+    pub total_execution_time_ms: u64,
+    pub easy_stats: DifficultyStats,
+    pub medium_stats: DifficultyStats,
+    pub hard_stats: DifficultyStats,
+}
+
+impl AggregateScore {
+    pub fn total_tasks(&self) -> u32 {
+        self.tasks_passed + self.tasks_failed
+    }
+}
+
+#[allow(dead_code)]
+pub fn score_task(result: &TaskResult) -> f64 {
+    if result.passed {
+        1.0
+    } else {
+        0.0
+    }
+}
+
+pub fn calculate_aggregate(tasks: &[TaskDefinition], results: &[TaskResult]) -> AggregateScore {
+    let mut passed: u32 = 0;
+    let mut failed: u32 = 0;
+    let mut total_execution_time_ms: u64 = 0;
+    let mut easy = DifficultyStats {
+        total: 0,
+        passed: 0,
+    };
+    let mut medium = DifficultyStats {
+        total: 0,
+        passed: 0,
+    };
+    let mut hard = DifficultyStats {
+        total: 0,
+        passed: 0,
+    };
+
+    for (task, result) in tasks.iter().zip(results.iter()) {
+        if result.passed {
+            passed += 1;
+        } else {
+            failed += 1;
+        }
+
+        total_execution_time_ms = total_execution_time_ms.saturating_add(result.execution_time_ms);
+
+        let stats = match task.difficulty {
+            Difficulty::Easy => &mut easy,
+            Difficulty::Medium => &mut medium,
+            Difficulty::Hard => &mut hard,
+        };
+        stats.total += 1;
+        if result.passed {
+            stats.passed += 1;
+        }
+    }
+
+    let total = passed + failed;
+    let pass_rate = if total > 0 {
+        passed as f64 / total as f64
+    } else {
+        0.0
+    };
+
+    AggregateScore {
+        tasks_passed: passed,
+        tasks_failed: failed,
+        pass_rate,
+        normalized_score: pass_rate,
+        total_execution_time_ms,
+        easy_stats: easy,
+        medium_stats: medium,
+        hard_stats: hard,
+    }
+}
+
+pub fn to_weight(score: &AggregateScore) -> f64 {
+    score.pass_rate.clamp(0.0, 1.0)
+}
+
+pub fn format_summary(score: &AggregateScore) -> String {
+    let mut msg = String::new();
+    let _ = write!(
+        msg,
+        "passed={}/{} rate={:.2}%",
+        score.tasks_passed,
+        score.total_tasks(),
+        score.pass_rate * 100.0,
+    );
+    if score.easy_stats.total > 0 {
+        let _ = write!(
+            msg,
+            " easy={}/{}",
+            score.easy_stats.passed, score.easy_stats.total,
+        );
+    }
+    if score.medium_stats.total > 0 {
+        let _ = write!(
+            msg,
+            " med={}/{}",
+            score.medium_stats.passed, score.medium_stats.total,
+        );
+    }
+    if score.hard_stats.total > 0 {
+        let _ = write!(
+            msg,
+            " hard={}/{}",
+            score.hard_stats.passed, score.hard_stats.total,
+        );
+    }
+    let _ = write!(msg, " time={}ms", score.total_execution_time_ms);
+    msg
+}

challenges/term-challenge-wasm/src/types.rs

@@ -0,0 +1,83 @@
+use alloc::string::String;
+use alloc::vec::Vec;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Difficulty {
+    Easy,
+    Medium,
+    Hard,
+}
+
+#[allow(dead_code)]
+impl Difficulty {
+    pub fn weight(self) -> f64 {
+        match self {
+            Difficulty::Easy => 1.0,
+            Difficulty::Medium => 2.0,
+            Difficulty::Hard => 3.0,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct TaskDefinition {
+    pub id: String,
+    pub name: String,
+    pub difficulty: Difficulty,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct TaskResult {
+    pub task_id: String,
+    pub passed: bool,
+    pub score: f64,
+    pub execution_time_ms: u64,
+    pub test_output: String,
+    pub agent_output: String,
+    pub error: Option<String>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct ChallengeParams {
+    pub tasks: Vec<TaskDefinition>,
+    pub llm_judge_url: Option<String>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Submission {
+    pub agent_hash: String,
+    pub miner_hotkey: String,
+    pub task_results: Vec<TaskResult>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct DifficultyStats {
+    pub total: u32,
+    pub passed: u32,
+}
+
+#[allow(dead_code)]
+impl DifficultyStats {
+    pub fn pass_rate(&self) -> f64 {
+        if self.total > 0 {
+            self.passed as f64 / self.total as f64
+        } else {
+            0.0
+        }
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct LlmJudgeRequest {
+    pub task_id: String,
+    pub instruction: String,
+    pub agent_output: String,
+    pub test_output: String,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct LlmJudgeResponse {
+    pub score: f64,
+    pub reasoning: String,
+}