diff --git a/bins/validator-node/src/main.rs b/bins/validator-node/src/main.rs index 60721271..2b4e7987 100644 --- a/bins/validator-node/src/main.rs +++ b/bins/validator-node/src/main.rs @@ -39,6 +39,26 @@ use wasm_executor::{WasmChallengeExecutor, WasmExecutorConfig}; /// Storage key for persisted chain state const STATE_STORAGE_KEY: &str = "chain_state"; +/// Maximum length for user-provided strings logged from P2P messages +const MAX_LOG_FIELD_LEN: usize = 256; +const JOB_TIMEOUT_MS: i64 = 300_000; + +/// Sanitize a user-provided string for safe logging. +/// +/// Replaces control characters (newlines, tabs, ANSI escapes) with spaces +/// and truncates to `MAX_LOG_FIELD_LEN` to prevent log injection attacks. +fn sanitize_for_log(s: &str) -> String { + let truncated = if s.len() > MAX_LOG_FIELD_LEN { + &s[..MAX_LOG_FIELD_LEN] + } else { + s + }; + truncated + .chars() + .map(|c| if c.is_control() { ' ' } else { c }) + .collect() +} + // ==================== Shutdown Handler ==================== /// Handles graceful shutdown with state persistence @@ -121,7 +141,7 @@ impl ShutdownHandler { // ==================== CLI ==================== -#[derive(Parser, Debug)] +#[derive(Parser)] #[command(name = "validator-node")] #[command(about = "Platform Validator - Decentralized P2P Architecture")] struct Args { @@ -178,6 +198,28 @@ struct Args { wasm_fuel_limit: Option, } +impl std::fmt::Debug for Args { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Args") + .field( + "secret_key", + &self.secret_key.as_ref().map(|_| "[REDACTED]"), + ) + .field("data_dir", &self.data_dir) + .field("listen_addr", &self.listen_addr) + .field("bootstrap", &self.bootstrap) + .field("subtensor_endpoint", &self.subtensor_endpoint) + .field("netuid", &self.netuid) + .field("version_key", &self.version_key) + .field("no_bittensor", &self.no_bittensor) + .field("wasm_module_dir", &self.wasm_module_dir) + .field("wasm_max_memory", &self.wasm_max_memory) + .field("wasm_enable_fuel", &self.wasm_enable_fuel) + .field("wasm_fuel_limit", &self.wasm_fuel_limit) + .finish() + } +} + // ==================== Main ==================== #[tokio::main] @@ -377,6 +419,7 @@ async fn main() -> Result<()> { fuel_limit: args.wasm_fuel_limit, storage_host_config: wasm_runtime_interface::StorageHostConfig::default(), storage_backend: std::sync::Arc::new(wasm_runtime_interface::InMemoryStorageBackend::new()), + chutes_api_key: None, }) { Ok(executor) => { info!( @@ -847,7 +890,7 @@ async fn handle_network_event( challenge_id: assignment.challenge_id, assigned_validator: assignment.assigned_validator, assigned_at: assignment.timestamp, - timeout_at: assignment.timestamp + 300_000, + timeout_at: assignment.timestamp + JOB_TIMEOUT_MS, status: JobStatus::Pending, }; state_manager.apply(|state| { @@ -952,6 +995,31 @@ async fn handle_network_event( "Received storage vote" ); } + P2PMessage::ReviewAssignment(msg) => { + debug!( + submission_id = %msg.submission_id, + assigner = %msg.assigner.to_hex(), + assigned_count = msg.assigned_validators.len(), + "Received review assignment" + ); + } + P2PMessage::ReviewDecline(msg) => { + let safe_reason = sanitize_for_log(&msg.reason); + debug!( + submission_id = %msg.submission_id, + validator = %msg.validator.to_hex(), + reason = %safe_reason, + "Received review decline" + ); + } + P2PMessage::ReviewResult(msg) => { + debug!( + submission_id = %msg.submission_id, + validator = %msg.validator.to_hex(), + score = msg.score, + "Received review result" + ); + } }, NetworkEvent::PeerConnected(peer_id) => { info!("Peer connected: {}", peer_id); diff --git a/bins/validator-node/src/wasm_executor.rs b/bins/validator-node/src/wasm_executor.rs index 0ce00dfe..6f488472 100644 --- a/bins/validator-node/src/wasm_executor.rs +++ b/bins/validator-node/src/wasm_executor.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use bincode::Options; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -7,11 +8,13 @@ use std::sync::Arc; use std::time::Instant; use tracing::{debug, info}; use wasm_runtime_interface::{ - ConsensusPolicy, ExecPolicy, InMemoryStorageBackend, InstanceConfig, NetworkHostFunctions, - NetworkPolicy, RuntimeConfig, SandboxHostFunctions, SandboxPolicy, StorageBackend, + ConsensusPolicy, ExecPolicy, InMemoryStorageBackend, InstanceConfig, LlmPolicy, + NetworkHostFunctions, NetworkPolicy, RuntimeConfig, SandboxPolicy, StorageBackend, StorageHostConfig, TerminalPolicy, TimePolicy, WasmModule, WasmRuntime, WasmRuntimeError, }; +const MAX_EVALUATION_OUTPUT_SIZE: u64 = 64 * 1024 * 1024; + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct EvaluationInput { pub agent_data: Vec, @@ -65,6 +68,22 @@ pub struct WasmExecutorConfig { pub fuel_limit: Option, pub storage_host_config: StorageHostConfig, pub storage_backend: Arc, + pub chutes_api_key: Option, +} + +impl std::fmt::Debug for WasmExecutorConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WasmExecutorConfig") + .field("module_dir", &self.module_dir) + .field("max_memory_bytes", &self.max_memory_bytes) + .field("enable_fuel", &self.enable_fuel) + .field("fuel_limit", &self.fuel_limit) + .field( + "chutes_api_key", + &self.chutes_api_key.as_ref().map(|_| "[REDACTED]"), + ) + .finish() + } } impl Default for WasmExecutorConfig { @@ -76,6 +95,7 @@ impl Default for WasmExecutorConfig { fuel_limit: None, storage_host_config: StorageHostConfig::default(), storage_backend: Arc::new(InMemoryStorageBackend::new()), + chutes_api_key: None, } } } @@ -164,7 +184,6 @@ impl WasmChallengeExecutor { bincode::serialize(&input).context("Failed to serialize EvaluationInput")?; let network_host_fns = Arc::new(NetworkHostFunctions::all()); - let _sandbox_host_fns = Arc::new(SandboxHostFunctions::all()); let instance_config = InstanceConfig { network_policy: network_policy.clone(), @@ -186,6 +205,10 @@ impl WasmChallengeExecutor { fixed_timestamp_ms: None, consensus_policy: ConsensusPolicy::default(), terminal_policy: TerminalPolicy::default(), + llm_policy: match &self.config.chutes_api_key { + Some(key) => LlmPolicy::with_api_key(key.clone()), + None => LlmPolicy::default(), + }, ..Default::default() }; @@ -229,7 +252,11 @@ impl WasmChallengeExecutor { anyhow::anyhow!("Failed to read evaluation output from WASM memory: {}", e) })?; - let output: EvaluationOutput = bincode::deserialize(&output_bytes) + let output: EvaluationOutput = bincode::DefaultOptions::new() + .with_limit(MAX_EVALUATION_OUTPUT_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() + .deserialize(&output_bytes) .context("Failed to deserialize EvaluationOutput from WASM module")?; let fuel_consumed = match (initial_fuel, instance.fuel_remaining()) { @@ -287,7 +314,6 @@ impl WasmChallengeExecutor { bincode::serialize(&input).context("Failed to serialize EvaluationInput")?; let network_host_fns = Arc::new(NetworkHostFunctions::all()); - let _sandbox_host_fns = Arc::new(SandboxHostFunctions::all()); let instance_config = InstanceConfig { network_policy: network_policy.clone(), @@ -309,6 +335,10 @@ impl WasmChallengeExecutor { fixed_timestamp_ms: None, consensus_policy: ConsensusPolicy::default(), terminal_policy: TerminalPolicy::default(), + llm_policy: match &self.config.chutes_api_key { + Some(key) => LlmPolicy::with_api_key(key.clone()), + None => LlmPolicy::default(), + }, ..Default::default() }; @@ -419,6 +449,10 @@ impl WasmChallengeExecutor { fixed_timestamp_ms: None, consensus_policy: ConsensusPolicy::default(), terminal_policy: TerminalPolicy::default(), + llm_policy: match &self.config.chutes_api_key { + Some(key) => LlmPolicy::with_api_key(key.clone()), + None => LlmPolicy::default(), + }, ..Default::default() }; @@ -439,7 +473,9 @@ impl WasmChallengeExecutor { let result_data = if out_ptr > 0 && out_len > 0 { instance .read_memory(out_ptr as usize, out_len as usize) - .unwrap_or_default() + .map_err(|e| { + anyhow::anyhow!("failed to read WASM memory for get_tasks output: {}", e) + })? } else { Vec::new() }; @@ -498,6 +534,10 @@ impl WasmChallengeExecutor { fixed_timestamp_ms: None, consensus_policy: ConsensusPolicy::default(), terminal_policy: TerminalPolicy::default(), + llm_policy: match &self.config.chutes_api_key { + Some(key) => LlmPolicy::with_api_key(key.clone()), + None => LlmPolicy::default(), + }, ..Default::default() }; diff --git a/challenges/term-challenge-wasm/src/lib.rs b/challenges/term-challenge-wasm/src/lib.rs index 90590ffe..36d60798 100644 --- a/challenges/term-challenge-wasm/src/lib.rs +++ b/challenges/term-challenge-wasm/src/lib.rs @@ -3,26 +3,94 @@ extern crate alloc; mod scoring; +mod tasks; mod types; -use alloc::string::String; use alloc::vec::Vec; -use platform_challenge_sdk_wasm::host_functions::host_http_post; +use bincode::Options; +use platform_challenge_sdk_wasm::host_functions::{ + host_consensus_get_epoch, host_http_post, host_storage_get, host_storage_set, +}; use platform_challenge_sdk_wasm::{Challenge, EvaluationInput, EvaluationOutput}; use crate::scoring::{calculate_aggregate, format_summary, to_weight}; -use crate::types::{ChallengeParams, LlmJudgeRequest, LlmJudgeResponse, Submission, TaskResult}; +use crate::types::{ + ChallengeParams, DatasetSelection, LlmJudgeRequest, LlmJudgeResponse, Submission, TaskResult, +}; + +use alloc::string::String; + +const MAX_SUBMISSION_SIZE: u64 = 64 * 1024 * 1024; +const MAX_PARAMS_SIZE: u64 = 4 * 1024 * 1024; +const MAX_LLM_RESPONSE_SIZE: u64 = 1024 * 1024; +const MAX_TASKS: usize = 256; +const EPOCH_RATE_LIMIT: u64 = 3; +const LLM_JUDGE_PASS_THRESHOLD: f64 = 0.5; +const SCORE_SCALE_FACTOR: f64 = 10_000.0; + +fn bincode_options_submission() -> impl Options { + bincode::DefaultOptions::new() + .with_limit(MAX_SUBMISSION_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() +} + +fn bincode_options_params() -> impl Options { + bincode::DefaultOptions::new() + .with_limit(MAX_PARAMS_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() +} + +fn bincode_options_llm() -> impl Options { + bincode::DefaultOptions::new() + .with_limit(MAX_LLM_RESPONSE_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() +} + +fn validate_task_result(result: &TaskResult) -> bool { + if result.task_id.is_empty() { + return false; + } + if !result.score.is_finite() || !(0.0..=1.0).contains(&result.score) { + return false; + } + true +} + +fn last_submission_key(miner_hotkey: &str) -> Vec { + let mut key = Vec::from(b"last_submission:" as &[u8]); + key.extend_from_slice(miner_hotkey.as_bytes()); + key +} + +fn get_last_submission_epoch(miner_hotkey: &str) -> Option { + let key = last_submission_key(miner_hotkey); + let data = host_storage_get(&key).ok()?; + if data.len() < 8 { + return None; + } + let mut buf = [0u8; 8]; + buf.copy_from_slice(&data[..8]); + Some(u64::from_le_bytes(buf)) +} + +fn set_last_submission_epoch(miner_hotkey: &str, epoch: u64) { + let key = last_submission_key(miner_hotkey); + let _ = host_storage_set(&key, &epoch.to_le_bytes()); +} -pub struct TermChallenge; +pub struct TermChallengeWasm; -impl Default for TermChallenge { +impl Default for TermChallengeWasm { fn default() -> Self { Self } } -impl TermChallenge { - const fn new() -> Self { +impl TermChallengeWasm { + pub const fn new() -> Self { Self } @@ -33,57 +101,63 @@ impl TermChallenge { agent_output: result.agent_output.clone(), test_output: result.test_output.clone(), }; - let url_bytes = url.as_bytes(); - let body = match bincode::serialize(&request) { + let body = match bincode_options_llm().serialize(&request) { Ok(b) => b, Err(_) => return None, }; - let response_bytes = match host_http_post(url_bytes, &body) { Ok(b) => b, Err(_) => return None, }; - - let judge_resp: LlmJudgeResponse = match bincode::deserialize(&response_bytes) { + let judge_resp: LlmJudgeResponse = match bincode_options_llm().deserialize(&response_bytes) + { Ok(r) => r, Err(_) => return None, }; - + if !judge_resp.score.is_finite() { + return None; + } Some(judge_resp.score.clamp(0.0, 1.0)) } } -impl Challenge for TermChallenge { +impl Challenge for TermChallengeWasm { fn name(&self) -> &'static str { "term-challenge" } fn version(&self) -> &'static str { - "2.0.0" + "4.0.0" } fn evaluate(&self, input: EvaluationInput) -> EvaluationOutput { - let submission: Submission = match bincode::deserialize(&input.agent_data) { - Ok(s) => s, - Err(_) => return EvaluationOutput::failure("failed to deserialize submission"), - }; - - let params: ChallengeParams = match bincode::deserialize(&input.params) { + let submission: Submission = + match bincode_options_submission().deserialize(&input.agent_data) { + Ok(s) => s, + Err(_) => return EvaluationOutput::failure("failed to deserialize submission"), + }; + let params: ChallengeParams = match bincode_options_params().deserialize(&input.params) { Ok(p) => p, Err(_) => return EvaluationOutput::failure("failed to deserialize challenge params"), }; - if submission.task_results.is_empty() { return EvaluationOutput::failure("submission contains no task results"); } - + if submission.task_results.len() > MAX_TASKS { + return EvaluationOutput::failure("submission exceeds maximum task count"); + } if submission.task_results.len() != params.tasks.len() { return EvaluationOutput::failure("task result count does not match task definitions"); } - + for result in &submission.task_results { + if !validate_task_result(result) { + return EvaluationOutput::failure( + "invalid task result: bad score or empty task_id", + ); + } + } let mut results: Vec = submission.task_results; - if let Some(ref url) = params.llm_judge_url { for (result, task) in results.iter_mut().zip(params.tasks.iter()) { if !result.passed { @@ -91,55 +165,85 @@ impl Challenge for TermChallenge { } if let Some(llm_score) = Self::try_llm_judge(url, result, &task.name) { result.score = llm_score; - if llm_score < 0.5 { + if llm_score < LLM_JUDGE_PASS_THRESHOLD { result.passed = false; } } } } - let aggregate = calculate_aggregate(¶ms.tasks, &results); let weight = to_weight(&aggregate); - let score = (weight * 10000.0) as i64; + let score = (weight * SCORE_SCALE_FACTOR) as i64; let message = format_summary(&aggregate); - + set_last_submission_epoch(&submission.miner_hotkey, submission.epoch); EvaluationOutput::success(score, &message) } fn validate(&self, input: EvaluationInput) -> bool { - let submission: Submission = match bincode::deserialize(&input.agent_data) { - Ok(s) => s, - Err(_) => return false, - }; - - let params: ChallengeParams = match bincode::deserialize(&input.params) { + let submission: Submission = + match bincode_options_submission().deserialize(&input.agent_data) { + Ok(s) => s, + Err(_) => return false, + }; + let params: ChallengeParams = match bincode_options_params().deserialize(&input.params) { Ok(p) => p, Err(_) => return false, }; - if submission.agent_hash.is_empty() || submission.miner_hotkey.is_empty() { return false; } - + if submission.signature.is_empty() { + return false; + } + if submission.package_zip.is_empty() { + return false; + } + if submission.basilica_instance.is_empty() + || submission.executor_url.is_empty() + || submission.executor_token.is_empty() + { + return false; + } + let current_epoch = host_consensus_get_epoch(); + if current_epoch >= 0 { + if let Some(last_epoch) = get_last_submission_epoch(&submission.miner_hotkey) { + let current = current_epoch as u64; + if current < last_epoch.saturating_add(EPOCH_RATE_LIMIT) { + return false; + } + } + } if submission.task_results.is_empty() { return false; } - + if submission.task_results.len() > MAX_TASKS { + return false; + } if submission.task_results.len() != params.tasks.len() { return false; } - for result in &submission.task_results { - if result.task_id.is_empty() { - return false; - } - if !(0.0..=1.0).contains(&result.score) { + if !validate_task_result(result) { return false; } } - true } + + fn tasks(&self) -> Vec { + match tasks::get_active_dataset() { + Some(task_defs) => bincode_options_params() + .serialize(&task_defs) + .unwrap_or_default(), + None => Vec::new(), + } + } + + fn configure(&self, config: &[u8]) { + if let Ok(selection) = bincode_options_params().deserialize::(config) { + let _ = tasks::store_dataset(&selection); + } + } } -platform_challenge_sdk_wasm::register_challenge!(TermChallenge, TermChallenge::new()); +platform_challenge_sdk_wasm::register_challenge!(TermChallengeWasm, TermChallengeWasm::new()); diff --git a/challenges/term-challenge-wasm/src/scoring.rs b/challenges/term-challenge-wasm/src/scoring.rs index f4808973..eb9047ab 100644 --- a/challenges/term-challenge-wasm/src/scoring.rs +++ b/challenges/term-challenge-wasm/src/scoring.rs @@ -3,12 +3,10 @@ use core::fmt::Write as _; use crate::types::{Difficulty, DifficultyStats, TaskDefinition, TaskResult}; -#[allow(dead_code)] pub struct AggregateScore { pub tasks_passed: u32, pub tasks_failed: u32, pub pass_rate: f64, - pub normalized_score: f64, pub total_execution_time_ms: u64, pub easy_stats: DifficultyStats, pub medium_stats: DifficultyStats, @@ -17,16 +15,7 @@ pub struct AggregateScore { impl AggregateScore { pub fn total_tasks(&self) -> u32 { - self.tasks_passed + self.tasks_failed - } -} - -#[allow(dead_code)] -pub fn score_task(result: &TaskResult) -> f64 { - if result.passed { - 1.0 - } else { - 0.0 + self.tasks_passed.saturating_add(self.tasks_failed) } } @@ -53,9 +42,7 @@ pub fn calculate_aggregate(tasks: &[TaskDefinition], results: &[TaskResult]) -> } else { failed += 1; } - total_execution_time_ms = total_execution_time_ms.saturating_add(result.execution_time_ms); - let stats = match task.difficulty { Difficulty::Easy => &mut easy, Difficulty::Medium => &mut medium, @@ -78,7 +65,6 @@ pub fn calculate_aggregate(tasks: &[TaskDefinition], results: &[TaskResult]) -> tasks_passed: passed, tasks_failed: failed, pass_rate, - normalized_score: pass_rate, total_execution_time_ms, easy_stats: easy, medium_stats: medium, @@ -103,21 +89,21 @@ pub fn format_summary(score: &AggregateScore) -> String { let _ = write!( msg, " easy={}/{}", - score.easy_stats.passed, score.easy_stats.total, + score.easy_stats.passed, score.easy_stats.total ); } if score.medium_stats.total > 0 { let _ = write!( msg, " med={}/{}", - score.medium_stats.passed, score.medium_stats.total, + score.medium_stats.passed, score.medium_stats.total ); } if score.hard_stats.total > 0 { let _ = write!( msg, " hard={}/{}", - score.hard_stats.passed, score.hard_stats.total, + score.hard_stats.passed, score.hard_stats.total ); } let _ = write!(msg, " time={}ms", score.total_execution_time_ms); diff --git a/challenges/term-challenge-wasm/src/tasks.rs b/challenges/term-challenge-wasm/src/tasks.rs new file mode 100644 index 00000000..192fbb7f --- /dev/null +++ b/challenges/term-challenge-wasm/src/tasks.rs @@ -0,0 +1,58 @@ +use crate::types::{DatasetSelection, TaskDefinition}; +use alloc::vec::Vec; +use bincode::Options; +use platform_challenge_sdk_wasm::host_functions::{host_storage_get, host_storage_set}; + +const ACTIVE_DATASET_KEY: &[u8] = b"active_dataset"; +const DATASET_HISTORY_KEY: &[u8] = b"dataset_history"; +const MAX_DATASET_SIZE: u64 = 8 * 1024 * 1024; +const MAX_DATASET_HISTORY: usize = 100; + +fn bincode_options_dataset() -> impl Options { + bincode::DefaultOptions::new() + .with_limit(MAX_DATASET_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() +} + +pub fn get_active_dataset() -> Option> { + let data = host_storage_get(ACTIVE_DATASET_KEY).ok()?; + if data.is_empty() { + return None; + } + bincode_options_dataset().deserialize(&data).ok() +} + +pub fn store_dataset(selection: &DatasetSelection) -> bool { + let data = match bincode_options_dataset().serialize(selection) { + Ok(d) => d, + Err(_) => return false, + }; + if host_storage_set(ACTIVE_DATASET_KEY, &data).is_err() { + return false; + } + let _ = append_dataset_history(selection); + true +} + +fn append_dataset_history(selection: &DatasetSelection) -> bool { + let mut history: Vec = host_storage_get(DATASET_HISTORY_KEY) + .ok() + .and_then(|d| { + if d.is_empty() { + None + } else { + bincode_options_dataset().deserialize(&d).ok() + } + }) + .unwrap_or_default(); + history.push(selection.clone()); + if history.len() > MAX_DATASET_HISTORY { + history.drain(0..history.len() - MAX_DATASET_HISTORY); + } + let data = match bincode_options_dataset().serialize(&history) { + Ok(d) => d, + Err(_) => return false, + }; + host_storage_set(DATASET_HISTORY_KEY, &data).is_ok() +} diff --git a/challenges/term-challenge-wasm/src/types.rs b/challenges/term-challenge-wasm/src/types.rs index 1dd80304..6c218aec 100644 --- a/challenges/term-challenge-wasm/src/types.rs +++ b/challenges/term-challenge-wasm/src/types.rs @@ -1,5 +1,6 @@ use alloc::string::String; use alloc::vec::Vec; +use core::fmt; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -9,22 +10,14 @@ pub enum Difficulty { Hard, } -#[allow(dead_code)] -impl Difficulty { - pub fn weight(self) -> f64 { - match self { - Difficulty::Easy => 1.0, - Difficulty::Medium => 2.0, - Difficulty::Hard => 3.0, - } - } -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct TaskDefinition { pub id: String, pub name: String, + pub repo: String, + pub base_commit: String, pub difficulty: Difficulty, + pub timeout_secs: u64, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -42,32 +35,51 @@ pub struct TaskResult { pub struct ChallengeParams { pub tasks: Vec, pub llm_judge_url: Option, + pub decay_params: Option, + pub active_dataset: Option>, } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct Submission { pub agent_hash: String, pub miner_hotkey: String, + pub signature: Vec, + pub epoch: u64, + pub package_zip: Vec, + pub basilica_instance: String, + pub executor_url: String, + pub executor_token: String, pub task_results: Vec, } +impl fmt::Debug for Submission { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Submission") + .field("agent_hash", &self.agent_hash) + .field("miner_hotkey", &self.miner_hotkey) + .field( + "signature", + &format_args!("[{} bytes]", self.signature.len()), + ) + .field("epoch", &self.epoch) + .field( + "package_zip", + &format_args!("[{} bytes]", self.package_zip.len()), + ) + .field("basilica_instance", &self.basilica_instance) + .field("executor_url", &self.executor_url) + .field("executor_token", &"[REDACTED]") + .field("task_results", &self.task_results) + .finish() + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct DifficultyStats { pub total: u32, pub passed: u32, } -#[allow(dead_code)] -impl DifficultyStats { - pub fn pass_rate(&self) -> f64 { - if self.total > 0 { - self.passed as f64 / self.total as f64 - } else { - 0.0 - } - } -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct LlmJudgeRequest { pub task_id: String, @@ -81,3 +93,27 @@ pub struct LlmJudgeResponse { pub score: f64, pub reasoning: String, } + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DecayParams { + pub grace_period_hours: u64, + pub half_life_hours: u64, + pub min_multiplier: f64, +} + +impl Default for DecayParams { + fn default() -> Self { + Self { + grace_period_hours: 72, + half_life_hours: 24, + min_multiplier: 0.0, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DatasetSelection { + pub tasks: Vec, + pub selected_at_epoch: u64, + pub dataset_hash: String, +} diff --git a/challenges/term-challenge/src/evaluation.rs b/challenges/term-challenge/src/evaluation.rs deleted file mode 100644 index 36e6a0c6..00000000 --- a/challenges/term-challenge/src/evaluation.rs +++ /dev/null @@ -1,76 +0,0 @@ -use alloc::string::String; -use alloc::vec::Vec; - -use platform_challenge_sdk_wasm::types::{EvaluationInput, EvaluationOutput}; - -use crate::scoring::score_submission; -use crate::types::{EvalParams, Submission}; - -pub fn evaluate(input: EvaluationInput) -> EvaluationOutput { - let params: EvalParams = match bincode::deserialize(&input.params) { - Ok(p) => p, - Err(_) => return EvaluationOutput::failure("failed to deserialize evaluation params"), - }; - - let submission: Submission = match bincode::deserialize(&input.agent_data) { - Ok(s) => s, - Err(_) => return EvaluationOutput::failure("failed to deserialize agent submission"), - }; - - if submission.tasks.is_empty() { - return EvaluationOutput::failure("submission contains no task results"); - } - - let expected_ids: Vec<&str> = params.tasks.iter().map(|t| t.id.as_str()).collect(); - for result in &submission.tasks { - if !expected_ids.contains(&result.task_id.as_str()) { - return EvaluationOutput::failure("submission contains unknown task id"); - } - } - - let (score, metrics) = score_submission(&submission); - - let message = match bincode::serialize(&metrics) { - Ok(encoded) => { - let _ = host_storage_set_metrics(&encoded); - alloc::format!( - "passed={}/{} rate={:.2}%", - metrics.tasks_passed, - metrics.total_tasks, - metrics.pass_rate * 100.0 - ) - } - Err(_) => String::from("scored"), - }; - - EvaluationOutput { - score, - valid: true, - message, - metrics: None, - details: None, - } -} - -fn host_storage_set_metrics(data: &[u8]) -> Result<(), i32> { - let key = b"term_eval_metrics"; - platform_challenge_sdk_wasm::host_functions::host_storage_set(key, data) -} - -pub fn validate(input: &EvaluationInput) -> bool { - if input.agent_data.is_empty() { - return false; - } - if input.params.is_empty() { - return false; - } - let _params: EvalParams = match bincode::deserialize(&input.params) { - Ok(p) => p, - Err(_) => return false, - }; - let submission: Submission = match bincode::deserialize(&input.agent_data) { - Ok(s) => s, - Err(_) => return false, - }; - !submission.tasks.is_empty() -} diff --git a/challenges/term-challenge/src/lib.rs b/challenges/term-challenge/src/lib.rs index d675fa81..36d60798 100644 --- a/challenges/term-challenge/src/lib.rs +++ b/challenges/term-challenge/src/lib.rs @@ -2,43 +2,248 @@ extern crate alloc; -mod evaluation; -pub mod scoring; -pub mod tasks; -pub mod types; +mod scoring; +mod tasks; +mod types; +use alloc::vec::Vec; +use bincode::Options; +use platform_challenge_sdk_wasm::host_functions::{ + host_consensus_get_epoch, host_http_post, host_storage_get, host_storage_set, +}; use platform_challenge_sdk_wasm::{Challenge, EvaluationInput, EvaluationOutput}; -pub struct TermChallenge; +use crate::scoring::{calculate_aggregate, format_summary, to_weight}; +use crate::types::{ + ChallengeParams, DatasetSelection, LlmJudgeRequest, LlmJudgeResponse, Submission, TaskResult, +}; -impl TermChallenge { - const fn new() -> Self { - Self +use alloc::string::String; + +const MAX_SUBMISSION_SIZE: u64 = 64 * 1024 * 1024; +const MAX_PARAMS_SIZE: u64 = 4 * 1024 * 1024; +const MAX_LLM_RESPONSE_SIZE: u64 = 1024 * 1024; +const MAX_TASKS: usize = 256; +const EPOCH_RATE_LIMIT: u64 = 3; +const LLM_JUDGE_PASS_THRESHOLD: f64 = 0.5; +const SCORE_SCALE_FACTOR: f64 = 10_000.0; + +fn bincode_options_submission() -> impl Options { + bincode::DefaultOptions::new() + .with_limit(MAX_SUBMISSION_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() +} + +fn bincode_options_params() -> impl Options { + bincode::DefaultOptions::new() + .with_limit(MAX_PARAMS_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() +} + +fn bincode_options_llm() -> impl Options { + bincode::DefaultOptions::new() + .with_limit(MAX_LLM_RESPONSE_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() +} + +fn validate_task_result(result: &TaskResult) -> bool { + if result.task_id.is_empty() { + return false; + } + if !result.score.is_finite() || !(0.0..=1.0).contains(&result.score) { + return false; + } + true +} + +fn last_submission_key(miner_hotkey: &str) -> Vec { + let mut key = Vec::from(b"last_submission:" as &[u8]); + key.extend_from_slice(miner_hotkey.as_bytes()); + key +} + +fn get_last_submission_epoch(miner_hotkey: &str) -> Option { + let key = last_submission_key(miner_hotkey); + let data = host_storage_get(&key).ok()?; + if data.len() < 8 { + return None; } + let mut buf = [0u8; 8]; + buf.copy_from_slice(&data[..8]); + Some(u64::from_le_bytes(buf)) } -impl Default for TermChallenge { +fn set_last_submission_epoch(miner_hotkey: &str, epoch: u64) { + let key = last_submission_key(miner_hotkey); + let _ = host_storage_set(&key, &epoch.to_le_bytes()); +} + +pub struct TermChallengeWasm; + +impl Default for TermChallengeWasm { fn default() -> Self { - Self::new() + Self + } +} + +impl TermChallengeWasm { + pub const fn new() -> Self { + Self + } + + fn try_llm_judge(url: &str, result: &TaskResult, instruction: &str) -> Option { + let request = LlmJudgeRequest { + task_id: result.task_id.clone(), + instruction: String::from(instruction), + agent_output: result.agent_output.clone(), + test_output: result.test_output.clone(), + }; + let url_bytes = url.as_bytes(); + let body = match bincode_options_llm().serialize(&request) { + Ok(b) => b, + Err(_) => return None, + }; + let response_bytes = match host_http_post(url_bytes, &body) { + Ok(b) => b, + Err(_) => return None, + }; + let judge_resp: LlmJudgeResponse = match bincode_options_llm().deserialize(&response_bytes) + { + Ok(r) => r, + Err(_) => return None, + }; + if !judge_resp.score.is_finite() { + return None; + } + Some(judge_resp.score.clamp(0.0, 1.0)) } } -impl Challenge for TermChallenge { +impl Challenge for TermChallengeWasm { fn name(&self) -> &'static str { "term-challenge" } fn version(&self) -> &'static str { - "0.1.0" + "4.0.0" } fn evaluate(&self, input: EvaluationInput) -> EvaluationOutput { - evaluation::evaluate(input) + let submission: Submission = + match bincode_options_submission().deserialize(&input.agent_data) { + Ok(s) => s, + Err(_) => return EvaluationOutput::failure("failed to deserialize submission"), + }; + let params: ChallengeParams = match bincode_options_params().deserialize(&input.params) { + Ok(p) => p, + Err(_) => return EvaluationOutput::failure("failed to deserialize challenge params"), + }; + if submission.task_results.is_empty() { + return EvaluationOutput::failure("submission contains no task results"); + } + if submission.task_results.len() > MAX_TASKS { + return EvaluationOutput::failure("submission exceeds maximum task count"); + } + if submission.task_results.len() != params.tasks.len() { + return EvaluationOutput::failure("task result count does not match task definitions"); + } + for result in &submission.task_results { + if !validate_task_result(result) { + return EvaluationOutput::failure( + "invalid task result: bad score or empty task_id", + ); + } + } + let mut results: Vec = submission.task_results; + if let Some(ref url) = params.llm_judge_url { + for (result, task) in results.iter_mut().zip(params.tasks.iter()) { + if !result.passed { + continue; + } + if let Some(llm_score) = Self::try_llm_judge(url, result, &task.name) { + result.score = llm_score; + if llm_score < LLM_JUDGE_PASS_THRESHOLD { + result.passed = false; + } + } + } + } + let aggregate = calculate_aggregate(¶ms.tasks, &results); + let weight = to_weight(&aggregate); + let score = (weight * SCORE_SCALE_FACTOR) as i64; + let message = format_summary(&aggregate); + set_last_submission_epoch(&submission.miner_hotkey, submission.epoch); + EvaluationOutput::success(score, &message) } fn validate(&self, input: EvaluationInput) -> bool { - evaluation::validate(&input) + let submission: Submission = + match bincode_options_submission().deserialize(&input.agent_data) { + Ok(s) => s, + Err(_) => return false, + }; + let params: ChallengeParams = match bincode_options_params().deserialize(&input.params) { + Ok(p) => p, + Err(_) => return false, + }; + if submission.agent_hash.is_empty() || submission.miner_hotkey.is_empty() { + return false; + } + if submission.signature.is_empty() { + return false; + } + if submission.package_zip.is_empty() { + return false; + } + if submission.basilica_instance.is_empty() + || submission.executor_url.is_empty() + || submission.executor_token.is_empty() + { + return false; + } + let current_epoch = host_consensus_get_epoch(); + if current_epoch >= 0 { + if let Some(last_epoch) = get_last_submission_epoch(&submission.miner_hotkey) { + let current = current_epoch as u64; + if current < last_epoch.saturating_add(EPOCH_RATE_LIMIT) { + return false; + } + } + } + if submission.task_results.is_empty() { + return false; + } + if submission.task_results.len() > MAX_TASKS { + return false; + } + if submission.task_results.len() != params.tasks.len() { + return false; + } + for result in &submission.task_results { + if !validate_task_result(result) { + return false; + } + } + true + } + + fn tasks(&self) -> Vec { + match tasks::get_active_dataset() { + Some(task_defs) => bincode_options_params() + .serialize(&task_defs) + .unwrap_or_default(), + None => Vec::new(), + } + } + + fn configure(&self, config: &[u8]) { + if let Ok(selection) = bincode_options_params().deserialize::(config) { + let _ = tasks::store_dataset(&selection); + } } } -platform_challenge_sdk_wasm::register_challenge!(TermChallenge, TermChallenge::new()); +platform_challenge_sdk_wasm::register_challenge!(TermChallengeWasm, TermChallengeWasm::new()); diff --git a/challenges/term-challenge/src/scoring.rs b/challenges/term-challenge/src/scoring.rs index 61c53ddb..eb9047ab 100644 --- a/challenges/term-challenge/src/scoring.rs +++ b/challenges/term-challenge/src/scoring.rs @@ -1,35 +1,111 @@ -use crate::types::{EvalMetrics, Submission}; +use alloc::string::String; +use core::fmt::Write as _; -pub fn score_submission(submission: &Submission) -> (i64, EvalMetrics) { - let total = submission.tasks.len() as u32; +use crate::types::{Difficulty, DifficultyStats, TaskDefinition, TaskResult}; + +pub struct AggregateScore { + pub tasks_passed: u32, + pub tasks_failed: u32, + pub pass_rate: f64, + pub total_execution_time_ms: u64, + pub easy_stats: DifficultyStats, + pub medium_stats: DifficultyStats, + pub hard_stats: DifficultyStats, +} + +impl AggregateScore { + pub fn total_tasks(&self) -> u32 { + self.tasks_passed.saturating_add(self.tasks_failed) + } +} + +pub fn calculate_aggregate(tasks: &[TaskDefinition], results: &[TaskResult]) -> AggregateScore { let mut passed: u32 = 0; let mut failed: u32 = 0; let mut total_execution_time_ms: u64 = 0; + let mut easy = DifficultyStats { + total: 0, + passed: 0, + }; + let mut medium = DifficultyStats { + total: 0, + passed: 0, + }; + let mut hard = DifficultyStats { + total: 0, + passed: 0, + }; - for result in &submission.tasks { + for (task, result) in tasks.iter().zip(results.iter()) { if result.passed { passed += 1; } else { failed += 1; } total_execution_time_ms = total_execution_time_ms.saturating_add(result.execution_time_ms); + let stats = match task.difficulty { + Difficulty::Easy => &mut easy, + Difficulty::Medium => &mut medium, + Difficulty::Hard => &mut hard, + }; + stats.total += 1; + if result.passed { + stats.passed += 1; + } } + let total = passed + failed; let pass_rate = if total > 0 { passed as f64 / total as f64 } else { 0.0 }; - let score = (pass_rate.clamp(0.0, 1.0) * 10_000.0) as i64; - - let metrics = EvalMetrics { + AggregateScore { tasks_passed: passed, tasks_failed: failed, - total_tasks: total, pass_rate, total_execution_time_ms, - }; + easy_stats: easy, + medium_stats: medium, + hard_stats: hard, + } +} - (score, metrics) +pub fn to_weight(score: &AggregateScore) -> f64 { + score.pass_rate.clamp(0.0, 1.0) +} + +pub fn format_summary(score: &AggregateScore) -> String { + let mut msg = String::new(); + let _ = write!( + msg, + "passed={}/{} rate={:.2}%", + score.tasks_passed, + score.total_tasks(), + score.pass_rate * 100.0, + ); + if score.easy_stats.total > 0 { + let _ = write!( + msg, + " easy={}/{}", + score.easy_stats.passed, score.easy_stats.total + ); + } + if score.medium_stats.total > 0 { + let _ = write!( + msg, + " med={}/{}", + score.medium_stats.passed, score.medium_stats.total + ); + } + if score.hard_stats.total > 0 { + let _ = write!( + msg, + " hard={}/{}", + score.hard_stats.passed, score.hard_stats.total + ); + } + let _ = write!(msg, " time={}ms", score.total_execution_time_ms); + msg } diff --git a/challenges/term-challenge/src/tasks.rs b/challenges/term-challenge/src/tasks.rs index b4524eb4..192fbb7f 100644 --- a/challenges/term-challenge/src/tasks.rs +++ b/challenges/term-challenge/src/tasks.rs @@ -1,147 +1,58 @@ -use alloc::string::String; -use alloc::vec; +use crate::types::{DatasetSelection, TaskDefinition}; use alloc::vec::Vec; +use bincode::Options; +use platform_challenge_sdk_wasm::host_functions::{host_storage_get, host_storage_set}; -use crate::types::{Difficulty, TaskDefinition}; +const ACTIVE_DATASET_KEY: &[u8] = b"active_dataset"; +const DATASET_HISTORY_KEY: &[u8] = b"dataset_history"; +const MAX_DATASET_SIZE: u64 = 8 * 1024 * 1024; +const MAX_DATASET_HISTORY: usize = 100; -fn task( - id: &str, - name: &str, - instruction: &str, - difficulty: Difficulty, - timeout_secs: u64, - docker_image: &str, - test_script: &str, -) -> TaskDefinition { - TaskDefinition { - id: String::from(id), - name: String::from(name), - instruction: String::from(instruction), - difficulty, - timeout_secs, - docker_image: String::from(docker_image), - test_script: String::from(test_script), - } +fn bincode_options_dataset() -> impl Options { + bincode::DefaultOptions::new() + .with_limit(MAX_DATASET_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() } -pub fn builtin_tasks() -> Vec { - vec![ - task( - "create-file", - "Create a File", - "Create a file called /app/hello.txt containing the text 'Hello, World!'", - Difficulty::Easy, - 60, - "ubuntu:22.04", - "#!/bin/bash\ntest -f /app/hello.txt && grep -q 'Hello, World!' /app/hello.txt", - ), - task( - "list-processes", - "List Running Processes", - "Write the output of `ps aux` to /app/processes.txt", - Difficulty::Easy, - 60, - "ubuntu:22.04", - "#!/bin/bash\ntest -f /app/processes.txt && [ -s /app/processes.txt ]", - ), - task( - "find-largest-file", - "Find Largest File", - "Find the largest file in /var/log and write its name to /app/largest.txt", - Difficulty::Medium, - 120, - "ubuntu:22.04", - "#!/bin/bash\ntest -f /app/largest.txt && [ -s /app/largest.txt ]", - ), - task( - "setup-nginx", - "Setup Nginx Config", - "Install nginx and configure it to serve static files from /var/www/html on port 8080. \ - Create an index.html with 'Welcome' as content.", - Difficulty::Medium, - 180, - "ubuntu:22.04", - "#!/bin/bash\ntest -f /var/www/html/index.html && grep -q 'Welcome' /var/www/html/index.html", - ), - task( - "parse-json-log", - "Parse JSON Logs", - "Parse the JSON log file at /app/input.log, extract all entries with level 'ERROR', \ - and write them to /app/errors.json as a JSON array.", - Difficulty::Medium, - 120, - "ubuntu:22.04", - "#!/bin/bash\ntest -f /app/errors.json && python3 -c \"import json; d=json.load(open('/app/errors.json')); assert isinstance(d, list)\"", - ), - task( - "create-user", - "Create System User", - "Create a new system user called 'appuser' with home directory /home/appuser and bash as default shell.", - Difficulty::Easy, - 60, - "ubuntu:22.04", - "#!/bin/bash\nid appuser && [ -d /home/appuser ] && getent passwd appuser | grep -q '/bin/bash'", - ), - task( - "compress-directory", - "Compress Directory", - "Create a tar.gz archive of /app/data directory and save it as /app/data.tar.gz. \ - The archive must preserve directory structure.", - Difficulty::Easy, - 60, - "ubuntu:22.04", - "#!/bin/bash\ntest -f /app/data.tar.gz && tar tzf /app/data.tar.gz | head -1", - ), - task( - "setup-cron", - "Setup Cron Job", - "Create a cron job that runs '/usr/local/bin/cleanup.sh' every day at 3:00 AM as root. \ - Write the crontab entry to /app/crontab.txt as well.", - Difficulty::Medium, - 120, - "ubuntu:22.04", - "#!/bin/bash\ntest -f /app/crontab.txt && grep -q '0 3' /app/crontab.txt && grep -q 'cleanup.sh' /app/crontab.txt", - ), - task( - "docker-compose", - "Write Docker Compose", - "Write a docker-compose.yml at /app/docker-compose.yml that defines two services: \ - 'web' using nginx:latest on port 80 and 'db' using postgres:15 with POSTGRES_PASSWORD=secret.", - Difficulty::Hard, - 180, - "ubuntu:22.04", - "#!/bin/bash\ntest -f /app/docker-compose.yml && grep -q 'nginx' /app/docker-compose.yml && grep -q 'postgres' /app/docker-compose.yml", - ), - task( - "iptables-rule", - "Configure Firewall Rule", - "Write an iptables rule set to /app/rules.sh that blocks all incoming traffic on port 22 \ - except from 10.0.0.0/8, and allows all outgoing traffic.", - Difficulty::Hard, - 180, - "ubuntu:22.04", - "#!/bin/bash\ntest -f /app/rules.sh && grep -q 'iptables' /app/rules.sh && grep -q '10.0.0.0' /app/rules.sh", - ), - ] +pub fn get_active_dataset() -> Option> { + let data = host_storage_get(ACTIVE_DATASET_KEY).ok()?; + if data.is_empty() { + return None; + } + bincode_options_dataset().deserialize(&data).ok() } -pub fn select_tasks(seed: u64, count: usize) -> Vec { - let all = builtin_tasks(); - if count >= all.len() { - return all; +pub fn store_dataset(selection: &DatasetSelection) -> bool { + let data = match bincode_options_dataset().serialize(selection) { + Ok(d) => d, + Err(_) => return false, + }; + if host_storage_set(ACTIVE_DATASET_KEY, &data).is_err() { + return false; } - let mut indices: Vec = (0..all.len()).collect(); - let mut rng = seed; - for i in (1..indices.len()).rev() { - rng = rng - .wrapping_mul(6364136223846793005) - .wrapping_add(1442695040888963407); - let j = (rng >> 33) as usize % (i + 1); - indices.swap(i, j); + let _ = append_dataset_history(selection); + true +} + +fn append_dataset_history(selection: &DatasetSelection) -> bool { + let mut history: Vec = host_storage_get(DATASET_HISTORY_KEY) + .ok() + .and_then(|d| { + if d.is_empty() { + None + } else { + bincode_options_dataset().deserialize(&d).ok() + } + }) + .unwrap_or_default(); + history.push(selection.clone()); + if history.len() > MAX_DATASET_HISTORY { + history.drain(0..history.len() - MAX_DATASET_HISTORY); } - indices - .into_iter() - .take(count) - .map(|i| all[i].clone()) - .collect() + let data = match bincode_options_dataset().serialize(&history) { + Ok(d) => d, + Err(_) => return false, + }; + host_storage_set(DATASET_HISTORY_KEY, &data).is_ok() } diff --git a/challenges/term-challenge/src/types.rs b/challenges/term-challenge/src/types.rs index 987d2269..6c218aec 100644 --- a/challenges/term-challenge/src/types.rs +++ b/challenges/term-challenge/src/types.rs @@ -1,5 +1,6 @@ use alloc::string::String; use alloc::vec::Vec; +use core::fmt; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -13,11 +14,10 @@ pub enum Difficulty { pub struct TaskDefinition { pub id: String, pub name: String, - pub instruction: String, + pub repo: String, + pub base_commit: String, pub difficulty: Difficulty, pub timeout_secs: u64, - pub docker_image: String, - pub test_script: String, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -26,25 +26,94 @@ pub struct TaskResult { pub passed: bool, pub score: f64, pub execution_time_ms: u64, - pub output: String, + pub test_output: String, + pub agent_output: String, pub error: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ChallengeParams { + pub tasks: Vec, + pub llm_judge_url: Option, + pub decay_params: Option, + pub active_dataset: Option>, +} + +#[derive(Clone, Serialize, Deserialize)] pub struct Submission { - pub tasks: Vec, + pub agent_hash: String, + pub miner_hotkey: String, + pub signature: Vec, + pub epoch: u64, + pub package_zip: Vec, + pub basilica_instance: String, + pub executor_url: String, + pub executor_token: String, + pub task_results: Vec, +} + +impl fmt::Debug for Submission { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Submission") + .field("agent_hash", &self.agent_hash) + .field("miner_hotkey", &self.miner_hotkey) + .field( + "signature", + &format_args!("[{} bytes]", self.signature.len()), + ) + .field("epoch", &self.epoch) + .field( + "package_zip", + &format_args!("[{} bytes]", self.package_zip.len()), + ) + .field("basilica_instance", &self.basilica_instance) + .field("executor_url", &self.executor_url) + .field("executor_token", &"[REDACTED]") + .field("task_results", &self.task_results) + .finish() + } } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct EvalParams { - pub tasks: Vec, +pub struct DifficultyStats { + pub total: u32, + pub passed: u32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LlmJudgeRequest { + pub task_id: String, + pub instruction: String, + pub agent_output: String, + pub test_output: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LlmJudgeResponse { + pub score: f64, + pub reasoning: String, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub struct EvalMetrics { - pub tasks_passed: u32, - pub tasks_failed: u32, - pub total_tasks: u32, - pub pass_rate: f64, - pub total_execution_time_ms: u64, +pub struct DecayParams { + pub grace_period_hours: u64, + pub half_life_hours: u64, + pub min_multiplier: f64, +} + +impl Default for DecayParams { + fn default() -> Self { + Self { + grace_period_hours: 72, + half_life_hours: 24, + min_multiplier: 0.0, + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct DatasetSelection { + pub tasks: Vec, + pub selected_at_epoch: u64, + pub dataset_hash: String, } diff --git a/crates/challenge-sdk-wasm/src/host_functions.rs b/crates/challenge-sdk-wasm/src/host_functions.rs index fe11096d..f0562338 100644 --- a/crates/challenge-sdk-wasm/src/host_functions.rs +++ b/crates/challenge-sdk-wasm/src/host_functions.rs @@ -1,6 +1,10 @@ use alloc::vec; use alloc::vec::Vec; +const RESPONSE_BUF_SMALL: usize = 4096; +const RESPONSE_BUF_MEDIUM: usize = 64 * 1024; +const RESPONSE_BUF_LARGE: usize = 256 * 1024; + #[link(wasm_import_module = "platform_network")] extern "C" { fn http_get(req_ptr: i32, req_len: i32, resp_ptr: i32, resp_len: i32) -> i32; @@ -25,7 +29,7 @@ extern "C" { } pub fn host_http_get(request: &[u8]) -> Result, i32> { - let mut response_buf = vec![0u8; 65536]; + let mut response_buf = vec![0u8; RESPONSE_BUF_MEDIUM]; let status = unsafe { http_get( request.as_ptr() as i32, @@ -42,7 +46,7 @@ pub fn host_http_get(request: &[u8]) -> Result, i32> { } pub fn host_http_post(request: &[u8], body: &[u8]) -> Result, i32> { - let mut response_buf = vec![0u8; 65536]; + let mut response_buf = vec![0u8; RESPONSE_BUF_MEDIUM]; let status = unsafe { http_post( request.as_ptr() as i32, @@ -60,7 +64,7 @@ pub fn host_http_post(request: &[u8], body: &[u8]) -> Result, i32> { } pub fn host_dns_resolve(request: &[u8]) -> Result, i32> { - let mut response_buf = vec![0u8; 4096]; + let mut response_buf = vec![0u8; RESPONSE_BUF_SMALL]; let status = unsafe { dns_resolve( request.as_ptr() as i32, @@ -76,7 +80,7 @@ pub fn host_dns_resolve(request: &[u8]) -> Result, i32> { } pub fn host_storage_get(key: &[u8]) -> Result, i32> { - let mut value_buf = vec![0u8; 65536]; + let mut value_buf = vec![0u8; RESPONSE_BUF_MEDIUM]; let status = unsafe { storage_get( key.as_ptr() as i32, @@ -107,7 +111,7 @@ pub fn host_storage_set(key: &[u8], value: &[u8]) -> Result<(), i32> { } pub fn host_terminal_exec(request: &[u8]) -> Result, i32> { - let mut result_buf = vec![0u8; 262144]; + let mut result_buf = vec![0u8; RESPONSE_BUF_LARGE]; let status = unsafe { terminal_exec( request.as_ptr() as i32, @@ -124,7 +128,7 @@ pub fn host_terminal_exec(request: &[u8]) -> Result, i32> { } pub fn host_read_file(path: &[u8]) -> Result, i32> { - let mut buf = vec![0u8; 262144]; + let mut buf = vec![0u8; RESPONSE_BUF_LARGE]; let status = unsafe { terminal_read_file( path.as_ptr() as i32, @@ -156,7 +160,7 @@ pub fn host_write_file(path: &[u8], data: &[u8]) -> Result<(), i32> { } pub fn host_list_dir(path: &[u8]) -> Result, i32> { - let mut buf = vec![0u8; 65536]; + let mut buf = vec![0u8; RESPONSE_BUF_MEDIUM]; let status = unsafe { terminal_list_dir( path.as_ptr() as i32, @@ -192,7 +196,7 @@ extern "C" { } pub fn host_sandbox_exec(request: &[u8]) -> Result, i32> { - let mut response_buf = vec![0u8; 262144]; + let mut response_buf = vec![0u8; RESPONSE_BUF_LARGE]; let status = unsafe { sandbox_exec( request.as_ptr() as i32, @@ -216,6 +220,33 @@ pub fn host_log(level: u8, msg: &str) { unsafe { log_message(level as i32, msg.as_ptr() as i32, msg.len() as i32) } } +#[link(wasm_import_module = "platform_llm")] +extern "C" { + fn llm_chat_completion(req_ptr: i32, req_len: i32, resp_ptr: i32, resp_len: i32) -> i32; + fn llm_is_available() -> i32; +} + +pub fn host_llm_chat_completion(request: &[u8]) -> Result, i32> { + let mut response_buf = vec![0u8; RESPONSE_BUF_LARGE]; + let status = unsafe { + llm_chat_completion( + request.as_ptr() as i32, + request.len() as i32, + response_buf.as_mut_ptr() as i32, + response_buf.len() as i32, + ) + }; + if status < 0 { + return Err(status); + } + response_buf.truncate(status as usize); + Ok(response_buf) +} + +pub fn host_llm_is_available() -> bool { + unsafe { llm_is_available() == 1 } +} + #[link(wasm_import_module = "platform_consensus")] extern "C" { fn consensus_get_epoch() -> i64; @@ -232,7 +263,7 @@ pub fn host_consensus_get_epoch() -> i64 { } pub fn host_consensus_get_validators() -> Result, i32> { - let mut buf = vec![0u8; 65536]; + let mut buf = vec![0u8; RESPONSE_BUF_MEDIUM]; let status = unsafe { consensus_get_validators(buf.as_mut_ptr() as i32, buf.len() as i32) }; if status < 0 { return Err(status); @@ -250,7 +281,7 @@ pub fn host_consensus_propose_weight(uid: i32, weight: i32) -> Result<(), i32> { } pub fn host_consensus_get_votes() -> Result, i32> { - let mut buf = vec![0u8; 65536]; + let mut buf = vec![0u8; RESPONSE_BUF_MEDIUM]; let status = unsafe { consensus_get_votes(buf.as_mut_ptr() as i32, buf.len() as i32) }; if status < 0 { return Err(status); diff --git a/crates/challenge-sdk-wasm/src/lib.rs b/crates/challenge-sdk-wasm/src/lib.rs index 8d278dd4..796552e9 100644 --- a/crates/challenge-sdk-wasm/src/lib.rs +++ b/crates/challenge-sdk-wasm/src/lib.rs @@ -4,9 +4,11 @@ extern crate alloc; pub mod alloc_impl; pub mod host_functions; +pub mod llm_types; pub mod term_types; pub mod types; +pub use llm_types::{LlmMessage, LlmRequest, LlmResponse, LlmUsage}; pub use term_types::*; pub use types::{ score_f64_scaled, SandboxExecRequest, SandboxExecResponse, TaskDefinition, TaskResult, diff --git a/crates/challenge-sdk-wasm/src/llm_types.rs b/crates/challenge-sdk-wasm/src/llm_types.rs new file mode 100644 index 00000000..76a2ec77 --- /dev/null +++ b/crates/challenge-sdk-wasm/src/llm_types.rs @@ -0,0 +1,30 @@ +use alloc::string::String; +use alloc::vec::Vec; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LlmRequest { + pub model: String, + pub messages: Vec, + pub max_tokens: u32, + pub temperature: f32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LlmMessage { + pub role: String, + pub content: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LlmResponse { + pub content: String, + pub usage: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LlmUsage { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, +} diff --git a/crates/p2p-consensus/src/consensus.rs b/crates/p2p-consensus/src/consensus.rs index 71b08dc8..1438e22d 100644 --- a/crates/p2p-consensus/src/consensus.rs +++ b/crates/p2p-consensus/src/consensus.rs @@ -749,12 +749,16 @@ impl ConsensusEngine { let (last_prepared_sequence, prepared_proof) = { let round = self.current_round.read(); if let Some(r) = round.as_ref() { - if r.phase >= ConsensusPhase::Prepared && r.pre_prepare.is_some() { - let proof = PreparedProof { - pre_prepare: r.pre_prepare.clone().unwrap(), - prepares: r.prepares.values().cloned().collect(), - }; - (Some(r.sequence), Some(proof)) + if let Some(pre_prepare) = r.pre_prepare.clone() { + if r.phase >= ConsensusPhase::Prepared { + let proof = PreparedProof { + pre_prepare, + prepares: r.prepares.values().cloned().collect(), + }; + (Some(r.sequence), Some(proof)) + } else { + (None, None) + } } else { (None, None) } diff --git a/crates/p2p-consensus/src/messages.rs b/crates/p2p-consensus/src/messages.rs index 15106a11..efcabd3f 100644 --- a/crates/p2p-consensus/src/messages.rs +++ b/crates/p2p-consensus/src/messages.rs @@ -3,9 +3,12 @@ //! Defines all message types used for inter-validator communication //! over the libp2p gossipsub network. +use bincode::Options; use platform_core::{ChallengeId, Hotkey}; use serde::{Deserialize, Serialize}; +pub const MAX_P2P_MESSAGE_SIZE: u64 = 16 * 1024 * 1024; + /// Unique identifier for a consensus round pub type RoundId = u64; @@ -51,6 +54,11 @@ pub enum P2PMessage { ChallengeUpdate(ChallengeUpdateMessage), StorageProposal(StorageProposalMessage), StorageVote(StorageVoteMessage), + + // Review assignment + ReviewAssignment(ReviewAssignmentMessage), + ReviewDecline(ReviewDeclineMessage), + ReviewResult(ReviewResultMessage), } impl P2PMessage { @@ -61,7 +69,18 @@ impl P2PMessage { /// Deserialize message from bytes pub fn from_bytes(bytes: &[u8]) -> Result { - bincode::deserialize(bytes) + if bytes.len() as u64 > MAX_P2P_MESSAGE_SIZE { + return Err(Box::new(bincode::ErrorKind::Custom(format!( + "message exceeds maximum size: {} > {}", + bytes.len(), + MAX_P2P_MESSAGE_SIZE + )))); + } + bincode::DefaultOptions::new() + .with_limit(MAX_P2P_MESSAGE_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() + .deserialize(bytes) } /// Get the message type name for logging @@ -91,6 +110,9 @@ impl P2PMessage { P2PMessage::ChallengeUpdate(_) => "ChallengeUpdate", P2PMessage::StorageProposal(_) => "StorageProposal", P2PMessage::StorageVote(_) => "StorageVote", + P2PMessage::ReviewAssignment(_) => "ReviewAssignment", + P2PMessage::ReviewDecline(_) => "ReviewDecline", + P2PMessage::ReviewResult(_) => "ReviewResult", } } } @@ -601,6 +623,72 @@ pub struct StorageVoteMessage { pub signature: Vec, } +// ============================================================================ +// Review Assignment Messages +// ============================================================================ + +/// Type of review to be performed +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum ReviewType { + /// LLM-based code review + Llm, + /// AST-based structural review + Ast, +} + +/// Assignment of review validators for a submission +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReviewAssignmentMessage { + /// Submission being reviewed + pub submission_id: String, + /// Type of review + pub review_type: ReviewType, + /// Validators assigned to perform the review + pub assigned_validators: Vec, + /// Deterministic seed used for selection + pub seed: [u8; 32], + /// Assignment timestamp + pub timestamp: i64, + /// Validator that made the assignment + pub assigner: Hotkey, + /// Assigner's signature + pub signature: Vec, +} + +/// Decline message when a validator cannot perform a review +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReviewDeclineMessage { + /// Submission being reviewed + pub submission_id: String, + /// Validator declining the review + pub validator: Hotkey, + /// Reason for declining + pub reason: String, + /// Decline timestamp + pub timestamp: i64, + /// Validator's signature + pub signature: Vec, +} + +/// Result of a review from a validator +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReviewResultMessage { + /// Submission being reviewed + pub submission_id: String, + /// Validator that performed the review + pub validator: Hotkey, + /// Type of review performed + pub review_type: ReviewType, + /// Review score (0.0 to 1.0) + pub score: f64, + /// Detailed review output + pub details: String, + /// Result timestamp + pub timestamp: i64, + /// Validator's signature + pub signature: Vec, +} + // ============================================================================ // Signed Message Wrapper // ============================================================================ diff --git a/crates/p2p-consensus/src/network.rs b/crates/p2p-consensus/src/network.rs index 5d80c3b3..6de620fb 100644 --- a/crates/p2p-consensus/src/network.rs +++ b/crates/p2p-consensus/src/network.rs @@ -4,8 +4,9 @@ //! Provides the networking foundation for PBFT consensus. use crate::config::P2PConfig; -use crate::messages::{P2PMessage, SignedP2PMessage, WeightVoteMessage}; +use crate::messages::{P2PMessage, SignedP2PMessage, WeightVoteMessage, MAX_P2P_MESSAGE_SIZE}; use crate::validator::ValidatorSet; +use bincode::Options; use libp2p::{ gossipsub::{self, IdentTopic, MessageAuthenticity, MessageId, ValidationMode}, identify, @@ -169,7 +170,6 @@ pub struct P2PNetwork { /// Peer mapping peer_mapping: Arc, /// Reference to validator set - #[allow(dead_code)] validator_set: Arc, /// Event sender #[allow(dead_code)] @@ -443,8 +443,12 @@ impl P2PNetwork { source: PeerId, data: &[u8], ) -> Result { - let signed: SignedP2PMessage = - bincode::deserialize(data).map_err(|e| NetworkError::Serialization(e.to_string()))?; + let signed: SignedP2PMessage = bincode::DefaultOptions::new() + .with_limit(MAX_P2P_MESSAGE_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() + .deserialize(data) + .map_err(|e| NetworkError::Serialization(e.to_string()))?; // Verify signature first if !self.verify_message(&signed) { @@ -711,6 +715,9 @@ fn expected_signer(message: &P2PMessage) -> Option<&Hotkey> { P2PMessage::ChallengeUpdate(msg) => Some(&msg.updater), P2PMessage::StorageProposal(msg) => Some(&msg.proposer), P2PMessage::StorageVote(msg) => Some(&msg.voter), + P2PMessage::ReviewAssignment(msg) => Some(&msg.assigner), + P2PMessage::ReviewDecline(msg) => Some(&msg.validator), + P2PMessage::ReviewResult(msg) => Some(&msg.validator), } } diff --git a/crates/p2p-consensus/src/state.rs b/crates/p2p-consensus/src/state.rs index c22b5191..8667cde1 100644 --- a/crates/p2p-consensus/src/state.rs +++ b/crates/p2p-consensus/src/state.rs @@ -4,6 +4,7 @@ //! evaluations, weights, and validator information. use crate::messages::{MerkleNode, MerkleProof, SequenceNumber}; +use bincode::Options; use parking_lot::RwLock; use platform_core::{hash_data, ChallengeId, Hotkey, SignedMessage}; use serde::{Deserialize, Serialize}; @@ -12,6 +13,8 @@ use std::collections::HashMap; use thiserror::Error; use tracing::{debug, info, warn}; +const MAX_STATE_DESERIALIZATION_SIZE: u64 = 256 * 1024 * 1024; + /// Errors related to state operations #[derive(Error, Debug)] pub enum StateError { @@ -183,6 +186,27 @@ pub struct ChainState { /// Storage roots per challenge #[serde(default)] pub challenge_storage_roots: HashMap, + /// Review assignments per submission + #[serde(default)] + pub review_assignments: HashMap>, +} + +/// Record of a review assignment +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReviewRecord { + pub submission_id: String, + pub review_type: crate::messages::ReviewType, + pub assigned_validators: Vec, + pub results: HashMap, + pub created_at: i64, +} + +/// Single review result entry +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ReviewResultEntry { + pub score: f64, + pub details: String, + pub timestamp: i64, } impl Default for ChainState { @@ -206,6 +230,7 @@ impl Default for ChainState { active_jobs: HashMap::new(), task_progress: HashMap::new(), challenge_storage_roots: HashMap::new(), + review_assignments: HashMap::new(), } } } @@ -298,7 +323,19 @@ impl ChainState { /// Deserialize state from bytes pub fn from_bytes(bytes: &[u8]) -> Result { - bincode::deserialize(bytes).map_err(|e| StateError::Serialization(e.to_string())) + if bytes.len() as u64 > MAX_STATE_DESERIALIZATION_SIZE { + return Err(StateError::Serialization(format!( + "state data exceeds maximum size: {} > {}", + bytes.len(), + MAX_STATE_DESERIALIZATION_SIZE + ))); + } + bincode::DefaultOptions::new() + .with_limit(MAX_STATE_DESERIALIZATION_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() + .deserialize(bytes) + .map_err(|e| StateError::Serialization(e.to_string())) } /// Add or update a validator @@ -683,6 +720,52 @@ impl ChainState { } removed } + + pub fn assign_review(&mut self, record: ReviewRecord) { + self.review_assignments + .entry(record.submission_id.clone()) + .or_default() + .push(record); + self.increment_sequence(); + } + + pub fn add_review_result( + &mut self, + submission_id: &str, + validator: &Hotkey, + score: f64, + details: String, + ) -> bool { + if !score.is_finite() || !(0.0..=1.0).contains(&score) { + warn!( + score, + submission_id, + "Rejecting review result with invalid score (must be finite and in 0.0..=1.0)" + ); + return false; + } + if let Some(reviews) = self.review_assignments.get_mut(submission_id) { + for review in reviews.iter_mut() { + if review.assigned_validators.contains(validator) { + review.results.insert( + validator.clone(), + ReviewResultEntry { + score, + details, + timestamp: chrono::Utc::now().timestamp_millis(), + }, + ); + self.update_hash(); + return true; + } + } + } + false + } + + pub fn get_review_status(&self, submission_id: &str) -> Option<&Vec> { + self.review_assignments.get(submission_id) + } } /// Thread-safe state manager diff --git a/crates/storage/src/dynamic.rs b/crates/storage/src/dynamic.rs index 134e50d6..8f8126b4 100644 --- a/crates/storage/src/dynamic.rs +++ b/crates/storage/src/dynamic.rs @@ -14,6 +14,7 @@ use crate::types::{ NamespaceStats, StorageChange, StorageEntry, StorageKey, StorageStats, StorageValue, }; +use bincode::Options; use parking_lot::RwLock; use platform_core::{ChallengeId, Hotkey, MiniChainError, Result}; use sled::Tree; @@ -22,6 +23,15 @@ use std::sync::Arc; use std::time::{Duration, SystemTime}; use tracing::{info, trace}; +const MAX_STORAGE_ENTRY_SIZE: u64 = 64 * 1024 * 1024; + +fn bincode_options_storage() -> impl Options { + bincode::DefaultOptions::new() + .with_limit(MAX_STORAGE_ENTRY_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() +} + /// Dynamic storage manager #[allow(clippy::type_complexity)] pub struct DynamicStorage { @@ -116,7 +126,8 @@ impl DynamicStorage { .map_err(|e| MiniChainError::Storage(e.to_string()))? { Some(data) => { - let entry: StorageEntry = bincode::deserialize(&data) + let entry: StorageEntry = bincode_options_storage() + .deserialize(&data) .map_err(|e| MiniChainError::Serialization(e.to_string()))?; // Check expiry @@ -355,7 +366,8 @@ impl DynamicStorage { for item in self.tree.scan_prefix(&prefix) { let (key_bytes, data) = item.map_err(|e| MiniChainError::Storage(e.to_string()))?; - let entry: StorageEntry = bincode::deserialize(&data) + let entry: StorageEntry = bincode_options_storage() + .deserialize(&data) .map_err(|e| MiniChainError::Serialization(e.to_string()))?; if entry.is_expired() { @@ -411,7 +423,7 @@ impl DynamicStorage { for item in self.tree.iter() { let (key, data) = item.map_err(|e| MiniChainError::Storage(e.to_string()))?; - if let Ok(entry) = bincode::deserialize::(&data) { + if let Ok(entry) = bincode_options_storage().deserialize::(&data) { if entry.is_expired() { to_remove.push(key.to_vec()); } @@ -475,6 +487,67 @@ impl DynamicStorage { .map_err(|e| MiniChainError::Storage(e.to_string()))?; Ok(()) } + + /// Query entries by prefix within a challenge namespace + pub fn query_by_prefix( + &self, + challenge_id: &ChallengeId, + prefix: &str, + ) -> Result)>> { + let namespace = challenge_id.0.to_string(); + let entries = self.scan_namespace(&namespace)?; + + entries + .into_iter() + .filter(|(k, _)| k.validator.is_none() && k.key.starts_with(prefix)) + .map(|(k, entry)| { + let value_bytes = bincode::serialize(&entry.value) + .map_err(|e| MiniChainError::Serialization(e.to_string()))?; + Ok((k.key, value_bytes)) + }) + .collect() + } + + /// Get a value as it existed at a specific block height + /// + /// Note: This is a best-effort operation. The current implementation + /// returns the current value if it was last modified at or before the + /// specified block height. Full block-level history requires a separate + /// versioned storage layer. + pub fn get_at_block( + &self, + challenge_id: &ChallengeId, + key: &str, + block: u64, + ) -> Result>> { + let storage_key = StorageKey::challenge(challenge_id, key); + let entry = self.get(&storage_key)?; + + match entry { + Some(e) => { + if e.version <= block { + let value_bytes = bincode::serialize(&e.value) + .map_err(|err| MiniChainError::Serialization(err.to_string()))?; + Ok(Some(value_bytes)) + } else { + Ok(None) + } + } + None => Ok(None), + } + } + + /// List all keys within a challenge namespace + pub fn list_keys(&self, challenge_id: &ChallengeId) -> Result> { + let namespace = challenge_id.0.to_string(); + let entries = self.scan_namespace(&namespace)?; + + Ok(entries + .into_iter() + .filter(|(k, _)| k.validator.is_none()) + .map(|(k, _)| k.key) + .collect()) + } } /// Scoped storage for a specific challenge @@ -552,6 +625,16 @@ impl<'a> ChallengeStorage<'a> { let storage_key = StorageKey::challenge(&self.challenge_id, key); self.storage.map_get(&storage_key, field) } + + /// Query entries by key prefix + pub fn query_by_prefix(&self, prefix: &str) -> Result)>> { + self.storage.query_by_prefix(&self.challenge_id, prefix) + } + + /// List all keys in this challenge + pub fn list_keys(&self) -> Result> { + self.storage.list_keys(&self.challenge_id) + } } /// Scoped storage for a specific validator diff --git a/crates/wasm-runtime-interface/src/lib.rs b/crates/wasm-runtime-interface/src/lib.rs index 7b57a4db..1f8f9e41 100644 --- a/crates/wasm-runtime-interface/src/lib.rs +++ b/crates/wasm-runtime-interface/src/lib.rs @@ -14,6 +14,7 @@ pub mod consensus; pub mod container; pub mod data; pub mod exec; +pub mod llm; pub mod network; pub mod runtime; pub mod sandbox; @@ -62,6 +63,7 @@ pub use data::{ DataBackend, DataError, DataHostFunctions, DataHostStatus, DataPolicy, DataState, FilesystemDataBackend, NoopDataBackend, HOST_DATA_GET, HOST_DATA_LIST, HOST_DATA_NAMESPACE, }; +pub use llm::{LlmHostFunctions, LlmHostStatus, LlmPolicy, LlmState, HOST_LLM_NAMESPACE}; pub use runtime::{ ChallengeInstance, HostFunctionRegistrar, InstanceConfig, RuntimeConfig, RuntimeState, WasmModule, WasmRuntime, WasmRuntimeError, diff --git a/crates/wasm-runtime-interface/src/llm.rs b/crates/wasm-runtime-interface/src/llm.rs new file mode 100644 index 00000000..074bacda --- /dev/null +++ b/crates/wasm-runtime-interface/src/llm.rs @@ -0,0 +1,480 @@ +//! LLM Host Functions for WASM Challenges +//! +//! Provides host functions that allow WASM code to perform LLM inference +//! via the Chutes API (llm.chutes.ai). Gated by `LlmPolicy`. +//! +//! # Host Functions +//! +//! - `llm_chat_completion(req_ptr, req_len, resp_ptr, resp_len) -> i32` — Send chat completion request +//! - `llm_is_available() -> i32` — Check if LLM inference is available (has API key) + +use crate::runtime::{HostFunctionRegistrar, RuntimeState, WasmRuntimeError}; +use bincode::Options; +use serde::{Deserialize, Serialize}; +use std::fmt; +use tracing::warn; +use wasmtime::{Caller, Linker, Memory}; + +const MAX_CHAT_REQUEST_SIZE: u64 = 4 * 1024 * 1024; +const LLM_REQUEST_TIMEOUT_SECS: u64 = 60; + +pub const HOST_LLM_NAMESPACE: &str = "platform_llm"; +pub const HOST_LLM_CHAT_COMPLETION: &str = "llm_chat_completion"; +pub const HOST_LLM_IS_AVAILABLE: &str = "llm_is_available"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i32)] +pub enum LlmHostStatus { + Success = 0, + Disabled = -1, + InvalidRequest = -2, + ApiError = -3, + BufferTooSmall = -4, + RateLimited = -5, + InternalError = -100, +} + +impl LlmHostStatus { + pub fn to_i32(self) -> i32 { + self as i32 + } +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct LlmPolicy { + pub enabled: bool, + #[serde(skip)] + pub api_key: Option, + pub endpoint: String, + pub max_requests: u32, + pub allowed_models: Vec, +} + +impl fmt::Debug for LlmPolicy { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("LlmPolicy") + .field("enabled", &self.enabled) + .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]")) + .field("endpoint", &self.endpoint) + .field("max_requests", &self.max_requests) + .field("allowed_models", &self.allowed_models) + .finish() + } +} + +impl Default for LlmPolicy { + fn default() -> Self { + Self { + enabled: false, + api_key: None, + endpoint: "https://llm.chutes.ai/v1/chat/completions".to_string(), + max_requests: 10, + allowed_models: Vec::new(), + } + } +} + +impl LlmPolicy { + pub fn with_api_key(api_key: String) -> Self { + Self { + enabled: true, + api_key: Some(api_key), + ..Default::default() + } + } + + pub fn is_available(&self) -> bool { + self.enabled && self.api_key.is_some() + } +} + +pub struct LlmState { + pub policy: LlmPolicy, + pub requests_made: u32, +} + +impl LlmState { + pub fn new(policy: LlmPolicy) -> Self { + Self { + policy, + requests_made: 0, + } + } +} + +#[derive(Clone, Debug)] +pub struct LlmHostFunctions; + +impl LlmHostFunctions { + pub fn new() -> Self { + Self + } +} + +impl Default for LlmHostFunctions { + fn default() -> Self { + Self::new() + } +} + +impl HostFunctionRegistrar for LlmHostFunctions { + fn register(&self, linker: &mut Linker) -> Result<(), WasmRuntimeError> { + linker + .func_wrap( + HOST_LLM_NAMESPACE, + HOST_LLM_CHAT_COMPLETION, + |mut caller: Caller, + req_ptr: i32, + req_len: i32, + resp_ptr: i32, + resp_len: i32| + -> i32 { + handle_chat_completion(&mut caller, req_ptr, req_len, resp_ptr, resp_len) + }, + ) + .map_err(|err| WasmRuntimeError::HostFunction(err.to_string()))?; + + linker + .func_wrap( + HOST_LLM_NAMESPACE, + HOST_LLM_IS_AVAILABLE, + |caller: Caller| -> i32 { handle_is_available(&caller) }, + ) + .map_err(|err| WasmRuntimeError::HostFunction(err.to_string()))?; + + Ok(()) + } +} + +fn handle_is_available(caller: &Caller) -> i32 { + let state = &caller.data().llm_state; + if state.policy.is_available() { + 1 + } else { + 0 + } +} + +fn handle_chat_completion( + caller: &mut Caller, + req_ptr: i32, + req_len: i32, + resp_ptr: i32, + resp_len: i32, +) -> i32 { + let policy_available; + let requests_made; + let max_requests; + { + let state = &caller.data().llm_state; + policy_available = state.policy.is_available(); + requests_made = state.requests_made; + max_requests = state.policy.max_requests; + } + + if !policy_available { + return LlmHostStatus::Disabled.to_i32(); + } + + if requests_made >= max_requests { + return LlmHostStatus::RateLimited.to_i32(); + } + + if req_ptr < 0 || req_len < 0 || resp_ptr < 0 || resp_len < 0 { + return LlmHostStatus::InvalidRequest.to_i32(); + } + + let request_bytes = match read_wasm_memory(caller, req_ptr, req_len as usize) { + Ok(b) => b, + Err(err) => { + warn!(error = %err, "llm_chat_completion: failed to read request from wasm memory"); + return LlmHostStatus::InternalError.to_i32(); + } + }; + + let api_key; + let endpoint; + { + let state = &caller.data().llm_state; + api_key = match &state.policy.api_key { + Some(k) => k.clone(), + None => return LlmHostStatus::Disabled.to_i32(), + }; + endpoint = state.policy.endpoint.clone(); + } + + #[derive(Deserialize)] + struct ChatRequest { + model: String, + messages: Vec, + max_tokens: u32, + temperature: f32, + } + + #[derive(Deserialize)] + struct ChatMessage { + role: String, + content: String, + } + + let chat_req: ChatRequest = match bincode::DefaultOptions::new() + .with_limit(MAX_CHAT_REQUEST_SIZE) + .with_fixint_encoding() + .allow_trailing_bytes() + .deserialize(&request_bytes) + { + Ok(r) => r, + Err(_) => return LlmHostStatus::InvalidRequest.to_i32(), + }; + + { + let state = &caller.data().llm_state; + let allowed = &state.policy.allowed_models; + if !allowed.is_empty() && !allowed.contains(&chat_req.model) { + warn!( + model = %chat_req.model, + "llm_chat_completion: model not in allowed list" + ); + return LlmHostStatus::InvalidRequest.to_i32(); + } + } + + #[derive(Serialize)] + struct OpenAiRequest { + model: String, + messages: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + max_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + temperature: Option, + } + + #[derive(Serialize)] + struct OpenAiMessage { + role: String, + content: String, + } + + let openai_req = OpenAiRequest { + model: chat_req.model, + messages: chat_req + .messages + .into_iter() + .map(|m| OpenAiMessage { + role: m.role, + content: m.content, + }) + .collect(), + max_tokens: Some(chat_req.max_tokens), + temperature: Some(chat_req.temperature), + }; + + let json_body = match serde_json::to_vec(&openai_req) { + Ok(b) => b, + Err(_) => return LlmHostStatus::InvalidRequest.to_i32(), + }; + + let client = reqwest::blocking::Client::new(); + let http_response = match client + .post(&endpoint) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {}", api_key)) + .body(json_body) + .timeout(std::time::Duration::from_secs(LLM_REQUEST_TIMEOUT_SECS)) + .send() + { + Ok(r) => r, + Err(err) => { + warn!(error = %err, "llm_chat_completion: HTTP request failed"); + return LlmHostStatus::ApiError.to_i32(); + } + }; + + let response_body = match http_response.bytes() { + Ok(b) => b.to_vec(), + Err(err) => { + warn!(error = %err, "llm_chat_completion: failed to read response body"); + return LlmHostStatus::ApiError.to_i32(); + } + }; + + #[derive(Deserialize)] + struct OpenAiResponse { + choices: Option>, + usage: Option, + } + + #[derive(Deserialize)] + struct OpenAiChoice { + message: Option, + } + + #[derive(Deserialize)] + struct OpenAiRespMessage { + content: Option, + } + + #[derive(Deserialize)] + struct OpenAiUsage { + prompt_tokens: Option, + completion_tokens: Option, + total_tokens: Option, + } + + let openai_resp: OpenAiResponse = match serde_json::from_slice(&response_body) { + Ok(r) => r, + Err(err) => { + warn!(error = %err, "llm_chat_completion: failed to parse OpenAI response"); + return LlmHostStatus::ApiError.to_i32(); + } + }; + + let content = openai_resp + .choices + .and_then(|mut c| c.pop()) + .and_then(|c| c.message) + .and_then(|m| m.content) + .unwrap_or_default(); + + #[derive(Serialize)] + struct LlmResponsePayload { + content: String, + usage: Option, + } + + #[derive(Serialize)] + struct LlmUsagePayload { + prompt_tokens: u32, + completion_tokens: u32, + total_tokens: u32, + } + + let usage = openai_resp.usage.map(|u| LlmUsagePayload { + prompt_tokens: u.prompt_tokens.unwrap_or(0), + completion_tokens: u.completion_tokens.unwrap_or(0), + total_tokens: u.total_tokens.unwrap_or(0), + }); + + let response_payload = LlmResponsePayload { content, usage }; + + let response_bytes = match bincode::serialize(&response_payload) { + Ok(b) => b, + Err(_) => return LlmHostStatus::InternalError.to_i32(), + }; + + if response_bytes.len() > resp_len as usize { + return LlmHostStatus::BufferTooSmall.to_i32(); + } + + if let Err(err) = write_wasm_memory(caller, resp_ptr, &response_bytes) { + warn!(error = %err, "llm_chat_completion: failed to write response to wasm memory"); + return LlmHostStatus::InternalError.to_i32(); + } + + caller.data_mut().llm_state.requests_made += 1; + + response_bytes.len() as i32 +} + +fn read_wasm_memory( + caller: &mut Caller, + ptr: i32, + len: usize, +) -> Result, String> { + if ptr < 0 { + return Err("negative pointer".to_string()); + } + let ptr = ptr as usize; + let memory = get_memory(caller).ok_or_else(|| "memory export not found".to_string())?; + let end = ptr + .checked_add(len) + .ok_or_else(|| "pointer overflow".to_string())?; + let data = memory.data(caller); + if end > data.len() { + return Err("memory read out of bounds".to_string()); + } + Ok(data[ptr..end].to_vec()) +} + +fn write_wasm_memory( + caller: &mut Caller, + ptr: i32, + bytes: &[u8], +) -> Result<(), String> { + if ptr < 0 { + return Err("negative pointer".to_string()); + } + let ptr = ptr as usize; + let memory = get_memory(caller).ok_or_else(|| "memory export not found".to_string())?; + let end = ptr + .checked_add(bytes.len()) + .ok_or_else(|| "pointer overflow".to_string())?; + let data = memory.data_mut(caller); + if end > data.len() { + return Err("memory write out of bounds".to_string()); + } + data[ptr..end].copy_from_slice(bytes); + Ok(()) +} + +fn get_memory(caller: &mut Caller) -> Option { + let memory_export = caller.data().memory_export.clone(); + caller + .get_export(&memory_export) + .and_then(|export| export.into_memory()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_llm_host_status_values() { + assert_eq!(LlmHostStatus::Success.to_i32(), 0); + assert_eq!(LlmHostStatus::Disabled.to_i32(), -1); + assert_eq!(LlmHostStatus::InvalidRequest.to_i32(), -2); + assert_eq!(LlmHostStatus::ApiError.to_i32(), -3); + assert_eq!(LlmHostStatus::BufferTooSmall.to_i32(), -4); + assert_eq!(LlmHostStatus::RateLimited.to_i32(), -5); + assert_eq!(LlmHostStatus::InternalError.to_i32(), -100); + } + + #[test] + fn test_llm_policy_default() { + let policy = LlmPolicy::default(); + assert!(!policy.enabled); + assert!(policy.api_key.is_none()); + assert!(!policy.is_available()); + } + + #[test] + fn test_llm_policy_with_api_key() { + let policy = LlmPolicy::with_api_key("test-key".to_string()); + assert!(policy.enabled); + assert!(policy.is_available()); + assert_eq!(policy.api_key, Some("test-key".to_string())); + } + + #[test] + fn test_llm_state_creation() { + let state = LlmState::new(LlmPolicy::default()); + assert_eq!(state.requests_made, 0); + assert!(!state.policy.is_available()); + } + + #[test] + fn test_llm_policy_debug_redacts_api_key() { + let policy = LlmPolicy::with_api_key("super-secret-key-12345".to_string()); + let debug_output = format!("{:?}", policy); + assert!(!debug_output.contains("super-secret-key-12345")); + assert!(debug_output.contains("[REDACTED]")); + } + + #[test] + fn test_llm_policy_serialize_skips_api_key() { + let policy = LlmPolicy::with_api_key("secret-key".to_string()); + let serialized = bincode::serialize(&policy).unwrap(); + let deserialized: LlmPolicy = bincode::deserialize(&serialized).unwrap(); + assert!(deserialized.api_key.is_none()); + } +} diff --git a/crates/wasm-runtime-interface/src/runtime.rs b/crates/wasm-runtime-interface/src/runtime.rs index 5dbc408a..38c5939b 100644 --- a/crates/wasm-runtime-interface/src/runtime.rs +++ b/crates/wasm-runtime-interface/src/runtime.rs @@ -3,6 +3,7 @@ use crate::consensus::{ConsensusHostFunctions, ConsensusPolicy, ConsensusState}; use crate::container::{ContainerHostFunctions, ContainerPolicy, ContainerState}; use crate::data::{DataBackend, DataHostFunctions, DataPolicy, DataState, NoopDataBackend}; use crate::exec::{ExecHostFunctions, ExecPolicy, ExecState}; +use crate::llm::{LlmHostFunctions, LlmPolicy, LlmState}; use crate::sandbox::SandboxHostFunctions; use crate::storage::{ InMemoryStorageBackend, StorageBackend, StorageHostConfig, StorageHostFunctions, @@ -130,6 +131,8 @@ pub struct InstanceConfig { pub data_backend: Arc, /// Container policy for WASM access to container execution. pub container_policy: ContainerPolicy, + /// LLM policy for WASM access to LLM inference. + pub llm_policy: LlmPolicy, } impl Default for InstanceConfig { @@ -153,6 +156,7 @@ impl Default for InstanceConfig { data_policy: DataPolicy::default(), data_backend: Arc::new(NoopDataBackend), container_policy: ContainerPolicy::default(), + llm_policy: LlmPolicy::default(), } } } @@ -190,6 +194,8 @@ pub struct RuntimeState { pub data_state: DataState, /// Container state for container execution host operations. pub container_state: ContainerState, + /// LLM state for LLM inference host operations. + pub llm_state: LlmState, limits: StoreLimits, } @@ -205,6 +211,7 @@ impl RuntimeState { terminal_state: TerminalState, data_state: DataState, container_state: ContainerState, + llm_state: LlmState, memory_export: String, challenge_id: String, validator_id: String, @@ -224,6 +231,7 @@ impl RuntimeState { terminal_state, data_state, container_state, + llm_state, memory_export, challenge_id, validator_id, @@ -353,6 +361,7 @@ impl WasmRuntime { instance_config.challenge_id.clone(), instance_config.validator_id.clone(), ); + let llm_state = LlmState::new(instance_config.llm_policy.clone()); let runtime_state = RuntimeState::new( instance_config.network_policy.clone(), instance_config.sandbox_policy.clone(), @@ -363,6 +372,7 @@ impl WasmRuntime { terminal_state, data_state, container_state, + llm_state, instance_config.memory_export.clone(), instance_config.challenge_id.clone(), instance_config.validator_id.clone(), @@ -410,6 +420,9 @@ impl WasmRuntime { let container_host_fns = ContainerHostFunctions::new(); container_host_fns.register(&mut linker)?; + let llm_host_fns = LlmHostFunctions::new(); + llm_host_fns.register(&mut linker)?; + let sandbox_host_fns = SandboxHostFunctions::all(); sandbox_host_fns.register(&mut linker)?;