diff --git a/README.md b/README.md index 7bc5cd8..8056862 100644 --- a/README.md +++ b/README.md @@ -20,25 +20,51 @@ A test implementation consists of three main components: ### 1. Configuration +#### Directory Structure + +Below is the recommended file structure for creating your test. See the `example` folder for sample file contents. + +``` +orca-container + ├── .env + ├──src/ + ├──tests/ + ├── .env + ├── data/ + │ ├── collection1.json + │ └── collection2.json + ├── config.yaml + ├── workers.json + ├── e2e.py + ├── steps.py + └── stages/ + ├── task.py + ├── submission.py + └── audit.py +``` + #### Test Configuration (config.yaml) ```yaml # Test Configuration -task_id: "your_task_id" # Task identifier +task_id: "your_task_id" # Task identifier, should match the middle server base_port: 5000 # Base port for worker servers, optional -max_rounds: 3 # Maximum test rounds, optional +max_rounds: 3 # Maximum test rounds, optional. +rounds_collection: "documentations" # By default number of rounds the task will run for equals the number of documents in this collection # Paths data_dir: data # Test data directory, optional. defaults to the /data dir within your tests folder -workers_config: workers.json # Worker configuration, relative to tests directory +workers_config: workers.json # Worker configuration, relative to tests directory, optional. defaults to workers.json in your tests folder # MongoDB Configuration (if needed) mongodb: database: your_database_name collections: - collection_name: - data_file: data.json # Relative to data_dir - required_count: 1 # Minimum required documents + tasks: # collection name + data_file: tasks.json # file containing data for this collection, relative to the data_dir you specified + required_count: 1 # minimum number of documents the collection must have + audits: + required_count: 0 # No data file, just needs to exist ``` #### Worker Configuration (workers.json) @@ -46,10 +72,23 @@ mongodb: ```json { "worker1": { - "port": 5001, - "env": { - "WORKER_ID": "worker1", - "OTHER_ENV": "value" + "port": 5001, // optional, will be automatically determined if not specified + + // this maps the env variable used by the server to the actual env variable defined in your .env file + // for example, if every worker needs its own github token, the server variable will be just `GITHUB_TOKEN` + // but we need to differentiate which token belongs to which worker, so we map the server variable to the specific worker variable + "env_vars": { + "GITHUB_TOKEN": "WORKER1_GITHUB_TOKEN", + "GITHUB_USERNAME": "WORKER1_GITHUB_USERNAME" + }, + + // Workers need keypairs to simulate the signatures generated in the node + // Depending on your task, you may need only one of these two. By default, namespaceWrapper.payloadSigning uses the public key. + // These do not need to be real staking and public keypairs from the node as they're only used for signing; any valid wallets will do + // Specify the keypair paths in your .env file using the variable names you specify here. + "keypairs": { + "staking": "WORKER1_STAKING_KEYPAIR", + "public": "WORKER1_PUBLIC_KEYPAIR" } }, "worker2": { @@ -58,6 +97,10 @@ mongodb: "WORKER_ID": "worker2" } } + "keypairs": { + "staking": "WORKER2_STAKING_KEYPAIR", + "public": "WORKER2_PUBLIC_KEYPAIR" + } } ``` @@ -67,6 +110,7 @@ Create a `steps.py` file to define your test sequence: ```python from prometheus_test import TestStep +from stages.step_name import your_prepare_function, your_execute_function steps = [ TestStep( @@ -74,7 +118,7 @@ steps = [ description="Step description", # Human-readable description prepare=your_prepare_function, # Setup function execute=your_execute_function, # Main execution function - worker="worker_name", # Worker that executes this step + worker="worker_name", # Worker that executes this step. Matches the worker names defined in workers.json ), # Add more steps... ] @@ -148,23 +192,6 @@ If you have an .env file in your agent's top level folder (for API keys, etc), t ## Test Data Management -### Directory Structure - -``` -orca-container - ├── .env - ├──src/ - ├──tests/ - ├── .env - ├── data/ - │ ├── collection1.json - │ └── collection2.json - ├── config.yaml - ├── workers.json - ├── e2e.py - └── steps.py -``` - ### Data Files Test data should be organized in JSON files within your data directory. Each file represents a collection's initial state. These files are then specified in your config.yaml (see above). diff --git a/example/tests/.env.example b/example/tests/.env.example new file mode 100644 index 0000000..bf12016 --- /dev/null +++ b/example/tests/.env.example @@ -0,0 +1,21 @@ +ANTHROPIC_API_KEY=your_anthropic_api_key + +MONGO_URI=mongodb://localhost:27017 + +# These should match the env variables you defined in your workers.json file +WORKER1_GITHUB_TOKEN="" +WORKER1_GITHUB_USERNAME="" +WORKER2_GITHUB_TOKEN="" +WORKER2_GITHUB_USERNAME="" + +# Depending on your task, you may need only one of these two. By default, namespaceWrapper.payloadSigning uses the public key. +# These do not need to be real staking and public keypairs from the node as they're only used for signing; any valid wallets will do +# These should match the variable names specified in your workers.json file +WORKER1_STAKING_KEYPAIR="/path/to/wallet.json" +WORKER1_PUBLIC_KEYPAIR="" +WORKER2_STAKING_KEYPAIR="" +WORKER2_PUBLIC_KEYPAIR="" + + +# Generally this should not be changed +TEST_MODE=true diff --git a/example/tests/config.yaml b/example/tests/config.yaml new file mode 100644 index 0000000..bb83b38 --- /dev/null +++ b/example/tests/config.yaml @@ -0,0 +1,22 @@ +# Test Configuration +task_id: "1111" # Task ID from config-task.yml +middle_server_url: "http://localhost:3000" +# collection used to determine the max_rounds, if the value is not directly set +rounds_collection: "documentations" +# base_port: 5000 # Base port for worker servers +# max_rounds: 1 # Maximum number of test rounds + +# Paths +# relative to the test directory +data_dir: data/minimal # Directory containing test data +# workers_config: workers.json # Worker configuration file + +# MongoDB Configuration +mongodb: + database: builder247 + collections: + documentations: + data_file: documentations.json + required_count: 1 + audits: + required_count: 0 # No data file, just needs to exist diff --git a/example/tests/data/tasks.json b/example/tests/data/tasks.json new file mode 100644 index 0000000..e0e2011 --- /dev/null +++ b/example/tests/data/tasks.json @@ -0,0 +1,6 @@ +[ + { + "description": "Do a task", + "status": "initialized" + } +] diff --git a/example/tests/e2e.py b/example/tests/e2e.py new file mode 100644 index 0000000..d16cb49 --- /dev/null +++ b/example/tests/e2e.py @@ -0,0 +1,45 @@ +"""End-to-end test for the summarizer task.""" + +from pathlib import Path +from prometheus_test import TestRunner +import dotenv +import argparse + + +dotenv.load_dotenv() + + +def parse_args(): + parser = argparse.ArgumentParser(description="Run summarizer test sequence") + parser.add_argument( + "--reset", + action="store_true", + help="Force reset of all databases before running tests", + ) + return parser.parse_args() + + +# Global reference to the test runner +runner = None + + +def main(): + global runner + args = parse_args() + + # Import steps here to avoid circular imports + from .steps import steps + + # Create test runner with config from YAML + base_dir = Path(__file__).parent + runner = TestRunner( + steps=steps, + config_file=base_dir / "config.yaml", + ) + + # Run test sequence + runner.run(force_reset=args.reset) + + +if __name__ == "__main__": + main() diff --git a/example/tests/stages/update_audit.py b/example/tests/stages/update_audit.py new file mode 100644 index 0000000..c9fe3bd --- /dev/null +++ b/example/tests/stages/update_audit.py @@ -0,0 +1,25 @@ +"""Stage for executing worker tasks.""" + +import requests + + +def prepare(runner, worker, role: str): + """Prepare data for worker task""" + + return { + "taskId": runner.config.task_id, + "round": runner.current_round, + } + + +def execute(runner, worker, data): + """Execute worker task step""" + url = f"{runner.config.middle_server_url}/summarizer/worker/update-audit-result" + response = requests.post( + url, + json=data, + ) + response.raise_for_status() + + # Return a formatted response regardless of type + return {"success": True, "message": response.text} diff --git a/example/tests/stages/worker_audit.py b/example/tests/stages/worker_audit.py new file mode 100644 index 0000000..be56f09 --- /dev/null +++ b/example/tests/stages/worker_audit.py @@ -0,0 +1,82 @@ +"""Stage for worker audits.""" + +import requests + +# from prometheus_test.utils import create_signature + + +def prepare(runner, worker, target_name): + """Prepare data for worker audit""" + round_state = runner.state["rounds"].get(str(runner.current_round), {}) + pr_urls = round_state.get("pr_urls", {}) + + if target_name not in pr_urls: + # Return None to indicate this step should be skipped + print( + f"✓ No PR URL found for {target_name}, skipping {worker.name} audit - continuing" + ) + return None + + # Get submission data from state + submission_data = round_state.get("submission_data", {}).get(target_name) + if not submission_data: + # Return None to indicate this step should be skipped + print( + f"✓ No submission data found for {target_name}, skipping {worker.name} audit - continuing" + ) + return None + + # Create auditor payload which is used to generate the signature + # auditor_payload = { + # "taskId": runner.config.task_id, + # "roundNumber": runner.current_round, + # "prUrl": pr_urls[target_name], + # "stakingKey": worker.staking_public_key, + # "pubKey": worker.public_key, + # } + + # Structure the payload according to what the server expects + # return { + # "submission": { + # "taskId": runner.config.task_id, + # "roundNumber": runner.current_round, + # "prUrl": pr_urls[target_name], + # "githubUsername": submission_data.get("githubUsername"), + # "repoOwner": submission_data.get("repoOwner"), + # "repoName": submission_data.get("repoName"), + # "stakingKey": submission_data.get("stakingKey"), + # "pubKey": submission_data.get("pubKey"), + # "uuid": submission_data.get("uuid"), + # "nodeType": submission_data.get("nodeType"), + # }, + # "submitterSignature": submission_data.get("signature"), + # "submitterStakingKey": submission_data.get("stakingKey"), + # "submitterPubKey": submission_data.get("pubKey"), + # "prUrl": pr_urls[target_name], + # "repoOwner": submission_data.get("repoOwner"), + # "repoName": submission_data.get("repoName"), + # "githubUsername": worker.env.get("GITHUB_USERNAME"), + # "stakingKey": worker.staking_public_key, + # "pubKey": worker.public_key, + # "stakingSignature": create_signature( + # worker.staking_signing_key, auditor_payload + # ), + # "publicSignature": create_signature(worker.public_signing_key, auditor_payload), + # } + return {"submission": submission_data} + + +def execute(runner, worker, data): + """Execute worker audit step""" + # If prepare returned None, skip this step + if data is None: + return { + "success": True, + "message": "Skipped due to missing PR URL or submission data", + } + + url = f"{worker.url}/worker-audit/{runner.current_round}" + response = requests.post(url, json=data) + result = response.json() + + return result diff --git a/example/tests/stages/worker_check.py b/example/tests/stages/worker_check.py new file mode 100644 index 0000000..86fab89 --- /dev/null +++ b/example/tests/stages/worker_check.py @@ -0,0 +1,41 @@ +"""Stage for executing worker tasks.""" + +import requests + + +def prepare(runner, worker): + """Prepare data for worker task""" + # Create fetch-todo payload for stakingSignature and publicSignature + round_state = runner.state["rounds"].get(str(runner.current_round), {}) + if not round_state.get("pr_urls"): + print(f"✓ No PR URLs found for {worker.name} - continuing") + return + return { + "stakingKey": worker.staking_public_key, + "roundNumber": runner.current_round, + "githubUsername": worker.env.get("GITHUB_USERNAME"), + "prUrl": round_state.get("pr_urls", {}).get(worker.name), + } + + +def execute(runner, worker, data): + """Execute worker task step""" + if not data: + return {"success": True, "message": "No PR URL found"} + url = f"{runner.config.middle_server_url}/summarizer/worker/check-todo" + response = requests.post( + url, + json=data, + ) + result = response.json() + + # Handle 409 gracefully - no eligible todos is an expected case + if response.status_code == 409: + print( + f"✓ {result.get('message', 'No eligible todos')} for {worker.name} - continuing" + ) + return {"success": True, "message": result.get("message")} + else: + response.raise_for_status() + + return result diff --git a/example/tests/stages/worker_fetch.py b/example/tests/stages/worker_fetch.py new file mode 100644 index 0000000..64b4530 --- /dev/null +++ b/example/tests/stages/worker_fetch.py @@ -0,0 +1,54 @@ +"""Stage for executing worker tasks.""" + +import requests +from prometheus_test.utils import create_signature + + +def prepare(runner, worker): + """Prepare data for worker task""" + # Create fetch-todo payload for stakingSignature and publicSignature + payload = { + "taskId": runner.config.task_id, + "roundNumber": runner.current_round, + "action": "fetch-todo", + "githubUsername": worker.env.get("GITHUB_USERNAME"), + "stakingKey": worker.staking_public_key, + "pubKey": worker.public_key, + } + + return { + "taskId": runner.config.task_id, + "roundNumber": runner.current_round, + "stakingKey": worker.staking_public_key, + "pubKey": worker.public_key, + "stakingSignature": create_signature(worker.staking_signing_key, payload), + "publicSignature": create_signature(worker.public_signing_key, payload), + } + + +def execute(runner, worker, data): + """Execute worker task step""" + url = f"{runner.config.middle_server_url}/summarizer/worker/fetch-todo" + response = requests.post( + url, + json={"signature": data["stakingSignature"], "stakingKey": data["stakingKey"]}, + ) + result = response.json() + + # Handle 409 gracefully - no eligible todos is an expected case + if response.status_code == 409: + print( + f"✓ {result.get('message', 'No eligible todos')} for {worker.name} - continuing" + ) + return {"success": True, "message": result.get("message")} + else: + response.raise_for_status() + + if result.get("success"): + round_key = str(runner.current_round) + round_state = runner.state["rounds"].setdefault(round_key, {}) + round_state["repo_url"] = ( + f"https://github.com/{result['data']['repo_owner']}/{result['data']['repo_name']}" + ) + + return result diff --git a/example/tests/stages/worker_pr.py b/example/tests/stages/worker_pr.py new file mode 100644 index 0000000..50f9ca9 --- /dev/null +++ b/example/tests/stages/worker_pr.py @@ -0,0 +1,48 @@ +import requests +from prometheus_test.utils import create_signature + + +def prepare(runner, worker): + round_state = runner.state["rounds"].get(str(runner.current_round), {}) + + if worker.name not in round_state.get("pr_urls", {}): + print(f"✓ No PR URL found for {worker.name} - continuing") + return None + + payload = { + "taskId": runner.config.task_id, + "action": "add-todo-pr", + "roundNumber": runner.current_round, + "prUrl": round_state["pr_urls"][worker.name], + "stakingKey": worker.staking_public_key, + "pubKey": worker.public_key, + } + return { + "signature": create_signature(worker.staking_signing_key, payload), + "stakingKey": worker.staking_public_key, + } + + +def execute(runner, worker, data): + """Add worker PR URL to middle server""" + + if data is None: + return {"success": True, "message": "Skipped due to missing PR URL"} + + url = f"{runner.config.middle_server_url}/summarizer/worker/add-todo-pr" + response = requests.post( + url, + json={"signature": data["signature"], "stakingKey": data["stakingKey"]}, + ) + result = response.json() + + # Handle 409 gracefully - no eligible todos is an expected case + if response.status_code == 409: + print( + f"✓ {result.get('message', 'No eligible todos')} for {worker.name} - continuing" + ) + return {"success": True, "message": result.get("message")} + else: + response.raise_for_status() + + return result diff --git a/example/tests/stages/worker_submission.py b/example/tests/stages/worker_submission.py new file mode 100644 index 0000000..39b0930 --- /dev/null +++ b/example/tests/stages/worker_submission.py @@ -0,0 +1,60 @@ +"""Stage for handling worker submissions.""" + +import requests +from prometheus_test.utils import create_signature + + +def prepare(runner, worker): + """Prepare data for worker submission""" + # Get the current round's state + round_state = runner.state.get("rounds", {}).get(str(runner.current_round), {}) + pr_urls = round_state.get("pr_urls", {}) + + if worker.name not in pr_urls: + # Return None to indicate this step should be skipped + print(f"✓ No PR URL found for {worker.name} - continuing") + return None + + # Get submission data from worker + url = f"{worker.url}/submission/{runner.current_round}" + response = requests.get(url) + response.raise_for_status() + submission_data = response.json() + + # Create signature for the submission + submitter_payload = { + "taskId": runner.config.task_id, + "roundNumber": runner.current_round, + "stakingKey": worker.staking_public_key, + "pubKey": worker.public_key, + "action": "audit", + **submission_data, + } + + return { + **submission_data, + "signature": create_signature(worker.staking_signing_key, submitter_payload), + "stakingKey": worker.staking_public_key, + "pubKey": worker.public_key, + } + + +def execute(runner, worker, data): + """Store worker submission data""" + # If prepare returned None, skip this step + if data is None: + return {"success": True, "message": "Skipped due to missing PR URL"} + + # Store submission data in state + round_key = str(runner.current_round) + round_state = runner.state["rounds"].setdefault(round_key, {}) + + # Initialize submission_data if not exists + if "submission_data" not in round_state: + round_state["submission_data"] = {} + + # Store or update submission data + round_state["submission_data"][worker.name] = data + + # Return success result + return {"success": True, "data": data} diff --git a/example/tests/stages/worker_task.py b/example/tests/stages/worker_task.py new file mode 100644 index 0000000..c8c9ca6 --- /dev/null +++ b/example/tests/stages/worker_task.py @@ -0,0 +1,43 @@ +"""Stage for executing worker tasks.""" + +import requests + + +def prepare(runner, worker): + """Prepare data for worker task""" + round_state = runner.state["rounds"].get(str(runner.current_round), {}) + if not round_state.get("repo_url"): + print(f"✓ No repo url found for {worker.name} - continuing") + return + return { + "taskId": runner.config.task_id, + "round_number": str(runner.current_round), + "repo_url": round_state["repo_url"], + } + + +def execute(runner, worker, data): + """Execute worker task step""" + if not data: + return {"success": True, "message": "No repo url found"} + url = f"{worker.url}/worker-task/{runner.current_round}" + response = requests.post(url, json=data) + result = response.json() + + # Handle 409 gracefully - no eligible todos is an expected case + if response.status_code == 409: + print( + f"✓ {result.get('message', 'No eligible todos')} for {worker.name} - continuing" + ) + return {"success": True, "message": result.get("message")} + + if result.get("success") and "pr_url" in result["result"]["data"]: + round_key = str(runner.current_round) + round_state = runner.state["rounds"].setdefault(round_key, {}) + + # Initialize pr_urls if not exists + if "pr_urls" not in round_state: + round_state["pr_urls"] = {} + round_state["pr_urls"][worker.name] = result["result"]["data"]["pr_url"] + + return result diff --git a/example/tests/steps.py b/example/tests/steps.py new file mode 100644 index 0000000..3e68217 --- /dev/null +++ b/example/tests/steps.py @@ -0,0 +1,34 @@ +"""Test step definitions.""" + +from prometheus_test import TestStep +from functools import partial +from .stages import ( + worker_task, + worker_submission, + worker_audit, +) + + +steps = [ + TestStep( + name="worker_task", + description="Execute worker task", + prepare=worker_task.prepare, + execute=worker_task.execute, + worker="worker1", + ), + TestStep( + name="worker_submission", + description="Submit worker task", + prepare=worker_submission.prepare, + execute=worker_submission.execute, + worker="worker1", + ), + TestStep( + name="worker_audit", + description="Worker2 audits Worker1", + prepare=partial(worker_audit.prepare, target_name="worker1"), + execute=worker_audit.execute, + worker="worker2", + ), +] diff --git a/example/tests/workers.json b/example/tests/workers.json new file mode 100644 index 0000000..0abe886 --- /dev/null +++ b/example/tests/workers.json @@ -0,0 +1,22 @@ +{ + "worker1": { + "env_vars": { + "GITHUB_TOKEN": "WORKER1_GITHUB_TOKEN", + "GITHUB_USERNAME": "WORKER1_GITHUB_USERNAME" + }, + "keypairs": { + "staking": "WORKER1_STAKING_KEYPAIR", + "public": "WORKER1_PUBLIC_KEYPAIR" + } + }, + "worker2": { + "env_vars": { + "GITHUB_TOKEN": "WORKER2_GITHUB_TOKEN", + "GITHUB_USERNAME": "WORKER2_GITHUB_USERNAME" + }, + "keypairs": { + "staking": "WORKER2_STAKING_KEYPAIR", + "public": "WORKER2_PUBLIC_KEYPAIR" + } + } +} diff --git a/prometheus_test/data.py b/prometheus_test/data.py index f8df6c4..bbc1819 100644 --- a/prometheus_test/data.py +++ b/prometheus_test/data.py @@ -28,22 +28,6 @@ def __init__(self, task_id=None, round_number=None): self.submission_data = {} self.last_completed_step = None - # Store keypair paths for each role - self.keypairs = { - "leader": { - "staking": os.getenv("LEADER_STAKING_KEYPAIR"), - "public": os.getenv("LEADER_PUBLIC_KEYPAIR"), - }, - "worker1": { - "staking": os.getenv("WORKER1_STAKING_KEYPAIR"), - "public": os.getenv("WORKER1_PUBLIC_KEYPAIR"), - }, - "worker2": { - "staking": os.getenv("WORKER2_STAKING_KEYPAIR"), - "public": os.getenv("WORKER2_PUBLIC_KEYPAIR"), - }, - } - def _parse_repo_info(self): """Parse repository owner and name from fork URL""" if not self.fork_url: diff --git a/prometheus_test/test_framework.py b/prometheus_test/test_framework.py deleted file mode 100644 index e69de29..0000000