WIP: Improve documentation

README.md

@@ -20,36 +20,75 @@ A test implementation consists of three main components:
 
 ### 1. Configuration
 
+#### Directory Structure
+
+Below is the recommended file structure for creating your test. See the `example` folder for sample file contents.
+
+```
+orca-container
+  ├── .env
+  ├──src/
+  ├──tests/
+    ├── .env
+    ├── data/
+    │    ├── collection1.json
+    │    └── collection2.json
+    ├── config.yaml
+    ├── workers.json
+    ├── e2e.py
+    ├── steps.py
+    └── stages/
+      ├── task.py
+      ├── submission.py
+      └── audit.py
+```
+
 #### Test Configuration (config.yaml)
 
 ```yaml
 # Test Configuration
-task_id: "your_task_id" # Task identifier
+task_id: "your_task_id" # Task identifier, should match the middle server
 base_port: 5000 # Base port for worker servers, optional
-max_rounds: 3 # Maximum test rounds, optional
+max_rounds: 3 # Maximum test rounds, optional.
+rounds_collection: "documentations" # By default number of rounds the task will run for equals the number of documents in this collection
 
 # Paths
 data_dir: data # Test data directory, optional. defaults to the /data dir within your tests folder
-workers_config: workers.json # Worker configuration, relative to tests directory
+workers_config: workers.json # Worker configuration, relative to tests directory, optional. defaults to workers.json in your tests folder
 
 # MongoDB Configuration (if needed)
 mongodb:
   database: your_database_name
   collections:
-    collection_name:
-      data_file: data.json # Relative to data_dir
-      required_count: 1 # Minimum required documents
+    tasks: # collection name
+      data_file: tasks.json # file containing data for this collection, relative to the data_dir you specified
+      required_count: 1 # minimum number of documents the collection must have
+    audits:
+      required_count: 0 # No data file, just needs to exist
 ```
 
 #### Worker Configuration (workers.json)
 
 ```json
 {
   "worker1": {
-    "port": 5001,
-    "env": {
-      "WORKER_ID": "worker1",
-      "OTHER_ENV": "value"
+    "port": 5001, // optional, will be automatically determined if not specified
+
+    // this maps the env variable used by the server to the actual env variable defined in your .env file
+    // for example, if every worker needs its own github token, the server variable will be just `GITHUB_TOKEN`
+    // but we need to differentiate which token belongs to which worker, so we map the server variable to the specific worker variable
+    "env_vars": {
+      "GITHUB_TOKEN": "WORKER1_GITHUB_TOKEN",
+      "GITHUB_USERNAME": "WORKER1_GITHUB_USERNAME"
+    },
+
+    // Workers need keypairs to simulate the signatures generated in the node
+    // Depending on your task, you may need only one of these two. By default, namespaceWrapper.payloadSigning uses the public key.
+    // These do not need to be real staking and public keypairs from the node as they're only used for signing; any valid wallets will do
+    // Specify the keypair paths in your .env file using the variable names you specify here.
+    "keypairs": {
+      "staking": "WORKER1_STAKING_KEYPAIR",
+      "public": "WORKER1_PUBLIC_KEYPAIR"
     }
   },
   "worker2": {
@@ -58,6 +97,10 @@ mongodb:
       "WORKER_ID": "worker2"
     }
   }
+      "keypairs": {
+      "staking": "WORKER2_STAKING_KEYPAIR",
+      "public": "WORKER2_PUBLIC_KEYPAIR"
+    }
 }
 ```
 
@@ -67,14 +110,15 @@ Create a `steps.py` file to define your test sequence:
 
 ```python
 from prometheus_test import TestStep
+from stages.step_name import your_prepare_function, your_execute_function
 
 steps = [
     TestStep(
         name="step_name",                    # Unique step identifier
         description="Step description",       # Human-readable description
         prepare=your_prepare_function,        # Setup function
         execute=your_execute_function,        # Main execution function
-        worker="worker_name",                # Worker that executes this step
+        worker="worker_name",                # Worker that executes this step. Matches the worker names defined in workers.json
     ),
     # Add more steps...
 ]
@@ -148,23 +192,6 @@ If you have an .env file in your agent's top level folder (for API keys, etc), t
 
 ## Test Data Management
 
-### Directory Structure
-
-```
-orca-container
-  ├── .env
-  ├──src/
-  ├──tests/
-    ├── .env
-    ├── data/
-    │    ├── collection1.json
-    │    └── collection2.json
-    ├── config.yaml
-    ├── workers.json
-    ├── e2e.py
-    └── steps.py
-```
-
 ### Data Files
 
 Test data should be organized in JSON files within your data directory. Each file represents a collection's initial state. These files are then specified in your config.yaml (see above).

example/tests/.env.example

@@ -0,0 +1,21 @@
+ANTHROPIC_API_KEY=your_anthropic_api_key
+
+MONGO_URI=mongodb://localhost:27017
+
+# These should match the env variables you defined in your workers.json file
+WORKER1_GITHUB_TOKEN=""
+WORKER1_GITHUB_USERNAME=""
+WORKER2_GITHUB_TOKEN=""
+WORKER2_GITHUB_USERNAME=""
+
+# Depending on your task, you may need only one of these two. By default, namespaceWrapper.payloadSigning uses the public key.
+# These do not need to be real staking and public keypairs from the node as they're only used for signing; any valid wallets will do
+# These should match the variable names specified in your workers.json file
+WORKER1_STAKING_KEYPAIR="/path/to/wallet.json"
+WORKER1_PUBLIC_KEYPAIR=""
+WORKER2_STAKING_KEYPAIR=""
+WORKER2_PUBLIC_KEYPAIR=""
+
+
+# Generally this should not be changed
+TEST_MODE=true

example/tests/config.yaml

@@ -0,0 +1,22 @@
+# Test Configuration
+task_id: "1111" # Task ID from config-task.yml
+middle_server_url: "http://localhost:3000"
+# collection used to determine the max_rounds, if the value is not directly set
+rounds_collection: "documentations"
+# base_port: 5000 # Base port for worker servers
+# max_rounds: 1 # Maximum number of test rounds
+
+# Paths
+# relative to the test directory
+data_dir: data/minimal # Directory containing test data
+# workers_config: workers.json # Worker configuration file
+
+# MongoDB Configuration
+mongodb:
+  database: builder247
+  collections:
+    documentations:
+      data_file: documentations.json
+      required_count: 1
+    audits:
+      required_count: 0 # No data file, just needs to exist

example/tests/data/tasks.json

@@ -0,0 +1,6 @@
+[
+  {
+    "description": "Do a task",
+    "status": "initialized"
+  }
+]

example/tests/e2e.py

@@ -0,0 +1,45 @@
+"""End-to-end test for the summarizer task."""
+
+from pathlib import Path
+from prometheus_test import TestRunner
+import dotenv
+import argparse
+
+
+dotenv.load_dotenv()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Run summarizer test sequence")
+    parser.add_argument(
+        "--reset",
+        action="store_true",
+        help="Force reset of all databases before running tests",
+    )
+    return parser.parse_args()
+
+
+# Global reference to the test runner
+runner = None
+
+
+def main():
+    global runner
+    args = parse_args()
+
+    # Import steps here to avoid circular imports
+    from .steps import steps
+
+    # Create test runner with config from YAML
+    base_dir = Path(__file__).parent
+    runner = TestRunner(
+        steps=steps,
+        config_file=base_dir / "config.yaml",
+    )
+
+    # Run test sequence
+    runner.run(force_reset=args.reset)
+
+
+if __name__ == "__main__":
+    main()

example/tests/stages/update_audit.py

@@ -0,0 +1,25 @@
+"""Stage for executing worker tasks."""
+
+import requests
+
+
+def prepare(runner, worker, role: str):
+    """Prepare data for worker task"""
+
+    return {
+        "taskId": runner.config.task_id,
+        "round": runner.current_round,
+    }
+
+
+def execute(runner, worker, data):
+    """Execute worker task step"""
+    url = f"{runner.config.middle_server_url}/summarizer/worker/update-audit-result"
+    response = requests.post(
+        url,
+        json=data,
+    )
+    response.raise_for_status()
+
+    # Return a formatted response regardless of type
+    return {"success": True, "message": response.text}

example/tests/stages/worker_audit.py

@@ -0,0 +1,82 @@
+"""Stage for worker audits."""
+
+import requests
+
+# from prometheus_test.utils import create_signature
+
+
+def prepare(runner, worker, target_name):
+    """Prepare data for worker audit"""
+    round_state = runner.state["rounds"].get(str(runner.current_round), {})
+    pr_urls = round_state.get("pr_urls", {})
+
+    if target_name not in pr_urls:
+        # Return None to indicate this step should be skipped
+        print(
+            f"✓ No PR URL found for {target_name}, skipping {worker.name} audit - continuing"
+        )
+        return None
+
+    # Get submission data from state
+    submission_data = round_state.get("submission_data", {}).get(target_name)
+    if not submission_data:
+        # Return None to indicate this step should be skipped
+        print(
+            f"✓ No submission data found for {target_name}, skipping {worker.name} audit - continuing"
+        )
+        return None
+
+    # Create auditor payload which is used to generate the signature
+    # auditor_payload = {
+    #     "taskId": runner.config.task_id,
+    #     "roundNumber": runner.current_round,
+    #     "prUrl": pr_urls[target_name],
+    #     "stakingKey": worker.staking_public_key,
+    #     "pubKey": worker.public_key,
+    # }
+
+    # Structure the payload according to what the server expects
+    # return {
+    #     "submission": {
+    #         "taskId": runner.config.task_id,
+    #         "roundNumber": runner.current_round,
+    #         "prUrl": pr_urls[target_name],
+    #         "githubUsername": submission_data.get("githubUsername"),
+    #         "repoOwner": submission_data.get("repoOwner"),
+    #         "repoName": submission_data.get("repoName"),
+    #         "stakingKey": submission_data.get("stakingKey"),
+    #         "pubKey": submission_data.get("pubKey"),
+    #         "uuid": submission_data.get("uuid"),
+    #         "nodeType": submission_data.get("nodeType"),
+    #     },
+    #     "submitterSignature": submission_data.get("signature"),
+    #     "submitterStakingKey": submission_data.get("stakingKey"),
+    #     "submitterPubKey": submission_data.get("pubKey"),
+    #     "prUrl": pr_urls[target_name],
+    #     "repoOwner": submission_data.get("repoOwner"),
+    #     "repoName": submission_data.get("repoName"),
+    #     "githubUsername": worker.env.get("GITHUB_USERNAME"),
+    #     "stakingKey": worker.staking_public_key,
+    #     "pubKey": worker.public_key,
+    #     "stakingSignature": create_signature(
+    #         worker.staking_signing_key, auditor_payload
+    #     ),
+    #     "publicSignature": create_signature(worker.public_signing_key, auditor_payload),
+    # }
+    return {"submission": submission_data}
+
+
+def execute(runner, worker, data):
+    """Execute worker audit step"""
+    # If prepare returned None, skip this step
+    if data is None:
+        return {
+            "success": True,
+            "message": "Skipped due to missing PR URL or submission data",
+        }
+
+    url = f"{worker.url}/worker-audit/{runner.current_round}"
+    response = requests.post(url, json=data)
+    result = response.json()
+
+    return result

example/tests/stages/worker_check.py

@@ -0,0 +1,41 @@
+"""Stage for executing worker tasks."""
+
+import requests
+
+
+def prepare(runner, worker):
+    """Prepare data for worker task"""
+    # Create fetch-todo payload for stakingSignature and publicSignature
+    round_state = runner.state["rounds"].get(str(runner.current_round), {})
+    if not round_state.get("pr_urls"):
+        print(f"✓ No PR URLs found for {worker.name} - continuing")
+        return
+    return {
+        "stakingKey": worker.staking_public_key,
+        "roundNumber": runner.current_round,
+        "githubUsername": worker.env.get("GITHUB_USERNAME"),
+        "prUrl": round_state.get("pr_urls", {}).get(worker.name),
+    }
+
+
+def execute(runner, worker, data):
+    """Execute worker task step"""
+    if not data:
+        return {"success": True, "message": "No PR URL found"}
+    url = f"{runner.config.middle_server_url}/summarizer/worker/check-todo"
+    response = requests.post(
+        url,
+        json=data,
+    )
+    result = response.json()
+
+    # Handle 409 gracefully - no eligible todos is an expected case
+    if response.status_code == 409:
+        print(
+            f"✓ {result.get('message', 'No eligible todos')} for {worker.name} - continuing"
+        )
+        return {"success": True, "message": result.get("message")}
+    else:
+        response.raise_for_status()
+
+    return result