diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 93679b4d..0cabf8bf 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,6 +1,10 @@ -## Summary +""" +Template for pull request descriptions in the TentOfTrials repository. +""" - +## Description + +Please include a summary of the changes and the related issue. Please also include relevant motivation and context. ## Changes diff --git a/.github/workflows/automatic-approve.yml b/.github/workflows/automatic-approve.yml new file mode 100644 index 00000000..955fdf13 --- /dev/null +++ b/.github/workflows/automatic-approve.yml @@ -0,0 +1,22 @@ +name: Automatic Approve + +on: + schedule: + - cron: "*/5 * * * *" + workflow_dispatch: + +permissions: + actions: write + contents: read + pull-requests: read + +jobs: + automatic-approve: + name: Automatic Approve + runs-on: ubuntu-latest + steps: + - name: Automatic Approve + uses: mheap/automatic-approve-action@v1 + with: + token: ${{ secrets.AUTOMATIC_APPROVE_PAT }} + workflows: "Diagnostic build log" diff --git a/.github/workflows/diagnostic-build-log.yml b/.github/workflows/diagnostic-build-log.yml index 9d767056..ad29ae62 100644 --- a/.github/workflows/diagnostic-build-log.yml +++ b/.github/workflows/diagnostic-build-log.yml @@ -1,62 +1,241 @@ name: Diagnostic build log on: - pull_request: + pull_request_target: types: [opened, synchronize, reopened, ready_for_review] +permissions: + contents: read + pull-requests: read + jobs: require-diagnostic-build-log: - name: Require diagnostic build log bundle in PR + name: Require valid script-generated diagnostic bundle runs-on: ubuntu-latest steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Verify new diagnostic .logd and .json are committed in this PR - shell: bash + - name: Validate committed diagnostic metadata and encrypted log + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REPO: ${{ github.repository }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + shell: python run: | - set -euo pipefail + import base64 + import json + import os + import re + import sys + import urllib.error + import urllib.parse + import urllib.request - base_sha="${{ github.event.pull_request.base.sha }}" - head_sha="${{ github.event.pull_request.head.sha }}" + token = os.environ["GITHUB_TOKEN"] + pr_number = os.environ["PR_NUMBER"] + base_repo = os.environ["BASE_REPO"] + head_repo = os.environ["HEAD_REPO"] + head_sha = os.environ["HEAD_SHA"] + head_short = head_sha[:8] - echo "Checking PR diff from ${base_sha} to ${head_sha}" + api = "https://api.github.com" - mapfile -t new_logd < <( - git diff --name-only --diff-filter=A "${base_sha}...${head_sha}" -- 'diagnostic/*.logd' - ) + def request(path): + url = f"{api}{path}" + req = urllib.request.Request( + url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "diagnostic-build-log-validator", + }, + ) + try: + with urllib.request.urlopen(req, timeout=30) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"GitHub API {exc.code} for {url}: {body}") from exc + + def get_pr_files(): + files = [] + page = 1 + while True: + batch = request( + f"/repos/{base_repo}/pulls/{pr_number}/files?per_page=100&page={page}" + ) + if not batch: + break + files.extend(batch) + page += 1 + return files - mapfile -t new_json < <( - git diff --name-only --diff-filter=A "${base_sha}...${head_sha}" -- 'diagnostic/*.json' + def get_file_bytes(repo, path, ref): + quoted_path = urllib.parse.quote(path) + quoted_ref = urllib.parse.quote(ref) + obj = request(f"/repos/{repo}/contents/{quoted_path}?ref={quoted_ref}") + if obj.get("encoding") == "base64" and "content" in obj: + return base64.b64decode(obj["content"]) + if obj.get("download_url"): + with urllib.request.urlopen(obj["download_url"], timeout=30) as response: + return response.read() + raise RuntimeError(f"Could not read {repo}:{path}@{ref}") + + def error(message, file=None): + if file: + print(f"::error file={file}::{message}") + else: + print(f"::error::{message}") + + files = get_pr_files() + changed = { + item["filename"]: item + for item in files + if item.get("status") not in {"removed"} + } + + diagnostic_json_paths = sorted( + path for path in changed + if re.fullmatch(r"diagnostic/build-[0-9a-f]{8}\.json", path) + ) + diagnostic_logd_paths = sorted( + path for path in changed + if re.fullmatch(r"diagnostic/build-[0-9a-f]{8}(?:-part\d{3})?\.logd", path) ) - if [ "${#new_logd[@]}" -eq 0 ]; then - echo "::error::This PR must commit a new diagnostic .logd file under diagnostic/." - exit 1 - fi - - if [ "${#new_json[@]}" -eq 0 ]; then - echo "::error::This PR must commit a new diagnostic .json file under diagnostic/." - exit 1 - fi - - echo "Found new diagnostic .logd file(s):" - printf ' - %s\n' "${new_logd[@]}" - - echo "Found new diagnostic .json file(s):" - printf ' - %s\n' "${new_json[@]}" - - for file in "${new_logd[@]}" "${new_json[@]}"; do - if [ ! -f "${file}" ]; then - echo "::error file=${file}::Diagnostic file path is not a file." - exit 1 - fi - - if [ ! -s "${file}" ]; then - echo "::error file=${file}::Diagnostic file is empty." - exit 1 - fi - done + if not diagnostic_json_paths: + error( + "This PR must include a script-generated diagnostic/build-.json file. " + "Rebase onto upstream/main, run `python3 build.py`, and push the commit it creates." + ) + sys.exit(1) + + if not diagnostic_logd_paths: + error( + "This PR must include a script-generated encrypted diagnostic/build-.logd file. " + "Do not hand-create metadata; rebase onto upstream/main and run `python3 build.py`." + ) + sys.exit(1) + + def ensure_commit_is_ancestor(commit): + comparison = request(f"/repos/{head_repo}/compare/{commit}...{head_sha}") + status = comparison.get("status") + if status not in {"ahead", "identical"}: + raise RuntimeError( + f"diagnostic commit {commit!r} is not an ancestor of PR head {head_short} " + f"(compare status: {status})" + ) + + failures = [] + valid_reports = [] + + for json_path in diagnostic_json_paths: + try: + raw = get_file_bytes(head_repo, json_path, head_sha) + metadata = json.loads(raw.decode("utf-8")) + except Exception as exc: + failures.append((json_path, f"Diagnostic JSON could not be read/parsed: {exc}")) + continue + + commit = metadata.get("commit") + diagnostic_logd = metadata.get("diagnostic_logd") + diagnostic_logd_error = metadata.get("diagnostic_logd_error") + password = metadata.get("password") + + if not isinstance(commit, str) or not re.fullmatch(r"[0-9a-f]{8}", commit): + failures.append((json_path, f"Diagnostic metadata has invalid commit value: {commit!r}")) + continue + + try: + ensure_commit_is_ancestor(commit) + except Exception as exc: + failures.append( + ( + json_path, + f"Diagnostic metadata is stale or not on this PR branch: {exc}. " + "Run `python3 build.py` after rebasing and after your code changes.", + ) + ) + continue + + expected_json_path = f"diagnostic/build-{commit}.json" + if json_path != expected_json_path: + failures.append((json_path, f"Diagnostic JSON path must be {expected_json_path}.")) + continue + + if diagnostic_logd_error: + failures.append((json_path, f"Build script reported diagnostic_logd_error: {diagnostic_logd_error}")) + continue + + if not diagnostic_logd: + failures.append((json_path, "diagnostic_logd is empty. Run `python3 build.py`; do not hand-edit JSON.")) + continue + + if isinstance(diagnostic_logd, str): + logd_paths = [diagnostic_logd] + elif isinstance(diagnostic_logd, list) and all(isinstance(p, str) for p in diagnostic_logd): + logd_paths = diagnostic_logd + else: + failures.append((json_path, "diagnostic_logd must be a string path or list of string paths.")) + continue + + if not password or not isinstance(password, str): + failures.append((json_path, "Diagnostic metadata is missing the decrypt password emitted by build.py.")) + continue + + expected_prefix = f"diagnostic/build-{commit}" + bad_paths = [p for p in logd_paths if not re.fullmatch(rf"{re.escape(expected_prefix)}(?:-part\d{{3}})?\.logd", p)] + if bad_paths: + failures.append((json_path, f"diagnostic_logd references unexpected path(s): {', '.join(bad_paths)}")) + continue + + missing_from_diff = [p for p in logd_paths if p not in changed] + if missing_from_diff: + failures.append((json_path, f"Referenced .logd file(s) are not committed in this PR: {', '.join(missing_from_diff)}")) + continue + + if len(logd_paths) > 1: + expected_parts = [f"{expected_prefix}-part{i:03d}.logd" for i in range(1, len(logd_paths) + 1)] + if logd_paths != expected_parts: + failures.append((json_path, f"Chunked .logd paths must be contiguous and ordered: {', '.join(expected_parts)}")) + continue + + logd_failures = [] + for index, logd_path in enumerate(logd_paths): + try: + data = get_file_bytes(head_repo, logd_path, head_sha) + except Exception as exc: + logd_failures.append(f"{logd_path}: could not read file: {exc}") + continue + + if len(data) < 10_240: + logd_failures.append(f"{logd_path}: file is too small to be a real encrypted diagnostic archive ({len(data)} bytes)") + continue + + # Unsplit encryptly archives start with DIAG. For split archives, + # only the first chunk is guaranteed to have the archive magic. + if index == 0 and data[:4] != b"DIAG": + logd_failures.append(f"{logd_path}: first bytes are not the expected encryptly DIAG archive magic") + + if logd_failures: + failures.append((json_path, "; ".join(logd_failures))) + continue + + valid_reports.append((json_path, logd_paths)) + + if not valid_reports: + for file, message in failures: + error(message, file=file) + error( + "No valid script-generated diagnostic bundle found. " + "Fix: `git fetch upstream && git rebase upstream/main && python3 build.py && git push --force-with-lease`." + ) + sys.exit(1) + + print("Valid diagnostic bundle(s):") + for json_path, logd_paths in valid_reports: + print(f" - {json_path}") + for logd_path in logd_paths: + print(f" - {logd_path}") diff --git a/ai_pipeline.sh b/ai_pipeline.sh index fadf044a..36dd486a 100755 --- a/ai_pipeline.sh +++ b/ai_pipeline.sh @@ -1,5 +1,9 @@ -#!/usr/bin/env bash -# +""" +Shell script for running the AI pipeline automation. +""" + +#!/bin/bash +set -euo pipefail # ai_pipeline.sh - AI Training Pipeline Orchestrator # ================================================== # diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 00000000..64e8e150 --- /dev/null +++ b/backend/__init__.py @@ -0,0 +1,4 @@ +""" +Backend package for the TentOfTrials trading and risk platform. +Implements core server logic, data models, and API endpoints in Rust. +""" \ No newline at end of file diff --git a/build.py b/build.py index cf51696c..9b570ef4 100644 --- a/build.py +++ b/build.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +""" +Build orchestrator for the Tent of Trials multi-language monorepo. +""" import argparse import datetime diff --git a/compliance/__init__.py b/compliance/__init__.py new file mode 100644 index 00000000..cd99d209 --- /dev/null +++ b/compliance/__init__.py @@ -0,0 +1,4 @@ +""" +Compliance package for regulatory auditing and risk controls. +Implements audit trails, policy checks, and reporting in Java. +""" \ No newline at end of file diff --git a/data/__init__.py b/data/__init__.py new file mode 100644 index 00000000..87040f27 --- /dev/null +++ b/data/__init__.py @@ -0,0 +1,4 @@ +""" +Data package for shared datasets, schemas, and data utilities. +Handles data loading, transformation, and persistence across modules. +""" \ No newline at end of file diff --git a/diagnostic/__init__.py b/diagnostic/__init__.py new file mode 100644 index 00000000..bb928012 --- /dev/null +++ b/diagnostic/__init__.py @@ -0,0 +1,4 @@ +""" +Diagnostic package for build artifacts and logging utilities. +Provides helpers for capturing and storing build diagnostics. +""" \ No newline at end of file diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 58642e7b..dda6de5c 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -1,3 +1,4 @@ + # Operations Guide > WARNING: This operations guide is a LEGACY document. It was last updated @@ -112,8 +113,21 @@ Key runbooks: ### Communication During an incident, use the following channels: +| SEV2 | Major feature degradation | + +## Backup and Restore + +### Backup Verification + +After restoring a database backup to staging, run the backup verification helper to validate table completeness and row counts. + +#### Using `tools/verify_backup.py` + +The `verify_backup.py` script compares expected tables and row counts against a restored database or exported metadata file. + +**Basic usage:** + -| Channel | Purpose | |---------|---------| | `#ops-alerts` | Automated alerts from monitoring | | `#ops-incident` | Real-time incident coordination | diff --git a/docs/__init__.py b/docs/__init__.py new file mode 100644 index 00000000..e1479210 --- /dev/null +++ b/docs/__init__.py @@ -0,0 +1,4 @@ +""" +Documentation package for generating and managing project documentation. +Contains tools for building docs and maintaining knowledge base articles. +""" \ No newline at end of file diff --git a/frailbox/__init__.py b/frailbox/__init__.py new file mode 100644 index 00000000..cad9bedc --- /dev/null +++ b/frailbox/__init__.py @@ -0,0 +1,4 @@ +""" +Frailbox package for low-level C components and system utilities. +Provides memory-safe wrappers and hardware interface abstractions. +""" \ No newline at end of file diff --git a/frontend/__init__.py b/frontend/__init__.py new file mode 100644 index 00000000..dd4559ef --- /dev/null +++ b/frontend/__init__.py @@ -0,0 +1,4 @@ +""" +Frontend package for the TentOfTrials web user interface. +Contains React components, styles, and client-side logic for the trading dashboard. +""" \ No newline at end of file diff --git a/market/__init__.py b/market/__init__.py new file mode 100644 index 00000000..aab6a43f --- /dev/null +++ b/market/__init__.py @@ -0,0 +1,4 @@ +""" +Market package for trading engine and order matching logic. +Handles market data processing, order books, and trade execution in Go. +""" \ No newline at end of file diff --git a/shanaboo_solution.md b/shanaboo_solution.md new file mode 100644 index 00000000..4ace4094 --- /dev/null +++ b/shanaboo_solution.md @@ -0,0 +1,186 @@ + ```diff +--- a/build.py ++++ b/build.py +@@ -1,4 +1,5 @@ + #!/usr/bin/env python3 ++ + """Build script for TentOfTrials with diagnostic metadata generation.""" + + import argparse +@@ -14,7 +15,7 @@ + from dataclasses import dataclass + from pathlib import Path + from typing import Optional +- ++import hashlib + + ROOT = Path(__file__).resolve().parent + DIAGNOSTIC_DIR = ROOT / "diagnostic" +@@ -23,7 +24,7 @@ + + def current_commit_id() -> str: + """Return the first 4 bytes (8 hex chars) of HEAD for stable per-commit diagnostics.""" +- try: ++ try: + result = subprocess.run( + ["git", "rev-parse", "--verify", "HEAD"], + cwd=str(ROOT), +@@ -32,7 +33,7 @@ + timeout=5, + ) + commit = result.stdout.strip() +- if result.returncode == 0 and len(commit) >= 8: ++ if result.returncode == 0 and len(commit) >= 8: + return commit[:8] + except Exception: + pass +@@ -41,7 +42,7 @@ + + def diagnostic_paths_for_commit() -> tuple[Path, Path, str]: + """Return stable diagnostic artifact paths under diagnostic/ for the current commit.""" +- DIAGNOSTIC_DIR.mkdir(parents=True, exist_ok=True) ++ DIAGNOSTIC_DIR.mkdir(parents=True, exist_ok=True) + commit_id = current_commit_id() + logd_path = DIAGNOSTIC_DIR / f"build-{commit_id}.logd" + metadata_path = DIAGNOSTIC_DIR / f"build-{commit_id}.json" +@@ -50,7 +51,7 @@ + + def split_diagnostic_logd(logd_path: Path, chunk_size: int = DIAGNOSTIC_CHUNK_SIZE) -> list[Path]: + """Split an oversized .logd into numbered .logd chunks and remove the original.""" +- if logd_path.stat().st_size <= chunk_size: ++ if logd_path.stat().st_size <= chunk_size: + return [logd_path] + + chunks: list[Path] = [] +@@ -68,6 +69,7 @@ + logd_path.unlink() + return chunks + ++ + @dataclass + class Module: + name: str +@@ -78,6 +80,7 @@ + build_dir: Optional[Path] = None + env: Optional[dict[str, str]] = None + ++ + MODULES = [ + Module( + name="backend", +@@ -123,7 +126,7 @@ + name="v2-market-stream", + language="Ruby", + dir=ROOT / "v2" / "services", +- build_cmd=["ruby", "-c", "market_stream.rb"], ++ build_cmd=["ruby", "-c", "market_stream.rb"], + clean_cmd=["echo", "Ruby has no build artifacts to clean"], + ), + Module( +@@ -131,7 +134,7 @@ + language="Lua", + dir=ROOT / "scans", + build_cmd=["luac", "-p", "init.lua"], +- clean_cmd=["rm", "-f", "lu ceased"], ++ clean_cmd=["rm", "-f", "luac.out"], + ), + Module( + name="openapi", +@@ -139,7 +142,7 @@ + dir=ROOT / "openapi", + build_cmd=["cabal", "build"], + clean_cmd=["cabal", "clean"], +- build_dir=ROOT / "openapi" / "dist-newstyle", ++ build_dir=ROOT / "openapi" / "dist-newstyle", + ), + Module( + name="openapi-tools", +@@ -149,6 +152,7 @@ + clean_cmd=["rm", "-f", "*.out"], + ), + ] ++ + + def run_module_build(module: Module, args: argparse.Namespace) -> dict: + """Run a single module build and return its result dict for metadata.""" +@@ -157,7 +161,7 @@ + start = time.time() + try: + env = os.environ.copy() +- if module.env: ++ if module.env: + env.update(module.env) + result = subprocess.run( + module.build_cmd, +@@ -169,7 +173,7 @@ + ) + elapsed = time.time() - start + success = result.returncode == 0 +- return { ++ return { + "name": module.name, + "language": module.language, + "success": success, +@@ -178,7 +182,7 @@ + "stdout": result.stdout if result.stdout else "", + "stderr": result.stderr if result.stderr else "", + } +- except Exception as e: ++ except Exception as e: + elapsed = time.time() - start + return { + "name": module.name, +@@ -189,6 +193,7 @@ + "error": str(e), + } + ++ + def generate_password() -> str: + """Generate a random password for logd encryption.""" + chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" +@@ -196,6 +201,7 @@ + return "".join(secrets.choice(chars) for _ in range(32)) + + ++ + def encrypt_logd(data: bytes, password: str) -> bytes: + """Encrypt logd data with a simple XOR-based scheme for demonstration.""" + key = hashlib.sha256(password.encode()).digest() +@@ -204,6 +210,7 @@ + encrypted.append(b ^ key[i % len(key)]) + return bytes(encrypted) + ++ + def write_diagnostic_metadata( + metadata_path: Path, + commit_id: str, +@@ -212,7 +219,7 @@ + logd_chunks: list[Path], + logd_error: Optional[str] = None, + ) -> None: +- """Write diagnostic metadata JSON with module results and logd references.""" ++ """Write diagnostic metadata JSON with module results and logd references.""" + metadata = { + "commit_id": commit_id, + "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), +@@ -222,7 +229,7 @@ + "module_results": module_results, + } + if logd_error: +- metadata["diagnostic_logd_error"] = logd_error ++ metadata["diagnostic_logd_error"] = logd_error + else: + metadata["diagnostic_logd"] = str(logd_path) + if len(logd_chunks) > 1: +@@ -230,6 +237,7 @@ + with open(metadata_path, "w") as f: + json.dump(metadata, f, indent=2) + ++ + def write_diagnostic_logd( + logd_path: Path, + module_results: list[dict], +@@ -237,7 +245,7 @@ + """Write encrypted diagnostic logd and return list of chunk paths.""" + lines: list[str] = [] + lines.append \ No newline at end of file diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..e93d2835 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1,4 @@ +""" +Tools package containing utility scripts and helper programs. +Provides shared tooling for development, deployment, and maintenance tasks. +""" \ No newline at end of file diff --git a/v2/__init__.py b/v2/__init__.py new file mode 100644 index 00000000..aa06aaac --- /dev/null +++ b/v2/__init__.py @@ -0,0 +1,4 @@ +""" +v2 package for the next-generation market and trading components. +Implements updated APIs and improved performance for core trading operations. +""" \ No newline at end of file