feat: add kubernetes execution engine

.dockerignore

@@ -0,0 +1,31 @@
+# Runtime artifacts that must never be baked into the image
+harbor_test_agent_results/
+
+# Local evaluator with 2 GB of cloned SWE-bench repos — never needed in K8s
+evaluator/
+__pycache__/
+**/__pycache__/
+*.pyc
+*.pyo
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.venv/
+
+# Local dev / secrets
+.env
+**/.env
+*.env.local
+
+# Dev scripts and notebooks
+hack/
+docs/
+
+# .git is excluded: COMMIT_HASH is injected as GIT_COMMIT build arg at image build time.
+# For local dev without a built image, utils/git.py falls back to git rev-parse HEAD.
+.git
+
+# Editor
+.cursor/
+.vscode/
+*.swp

.github/workflows/build.yml

@@ -0,0 +1,66 @@
+name: Build and Push Images
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  push:
+    branches: [main]
+    tags: ["v*"]
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  REGISTRY: ghcr.io
+
+jobs:
+  build:
+    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: ridges-api
+            dockerfile: Dockerfile.api
+          - image: ridges-validator
+            dockerfile: Dockerfile.validator
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/ridgesai/${{ matrix.image }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha,prefix=
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ${{ matrix.dockerfile }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: GIT_COMMIT=${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -31,3 +31,8 @@ analyzer/
 # OLD
 subnet_hotkeys_cache.json
 .python-version
+
+# Operational metadata
+utils/registered_hotkeys.json
+
+.idea/

Dockerfile.api

@@ -0,0 +1,28 @@
+# --- builder ---
+FROM python:3.12-slim AS builder
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+WORKDIR /app
+
+# Install dependencies first (cached layer — only invalidated when lockfile changes)
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --no-dev --no-install-project
+
+# Install the project itself
+COPY . .
+RUN uv sync --frozen --no-dev
+
+# --- runtime ---
+FROM python:3.12-slim
+
+WORKDIR /app
+
+COPY --from=builder /app /app
+
+ARG GIT_COMMIT=unknown
+ENV GIT_COMMIT=${GIT_COMMIT}
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/app/.venv/bin:$PATH"
+
+ENTRYPOINT ["python", "-m", "api.src.main"]

Dockerfile.validator

@@ -0,0 +1,38 @@
+# --- builder ---
+FROM python:3.12-slim AS builder
+
+RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+
+WORKDIR /app
+
+# Install dependencies first (cached layer — only invalidated when lockfile changes)
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen --no-dev --no-install-project
+
+# Install the project itself
+COPY . .
+RUN uv sync --frozen --no-dev
+
+# --- runtime ---
+FROM python:3.12-slim
+
+RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/*
+
+RUN useradd --create-home --shell /bin/bash ridges
+
+WORKDIR /app
+
+COPY --from=builder /app /app
+
+RUN chown -R ridges:ridges /app
+
+USER ridges
+
+ARG GIT_COMMIT=unknown
+ENV GIT_COMMIT=${GIT_COMMIT}
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/app/.venv/bin:$PATH"
+
+ENTRYPOINT ["python", "-m", "validator.main"]

Makefile

@@ -40,3 +40,102 @@ migrations-history: $(_DB_GUARDS)
 ## Show the current applied revision.
 migrations-current: $(_DB_GUARDS)
 	$(_DB_ENV) uv run alembic current
+
+# ---------------------------------------------------------------------------
+# Kubernetes local dev (kind)
+#
+# Prerequisites: kind, kubectl, docker, kustomize
+# Usage:
+#   make k8s-setup     — full cluster from scratch (run once)
+#   make k8s-images    — rebuild + reload validator image after code changes
+#   make k8s-network   — re-apply manifests after editing k8s/ files
+#   make k8s-screener  — run screener locally against the kind cluster
+#   make k8s-down      — destroy the kind cluster
+# ---------------------------------------------------------------------------
+
+K8S_CLUSTER := ridges
+K8S_NS      := ridges
+K8S_CTX     := kind-$(K8S_CLUSTER)
+KUBECTL     := kubectl --context=$(K8S_CTX)
+
+.PHONY: k8s-up k8s-down k8s-images k8s-network k8s-setup k8s-screener
+
+## Create kind cluster with Calico CNI and ridges namespace (skip if already running)
+k8s-up:
+	@if kind get clusters 2>/dev/null | grep -qx '$(K8S_CLUSTER)'; then \
+	    echo "Kind cluster '$(K8S_CLUSTER)' already exists — reusing"; \
+	else \
+	    kind create cluster --name $(K8S_CLUSTER) --config k8s/kind-config.yaml; \
+	    $(KUBECTL) apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.27.0/manifests/calico.yaml; \
+	    $(KUBECTL) wait --for=condition=ready pod -l k8s-app=calico-node -n kube-system --timeout=180s; \
+	fi
+	$(KUBECTL) create namespace $(K8S_NS) --dry-run=client -o yaml | $(KUBECTL) apply -f -
+
+## Destroy the kind cluster
+k8s-down:
+	kind delete cluster --name $(K8S_CLUSTER)
+
+## Build and load the validator/screener image into every kind node.
+## Re-run this after any code change to pick up the new image.
+k8s-images:
+	docker build -t ridges-validator:latest -f Dockerfile.validator \
+	    --build-arg GIT_COMMIT=$$(git rev-parse HEAD) .
+	kind load docker-image ridges-validator:latest --name $(K8S_CLUSTER)
+
+## Apply all k8s/ manifests (NetworkPolicies, RBAC, registry, dockerhost).
+## Re-run this after editing any file under k8s/.
+k8s-network:
+	kustomize build k8s/local | $(KUBECTL) apply -f -
+	$(KUBECTL) wait --for=condition=ready pod -l app=registry -n $(K8S_NS) --timeout=120s
+
+## Full setup: cluster → registry creds → images → manifests → secrets.
+## Run once after k8s-up (or after k8s-down to start fresh).
+k8s-setup: k8s-up
+	@# --- Registry credentials (idempotent, auto-generated on first run) ---
+	@if ! $(KUBECTL) get secret registry-htpasswd -n $(K8S_NS) >/dev/null 2>&1; then \
+	    echo "Generating registry credentials..."; \
+	    PASS=$$(python3 -c "import secrets; print(secrets.token_urlsafe(32))"); \
+	    HTPASSWD=$$(docker run --rm httpd:2-alpine htpasswd -bBn kaniko "$$PASS"); \
+	    $(KUBECTL) create secret generic registry-htpasswd \
+	        --from-literal=htpasswd="$$HTPASSWD" --namespace=$(K8S_NS); \
+	    $(KUBECTL) create secret docker-registry registry-creds \
+	        --docker-server=registry.$(K8S_NS).svc:5000 \
+	        --docker-username=kaniko --docker-password="$$PASS" \
+	        --namespace=$(K8S_NS); \
+	    $(KUBECTL) create secret generic registry-password \
+	        --from-literal=password="$$PASS" --namespace=$(K8S_NS); \
+	else \
+	    echo "Registry credentials already exist — skipping"; \
+	fi
+	@# --- Configure Kind nodes for plain-HTTP pulls from the in-cluster registry ---
+	@REGISTRY_IP=$$($(KUBECTL) get svc registry -n $(K8S_NS) -o jsonpath='{.spec.clusterIP}' 2>/dev/null || echo ""); \
+	for node in $$(kind get nodes --name $(K8S_CLUSTER)); do \
+	    docker exec $$node mkdir -p /etc/containerd/certs.d/registry.$(K8S_NS).svc:5000; \
+	    printf '[host."http://registry.$(K8S_NS).svc:5000"]\n' | \
+	        docker exec -i $$node tee /etc/containerd/certs.d/registry.$(K8S_NS).svc:5000/hosts.toml > /dev/null; \
+	    docker exec $$node sh -c \
+	        "grep -q 'registry.$(K8S_NS).svc' /etc/hosts || echo '$$REGISTRY_IP registry.$(K8S_NS).svc' >> /etc/hosts"; \
+	done
+	$(MAKE) k8s-images
+	$(MAKE) k8s-network
+	@# --- Screener secrets from validator/.env ---
+	@if [ ! -f validator/.env ]; then \
+	    echo "ERROR: validator/.env not found. Copy validator/.env.example and fill in values."; \
+	    exit 1; \
+	fi
+	$(KUBECTL) create secret generic ridges-screener-secrets \
+	    --from-env-file=validator/.env --namespace=$(K8S_NS) \
+	    --dry-run=client -o yaml | $(KUBECTL) apply -f -
+	@echo ""
+	@echo "Kind cluster ready — context: $(K8S_CTX)"
+	@echo "  Run:   make k8s-screener"
+	@echo "  Scale: $(KUBECTL) scale sts ridges-screener-1 --replicas=1 -n $(K8S_NS)"
+
+## Run screener locally against the kind cluster
+k8s-screener:
+	RIDGES_ENVIRONMENT_TYPE=kubernetes \
+	K8S_CONTEXT=$(K8S_CTX) \
+	K8S_NAMESPACE=$(K8S_NS) \
+	K8S_REGISTRY=registry.$(K8S_NS).svc:5000 \
+	uv run python -m validator.main
+

api/endpoints/scaling.py

@@ -0,0 +1,30 @@
+from fastapi import APIRouter
+
+from api.endpoints.validator import SESSION_ID_TO_VALIDATOR
+from queries.agent import get_pending_work_counts
+
+router = APIRouter()
+
+
+@router.get("/pending-work")
+async def get_pending_work():
+    """Return queue depths for KEDA autoscaling and screener capacity for observability.
+
+    KEDA's metrics-api trigger reads ``screener_1_pending`` and
+    ``screener_2_pending`` via ``valueLocation``.  The ``*_connected``,
+    ``*_busy``, and ``*_idle`` fields are not consumed by KEDA but are useful
+    for dashboards.
+
+    No authentication — this endpoint is ClusterIP-only.
+    """
+    counts = await get_pending_work_counts()
+
+    for class_num in ("1", "2"):
+        prefix = f"screener-{class_num}-"
+        screeners = [v for v in SESSION_ID_TO_VALIDATOR.values() if v.hotkey.startswith(prefix)]
+        busy = sum(1 for s in screeners if s.current_evaluation_id is not None)
+        counts[f"screener_{class_num}_connected"] = len(screeners)
+        counts[f"screener_{class_num}_busy"] = busy
+        counts[f"screener_{class_num}_idle"] = len(screeners) - busy
+
+    return counts

api/src/main.py

@@ -18,6 +18,7 @@
 from api.endpoints.evaluation_sets import router as evaluation_sets_router
 from api.endpoints.evaluations import router as evaluations_router
 from api.endpoints.retrieval import router as retrieval_router
+from api.endpoints.scaling import router as scaling_router
 from api.endpoints.scoring import router as scoring_router
 from api.endpoints.statistics import router as statistics_router
 from api.endpoints.validator import router as validator_router
@@ -132,6 +133,7 @@ async def lifespan(app: FastAPI):
 app.include_router(evaluation_run_router, prefix="/evaluation-run")
 app.include_router(evaluations_router, prefix="/evaluation")
 app.include_router(statistics_router, prefix="/statistics")
+app.include_router(scaling_router, prefix="/scaling")
 
 
 if __name__ == "__main__":

docker-compose.yml

@@ -5,9 +5,9 @@ services:
     ports:
       - "5432:5432"
     environment:
-      POSTGRES_USER: 
-      POSTGRES_PASSWORD: 
-      POSTGRES_DB: 
+      POSTGRES_USER: ridges
+      POSTGRES_PASSWORD: ridges123
+      POSTGRES_DB: ridges
     volumes:
       - postgres_data:/var/lib/postgresql/data
     healthcheck:
@@ -20,24 +20,31 @@ services:
   api:
     build:
       context: .
-      dockerfile: ./api/Dockerfile
+      dockerfile: ./Dockerfile.api
     ports:
       - "8000:8000"
     env_file:
       - ./api/.env
+    volumes:
+      - ./.git:/app/.git:ro
     depends_on:
       db:
         condition: service_healthy
-      s3mock:
+      s3:
         condition: service_started
 
-  s3mock:
+  s3:
     image: adobe/s3mock:latest
     environment:
-      - COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS=ridges-local
+      - COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS=ridges-local,ridges-registry
+      - COM_ADOBE_TESTING_S3MOCK_STORE_ROOT=/s3mockroot
+      - COM_ADOBE_TESTING_S3MOCK_STORE_RETAINFILESONEXIT=true
     ports:
       - "9090:9090"
       - "9191:9191"
+    volumes:
+      - s3_data:/s3mockroot
 
 volumes:
   postgres_data:
+  s3_data: