feat: infrastructure — AWS EC2 deploy, Docker stack, Grafana, Tailscale

.dockerignore

@@ -1,5 +1,33 @@
-.vscode
-node_modules
-bots/*/
-!bots/*
-keys.json
+# Runtime bot data — large and should never be in the image
+bots/*/logs/
+bots/*/histories/
+bots/*/action-code/
+bots/*/ensemble_log.json
+
+# Git history
+.git/
+
+# Local secrets / keys
+keys.json
+.env
+.env.*
+
+# Node dev artifacts
+node_modules/
+
+# Editor / OS
+.vscode/
+*.DS_Store
+Thumbs.db
+
+# Tasks output
+tasks/**/__pycache__/
+tasks/**/*.pyc
+
+# AWS deploy scripts (not needed in container)
+aws/
+
+# Docs not needed at runtime
+docs/
+*.md
+!README.md

Dockerfile

@@ -27,9 +27,19 @@ RUN apt-get update && \
 
 WORKDIR /app
 
-COPY package*.json .
+# Copy package files and patches for better caching
+COPY package*.json ./
+COPY patches/ ./patches/
 RUN npm install
 
+# Copy source code
 COPY . .
 
+# Run tests during build
+RUN npm test
+
+# Drop root privileges — node:slim includes a non-root 'node' user
+RUN chown -R node:node /app
+USER node
+
 CMD ["npm", "start"]

Tasks.Dockerfile

@@ -1,51 +1,49 @@
-# Specify a base image
-# FROM ubuntu:22.04
-FROM node:18
-
-#Install some dependencies
-
-RUN apt-get -y update
-RUN apt-get -y install git
-RUN apt-get -y install unzip
-RUN apt-get -y install python3
-RUN apt-get -y install python3-pip
-RUN apt-get -y install python3-boto3
-RUN apt-get -y install python3-tqdm
-RUN apt-get -y install tmux
-
-RUN git clone https://github.com/mindcraft-bots/mindcraft.git /mindcraft
+# Tasks.Dockerfile — Evaluation / benchmark runner
+# Builds a container with Mindcraft + Java 21 + AWS CLI for automated tasks.
+
+FROM node:22-slim AS base
+
+# ── System dependencies (single layer) ──────────────────────────────────────
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      git unzip curl wget ca-certificates gnupg lsb-release \
+      python3 python3-pip python3-boto3 python3-tqdm tmux \
+      apt-transport-https \
+    && rm -rf /var/lib/apt/lists/*
+
+# ── Adoptium Java 21 (proper GPG keyring, not deprecated apt-key) ───────────
+RUN mkdir -p /etc/apt/keyrings \
+    && wget -qO- https://packages.adoptium.net/artifactory/api/gpg/key/public \
+       | gpg --dearmor -o /etc/apt/keyrings/adoptium.gpg \
+    && echo "deb [signed-by=/etc/apt/keyrings/adoptium.gpg] \
+       https://packages.adoptium.net/artifactory/deb $(lsb_release -cs) main" \
+       > /etc/apt/sources.list.d/adoptium.list \
+    && apt-get update && apt-get install -y --no-install-recommends temurin-21-jdk \
+    && rm -rf /var/lib/apt/lists/*
+
+# ── AWS CLI v2 ──────────────────────────────────────────────────────────────
+RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip \
+    && unzip -q /tmp/awscliv2.zip -d /tmp \
+    && /tmp/aws/install \
+    && rm -rf /tmp/awscliv2.zip /tmp/aws
+
+# ── Application code ───────────────────────────────────────────────────────
 WORKDIR /mindcraft
-COPY ./server_data.zip /mindcraft
-RUN unzip server_data.zip
+# Copy source from the build context (this repo) rather than cloning from
+# GitHub at build time. A live git clone:
+#   1. Breaks reproducibility (upstream HEAD can change between builds)
+#   2. Fails in offline/air-gapped CI environments
+#   3. Introduces supply-chain risk (external fetch during image build)
+COPY . .
 
-RUN npm install
+COPY ./server_data.zip /mindcraft/
+RUN unzip -q server_data.zip && rm server_data.zip
 
+RUN npm ci --omit=dev
 
-# Copy the rest of the application code to the working directory
-# RUN apt update
-# RUN apt install bash ca-certificates wget git -y # install first to avoid openjdk install bug
-# RUN apt install openjdk-17-jre-headless -y
-RUN apt install -y wget apt-transport-https gnupg lsb-release
-
-# Add Adoptium repository key
-RUN wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add -
-
-# Add Adoptium repository
-RUN echo "deb https://packages.adoptium.net/artifactory/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/adoptium.list
-
-# Update package lists
-RUN apt update
-
-# Install Temurin (Adoptium) Java 21
-RUN apt install temurin-21-jdk -y
-
-# Install unzip
-
-
-RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
-RUN unzip awscliv2.zip
-RUN ./aws/install
+# ── Non-root user ──────────────────────────────────────────────────────────
+RUN groupadd -r mindcraft && useradd -r -g mindcraft -d /mindcraft mindcraft \
+    && chown -R mindcraft:mindcraft /mindcraft
+USER mindcraft
 
 VOLUME /data
-
 EXPOSE 8000

aws/backup.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+# =============================================================================
+# aws/backup.sh — Backup Minecraft world and bot memory to S3
+# =============================================================================
+# Run manually: bash aws/backup.sh
+# Also runs automatically every 6 hours via cron (installed by deploy.sh)
+# =============================================================================
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+CONFIG_FILE="${SCRIPT_DIR}/config.env"
+
+GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m'
+info() { echo -e "${GREEN}[BACKUP $(date '+%Y-%m-%d %H:%M:%S')]${NC} $*"; }
+warn() { echo -e "${YELLOW}[WARN]${NC} $*"; }
+
+# ── Detect if running locally (SSH to EC2) or on EC2 directly ────────────────
+if [[ -f "$CONFIG_FILE" ]]; then
+  # Running locally — SSH to EC2 and run backup there
+  # shellcheck source=/dev/null
+  source "$CONFIG_FILE"
+  [[ -n "${EC2_IP:-}" ]] || { echo "EC2_IP not set"; exit 1; }
+  SSH_OPTS="-i ${KEY_FILE} -o StrictHostKeyChecking=no"
+  info "Running backup on EC2 (${EC2_IP}) via SSH..."
+  ssh ${SSH_OPTS} ubuntu@${EC2_IP} "bash /app/aws/backup.sh"
+  exit 0
+fi
+
+# ── Running ON EC2 ────────────────────────────────────────────────────────────
+# Get region and bucket from instance metadata + SSM
+REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/region 2>/dev/null \
+  || echo "${AWS_DEFAULT_REGION:-us-east-1}")
+S3_BUCKET=$(aws ssm get-parameter \
+  --region "$REGION" \
+  --name "/mindcraft/S3_BUCKET" \
+  --with-decryption \
+  --query 'Parameter.Value' \
+  --output text 2>/dev/null \
+  || grep S3_BUCKET /app/.env | cut -d= -f2 || "")
+
+[[ -n "$S3_BUCKET" ]] || { echo "ERROR: S3_BUCKET not found"; exit 1; }
+
+APP_DIR="/app"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+
+# ── Stop Minecraft briefly (prevents corrupted world files) ───────────────────
+MINECRAFT_WAS_RUNNING=false
+if docker compose -f "${APP_DIR}/docker-compose.aws.yml" ps minecraft 2>/dev/null | grep -q "Up"; then
+  MINECRAFT_WAS_RUNNING=true
+  info "Stopping Minecraft for consistent backup..."
+  docker compose -f "${APP_DIR}/docker-compose.aws.yml" stop minecraft
+  sleep 2
+fi
+
+# ── Backup world to S3 ────────────────────────────────────────────────────────
+info "Syncing minecraft-data → s3://${S3_BUCKET}/minecraft-data/ ..."
+aws s3 sync \
+  "${APP_DIR}/minecraft-data" \
+  "s3://${S3_BUCKET}/minecraft-data/" \
+  --sse AES256 \
+  --region "$REGION" \
+  --delete
+
+# ── Backup bot memory to S3 ───────────────────────────────────────────────────
+info "Syncing bots/ memory → s3://${S3_BUCKET}/bots/ ..."
+# Only sync memory.json and learnings.json (skip histories/ which are huge)
+find "${APP_DIR}/bots" -maxdepth 2 \( -name "memory.json" -o -name "learnings.json" \) \
+  -exec aws s3 cp {} "s3://${S3_BUCKET}/bots/$(basename "$(dirname {})")/$(basename {})" \
+    --sse AES256 --region "$REGION" \;
+
+# ── Restart Minecraft ─────────────────────────────────────────────────────────
+if [[ "$MINECRAFT_WAS_RUNNING" == "true" ]]; then
+  info "Restarting Minecraft..."
+  docker compose -f "${APP_DIR}/docker-compose.aws.yml" start minecraft
+fi
+
+info "Backup complete. s3://${S3_BUCKET}/ (timestamp: ${TIMESTAMP})"