From a5ddf9438297a90112ccbe33886b9f78a6cec2ac Mon Sep 17 00:00:00 2001 From: Tyler Date: Tue, 3 Mar 2026 20:05:29 -0500 Subject: [PATCH] =?UTF-8?q?feat:=20infrastructure=20&=20deployment=20?= =?UTF-8?q?=E2=80=94=20AWS=20EC2,=20Docker,=20Grafana,=20Tailscale?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - aws/: EC2 deploy/teardown scripts, S3 backup/restore, Ollama proxy setup, user-data bootstrap, env toggle for prod/dev switching - docker-compose.yml: multi-container stack (bot + viaproxy + chromadb) - docker-compose.aws.yml: EC2 production override with LiteLLM proxy + Tailscale - Dockerfile: multi-stage build, non-root node user, secrets excluded from context - Tasks.Dockerfile: isolated task runner container - prometheus-aws.yml: Prometheus scrape config for EC2 metrics - grafana-provisioning/: pre-built dashboards and alerting rules - start.ps1: cross-platform startup helper --- .dockerignore | 38 +- Dockerfile | 12 +- Tasks.Dockerfile | 86 ++-- aws/backup.sh | 77 +++ aws/deploy.sh | 212 +++++++++ aws/ec2-deploy.sh | 206 ++++++++ aws/ec2-go.sh | 244 ++++++++++ aws/env-toggle.sh | 145 ++++++ aws/restore.sh | 79 ++++ aws/s3-policy.json | 51 ++ aws/setup-ollama-proxy.sh | 93 ++++ aws/setup.sh | 472 +++++++++++++++++++ aws/teardown.sh | 150 ++++++ aws/user-data.sh | 73 +++ docker-compose.aws.yml | 280 +++++++++++ docker-compose.yml | 209 +++++++- grafana-provisioning/alerting/.gitkeep | 1 + grafana-provisioning/alerting/rules.yml | 7 + grafana-provisioning/dashboard-json/.gitkeep | 1 + grafana-provisioning/dashboards.yml | 11 + grafana-provisioning/datasources.yml | 9 + prometheus-aws.yml | 16 + services/viaproxy/README.md | 19 +- start.ps1 | 19 + 24 files changed, 2445 insertions(+), 65 deletions(-) create mode 100644 aws/backup.sh create mode 100644 aws/deploy.sh create mode 100644 aws/ec2-deploy.sh create mode 100644 aws/ec2-go.sh create mode 100644 aws/env-toggle.sh create mode 100644 aws/restore.sh create mode 100644 aws/s3-policy.json create mode 100644 aws/setup-ollama-proxy.sh create mode 100644 aws/setup.sh create mode 100644 aws/teardown.sh create mode 100644 aws/user-data.sh create mode 100644 docker-compose.aws.yml create mode 100644 grafana-provisioning/alerting/.gitkeep create mode 100644 grafana-provisioning/alerting/rules.yml create mode 100644 grafana-provisioning/dashboard-json/.gitkeep create mode 100644 grafana-provisioning/dashboards.yml create mode 100644 grafana-provisioning/datasources.yml create mode 100644 prometheus-aws.yml create mode 100644 start.ps1 diff --git a/.dockerignore b/.dockerignore index 9f56aa826..b7d6bb62e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,33 @@ -.vscode -node_modules -bots/*/ -!bots/* -keys.json \ No newline at end of file +# Runtime bot data — large and should never be in the image +bots/*/logs/ +bots/*/histories/ +bots/*/action-code/ +bots/*/ensemble_log.json + +# Git history +.git/ + +# Local secrets / keys +keys.json +.env +.env.* + +# Node dev artifacts +node_modules/ + +# Editor / OS +.vscode/ +*.DS_Store +Thumbs.db + +# Tasks output +tasks/**/__pycache__/ +tasks/**/*.pyc + +# AWS deploy scripts (not needed in container) +aws/ + +# Docs not needed at runtime +docs/ +*.md +!README.md diff --git a/Dockerfile b/Dockerfile index 153624512..c01c70368 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,9 +27,19 @@ RUN apt-get update && \ WORKDIR /app -COPY package*.json . +# Copy package files and patches for better caching +COPY package*.json ./ +COPY patches/ ./patches/ RUN npm install +# Copy source code COPY . . +# Run tests during build +RUN npm test + +# Drop root privileges — node:slim includes a non-root 'node' user +RUN chown -R node:node /app +USER node + CMD ["npm", "start"] \ No newline at end of file diff --git a/Tasks.Dockerfile b/Tasks.Dockerfile index 509d3abda..e43ddb894 100644 --- a/Tasks.Dockerfile +++ b/Tasks.Dockerfile @@ -1,51 +1,49 @@ -# Specify a base image -# FROM ubuntu:22.04 -FROM node:18 - -#Install some dependencies - -RUN apt-get -y update -RUN apt-get -y install git -RUN apt-get -y install unzip -RUN apt-get -y install python3 -RUN apt-get -y install python3-pip -RUN apt-get -y install python3-boto3 -RUN apt-get -y install python3-tqdm -RUN apt-get -y install tmux - -RUN git clone https://github.com/mindcraft-bots/mindcraft.git /mindcraft +# Tasks.Dockerfile — Evaluation / benchmark runner +# Builds a container with Mindcraft + Java 21 + AWS CLI for automated tasks. + +FROM node:22-slim AS base + +# ── System dependencies (single layer) ────────────────────────────────────── +RUN apt-get update && apt-get install -y --no-install-recommends \ + git unzip curl wget ca-certificates gnupg lsb-release \ + python3 python3-pip python3-boto3 python3-tqdm tmux \ + apt-transport-https \ + && rm -rf /var/lib/apt/lists/* + +# ── Adoptium Java 21 (proper GPG keyring, not deprecated apt-key) ─────────── +RUN mkdir -p /etc/apt/keyrings \ + && wget -qO- https://packages.adoptium.net/artifactory/api/gpg/key/public \ + | gpg --dearmor -o /etc/apt/keyrings/adoptium.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/adoptium.gpg] \ + https://packages.adoptium.net/artifactory/deb $(lsb_release -cs) main" \ + > /etc/apt/sources.list.d/adoptium.list \ + && apt-get update && apt-get install -y --no-install-recommends temurin-21-jdk \ + && rm -rf /var/lib/apt/lists/* + +# ── AWS CLI v2 ────────────────────────────────────────────────────────────── +RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip \ + && unzip -q /tmp/awscliv2.zip -d /tmp \ + && /tmp/aws/install \ + && rm -rf /tmp/awscliv2.zip /tmp/aws + +# ── Application code ─────────────────────────────────────────────────────── WORKDIR /mindcraft -COPY ./server_data.zip /mindcraft -RUN unzip server_data.zip +# Copy source from the build context (this repo) rather than cloning from +# GitHub at build time. A live git clone: +# 1. Breaks reproducibility (upstream HEAD can change between builds) +# 2. Fails in offline/air-gapped CI environments +# 3. Introduces supply-chain risk (external fetch during image build) +COPY . . -RUN npm install +COPY ./server_data.zip /mindcraft/ +RUN unzip -q server_data.zip && rm server_data.zip +RUN npm ci --omit=dev -# Copy the rest of the application code to the working directory -# RUN apt update -# RUN apt install bash ca-certificates wget git -y # install first to avoid openjdk install bug -# RUN apt install openjdk-17-jre-headless -y -RUN apt install -y wget apt-transport-https gnupg lsb-release - -# Add Adoptium repository key -RUN wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - - -# Add Adoptium repository -RUN echo "deb https://packages.adoptium.net/artifactory/deb $(lsb_release -cs) main" > /etc/apt/sources.list.d/adoptium.list - -# Update package lists -RUN apt update - -# Install Temurin (Adoptium) Java 21 -RUN apt install temurin-21-jdk -y - -# Install unzip - - -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -RUN unzip awscliv2.zip -RUN ./aws/install +# ── Non-root user ────────────────────────────────────────────────────────── +RUN groupadd -r mindcraft && useradd -r -g mindcraft -d /mindcraft mindcraft \ + && chown -R mindcraft:mindcraft /mindcraft +USER mindcraft VOLUME /data - EXPOSE 8000 diff --git a/aws/backup.sh b/aws/backup.sh new file mode 100644 index 000000000..2229f88e9 --- /dev/null +++ b/aws/backup.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# ============================================================================= +# aws/backup.sh — Backup Minecraft world and bot memory to S3 +# ============================================================================= +# Run manually: bash aws/backup.sh +# Also runs automatically every 6 hours via cron (installed by deploy.sh) +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CONFIG_FILE="${SCRIPT_DIR}/config.env" + +GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m' +info() { echo -e "${GREEN}[BACKUP $(date '+%Y-%m-%d %H:%M:%S')]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } + +# ── Detect if running locally (SSH to EC2) or on EC2 directly ──────────────── +if [[ -f "$CONFIG_FILE" ]]; then + # Running locally — SSH to EC2 and run backup there + # shellcheck source=/dev/null + source "$CONFIG_FILE" + [[ -n "${EC2_IP:-}" ]] || { echo "EC2_IP not set"; exit 1; } + SSH_OPTS="-i ${KEY_FILE} -o StrictHostKeyChecking=no" + info "Running backup on EC2 (${EC2_IP}) via SSH..." + ssh ${SSH_OPTS} ubuntu@${EC2_IP} "bash /app/aws/backup.sh" + exit 0 +fi + +# ── Running ON EC2 ──────────────────────────────────────────────────────────── +# Get region and bucket from instance metadata + SSM +REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/region 2>/dev/null \ + || echo "${AWS_DEFAULT_REGION:-us-east-1}") +S3_BUCKET=$(aws ssm get-parameter \ + --region "$REGION" \ + --name "/mindcraft/S3_BUCKET" \ + --with-decryption \ + --query 'Parameter.Value' \ + --output text 2>/dev/null \ + || grep S3_BUCKET /app/.env | cut -d= -f2 || "") + +[[ -n "$S3_BUCKET" ]] || { echo "ERROR: S3_BUCKET not found"; exit 1; } + +APP_DIR="/app" +TIMESTAMP=$(date +%Y%m%d-%H%M%S) + +# ── Stop Minecraft briefly (prevents corrupted world files) ─────────────────── +MINECRAFT_WAS_RUNNING=false +if docker compose -f "${APP_DIR}/docker-compose.aws.yml" ps minecraft 2>/dev/null | grep -q "Up"; then + MINECRAFT_WAS_RUNNING=true + info "Stopping Minecraft for consistent backup..." + docker compose -f "${APP_DIR}/docker-compose.aws.yml" stop minecraft + sleep 2 +fi + +# ── Backup world to S3 ──────────────────────────────────────────────────────── +info "Syncing minecraft-data → s3://${S3_BUCKET}/minecraft-data/ ..." +aws s3 sync \ + "${APP_DIR}/minecraft-data" \ + "s3://${S3_BUCKET}/minecraft-data/" \ + --sse AES256 \ + --region "$REGION" \ + --delete + +# ── Backup bot memory to S3 ─────────────────────────────────────────────────── +info "Syncing bots/ memory → s3://${S3_BUCKET}/bots/ ..." +# Only sync memory.json and learnings.json (skip histories/ which are huge) +find "${APP_DIR}/bots" -maxdepth 2 \( -name "memory.json" -o -name "learnings.json" \) \ + -exec aws s3 cp {} "s3://${S3_BUCKET}/bots/$(basename "$(dirname {})")/$(basename {})" \ + --sse AES256 --region "$REGION" \; + +# ── Restart Minecraft ───────────────────────────────────────────────────────── +if [[ "$MINECRAFT_WAS_RUNNING" == "true" ]]; then + info "Restarting Minecraft..." + docker compose -f "${APP_DIR}/docker-compose.aws.yml" start minecraft +fi + +info "Backup complete. s3://${S3_BUCKET}/ (timestamp: ${TIMESTAMP})" diff --git a/aws/deploy.sh b/aws/deploy.sh new file mode 100644 index 000000000..a03cbb213 --- /dev/null +++ b/aws/deploy.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# ============================================================================= +# aws/deploy.sh — Deploy / Redeploy Mindcraft to EC2 +# ============================================================================= +# Run from WSL: bash aws/deploy.sh +# On first run: copies all app files and starts containers +# On subsequent runs: syncs changes and restarts +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_FILE="${SCRIPT_DIR}/config.env" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m' +info() { echo -e "${GREEN}[INFO]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; } + +# ── Load config ─────────────────────────────────────────────────────────────── +[[ -f "$CONFIG_FILE" ]] || error "config.env not found. Run aws/setup.sh first." +# shellcheck source=/dev/null +source "$CONFIG_FILE" + +[[ -n "${EC2_IP:-}" ]] || error "EC2_IP not set in config.env" +[[ -n "${KEY_FILE:-}" ]] || error "KEY_FILE not set in config.env" +[[ -f "$KEY_FILE" ]] || error "SSH key not found: ${KEY_FILE}. Run aws/setup.sh first." + +SSH_OPTS="-i ${KEY_FILE} -o StrictHostKeyChecking=no -o ConnectTimeout=10" +SSH="ssh ${SSH_OPTS} ubuntu@${EC2_IP}" +SCP="scp ${SSH_OPTS}" + +info "Deploying to ${EC2_IP}..." + +# ── Wait for EC2 to be SSH-ready ────────────────────────────────────────────── +info "Checking SSH connectivity..." +RETRIES=20 +for i in $(seq 1 $RETRIES); do + if $SSH "echo ok" >/dev/null 2>&1; then + break + fi + if [[ $i -eq $RETRIES ]]; then + error "Cannot SSH to ${EC2_IP} after ${RETRIES} attempts. Is the instance running?" + fi + warn "SSH not ready yet (attempt ${i}/${RETRIES}), waiting 15s..." + sleep 15 +done +info "SSH connected." + +# ── Wait for bootstrap to finish ────────────────────────────────────────────── +info "Checking if EC2 bootstrap is complete..." +RETRIES=30 +for i in $(seq 1 $RETRIES); do + if $SSH "test -f /var/lib/cloud/instance/mindcraft-bootstrap-done" 2>/dev/null; then + break + fi + if [[ $i -eq $RETRIES ]]; then + warn "Bootstrap may not be done yet — proceeding anyway." + break + fi + warn "Bootstrap still running (attempt ${i}/${RETRIES}), waiting 15s..." + sleep 15 +done + +# ── Rsync app files ─────────────────────────────────────────────────────────── +info "Syncing application files..." +rsync -avz --delete --ignore-errors \ + -e "ssh ${SSH_OPTS}" \ + --exclude 'node_modules/' \ + --exclude 'minecraft-data/' \ + --exclude 'bots/*/histories/' \ + --exclude 'bots/*/action-code/' \ + --exclude '.git/' \ + --exclude 'aws/mindcraft-ec2.pem' \ + --exclude 'aws/config.env' \ + --exclude 'keys.json' \ + --exclude '.env' \ + --exclude '*.pem' \ + --exclude '*.key' \ + --exclude 'services/viaproxy/logs/' \ + --exclude 'services/viaproxy/jars/' \ + --exclude 'services/viaproxy/plugins/' \ + --exclude 'services/viaproxy/ViaLoader/' \ + --exclude 'services/viaproxy/saves.json' \ + --exclude 'services/viaproxy/viaproxy.yml' \ + --filter 'protect minecraft-data/' \ + --filter 'protect bots/' \ + "${PROJECT_ROOT}/" \ + "ubuntu@${EC2_IP}:/app/" + +# ── Generate keys.json from SSM ─────────────────────────────────────────────── +info "Pulling secrets from SSM → /app/keys.json on EC2..." +$SSH bash -s <<'REMOTE' +set -euo pipefail + +get_param() { + local name="$1" + aws ssm get-parameter \ + --region "$(curl -s http://169.254.169.254/latest/meta-data/placement/region)" \ + --name "/mindcraft/${name}" \ + --with-decryption \ + --query 'Parameter.Value' \ + --output text 2>/dev/null || echo "" +} + +REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/region) +GEMINI_API_KEY=$(get_param GEMINI_API_KEY) +XAI_API_KEY=$(get_param XAI_API_KEY) +ANTHROPIC_API_KEY=$(get_param ANTHROPIC_API_KEY) +DISCORD_BOT_TOKEN=$(get_param DISCORD_BOT_TOKEN) + +cat > /app/keys.json </dev/null || echo "" +} + +cat > /app/.env </dev/null || \ + sudo crontab -u ubuntu /app/aws-cron.tab + echo "Backup cron installed." +fi +REMOTE + +# ── Done ────────────────────────────────────────────────────────────────────── +echo "" +echo -e "${GREEN}============================================================${NC}" +echo -e "${GREEN} Deploy complete!${NC}" +echo -e "${GREEN}============================================================${NC}" +echo "" +echo " Minecraft: ${EC2_IP}:${MINECRAFT_PORT:-42069}" +echo " Grafana: http://${EC2_IP}:3004 (admin / admin — change on first login)" +echo " MindServer: http://${EC2_IP}:8080" +echo "" +echo " SSH: ssh -i ${KEY_FILE} ubuntu@${EC2_IP}" +echo " Logs: ssh ... 'docker compose -f /app/docker-compose.aws.yml logs -f'" +echo "" diff --git a/aws/ec2-deploy.sh b/aws/ec2-deploy.sh new file mode 100644 index 000000000..5010c76b8 --- /dev/null +++ b/aws/ec2-deploy.sh @@ -0,0 +1,206 @@ +#!/usr/bin/env bash +# ============================================================================= +# aws/ec2-deploy.sh — Bootstrap / Update Mindcraft directly on EC2 +# ============================================================================= +# Run this INSIDE the EC2 instance (browser SSH / EC2 Instance Connect). +# Handles first-time clone OR subsequent git pull, then starts containers. +# +# Usage: +# GITHUB_TOKEN=ghp_xxxx bash /tmp/ec2-deploy.sh +# # or, if already at /app: +# GITHUB_TOKEN=ghp_xxxx bash /app/aws/ec2-deploy.sh +# +# The GITHUB_TOKEN needs repo read access (classic PAT or fine-grained). +# ============================================================================= +set -euo pipefail + +REPO_HTTPS="https://github.com/Z0mb13V1/mindcraft-0.1.3.git" +APP_DIR="/app" +COMPOSE_FILE="docker-compose.aws.yml" +REGION="${AWS_DEFAULT_REGION:-us-east-1}" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m' +info() { echo -e "${GREEN}[INFO]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; } +step() { echo -e "\n${CYAN}=== $* ===${NC}"; } + +# ── Require token ────────────────────────────────────────────────────────────── +if [[ -z "${GITHUB_TOKEN:-}" ]]; then + echo "" + echo " No GITHUB_TOKEN set. The repo is private — a token is required." + echo " Create one at: https://github.com/settings/tokens" + echo " (needs 'repo' read scope)" + echo "" + read -rsp " Paste your GitHub Personal Access Token: " GITHUB_TOKEN + echo "" + [[ -n "$GITHUB_TOKEN" ]] || error "Token is required." +fi + +CLONE_URL="https://${GITHUB_TOKEN}@github.com/Z0mb13V1/mindcraft-0.1.3.git" + +# ── Detect region from IMDS ──────────────────────────────────────────────────── +IMDS_TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \ + -H "X-aws-ec2-metadata-token-ttl-seconds: 60" 2>/dev/null || echo "") +if [[ -n "$IMDS_TOKEN" ]]; then + REGION=$(curl -s -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" \ + "http://169.254.169.254/latest/meta-data/placement/region" 2>/dev/null \ + || echo "$REGION") +fi +info "Region: ${REGION}" + +# ── SSM helper ──────────────────────────────────────────────────────────────── +get_param() { + aws ssm get-parameter \ + --region "$REGION" \ + --name "/mindcraft/$1" \ + --with-decryption \ + --query 'Parameter.Value' \ + --output text 2>/dev/null || echo "" +} + +# ── Step 1: Clone or update ──────────────────────────────────────────────────── +step "1. Sync code" +if [[ -d "${APP_DIR}/.git" ]]; then + info "Repo already at ${APP_DIR} — pulling latest..." + cd "$APP_DIR" + git remote set-url origin "$CLONE_URL" + git fetch origin + git reset --hard origin/main + git clean -fd --exclude=chromadb-data/ 2>/dev/null || true + info "Updated to: $(git log --oneline -1)" +elif [[ -d "${APP_DIR}" ]]; then + # Dir exists but no .git — init in-place and pull + info "${APP_DIR} exists but has no git repo — initialising in-place..." + cd "$APP_DIR" + git init -b main + git remote add origin "$CLONE_URL" + git fetch origin main + git reset --hard origin/main + info "Initialised: $(git log --oneline -1)" +else + info "Cloning repo to ${APP_DIR}..." + git clone "$CLONE_URL" "$APP_DIR" + cd "$APP_DIR" + info "Cloned: $(git log --oneline -1)" +fi + +# Secure the git remote URL so the token isn't visible in git log output +git remote set-url origin "$REPO_HTTPS" + +# ── Step 2: Pull secrets from SSM → keys.json ───────────────────────────────── +step "2. Pull secrets from SSM" + +GEMINI_API_KEY=$(get_param GEMINI_API_KEY) +XAI_API_KEY=$(get_param XAI_API_KEY) +ANTHROPIC_API_KEY=$(get_param ANTHROPIC_API_KEY) +DISCORD_BOT_TOKEN=$(get_param DISCORD_BOT_TOKEN) + +if [[ -z "$GEMINI_API_KEY" && -z "$XAI_API_KEY" ]]; then + warn "SSM returned empty keys — either IAM role lacks access or params not set." + warn "Continuing anyway; containers may fail if keys are missing." +fi + +cat > "${APP_DIR}/keys.json" < "${APP_DIR}/.env" </dev/null || true +docker compose -f "$COMPOSE_FILE" up -d --build + +# ── Step 6: Install backup cron ─────────────────────────────────────────────── +step "6. Install backup cron" +if [[ -f "${APP_DIR}/aws-cron.tab" ]]; then + crontab "${APP_DIR}/aws-cron.tab" 2>/dev/null \ + || sudo crontab -u ubuntu "${APP_DIR}/aws-cron.tab" 2>/dev/null \ + || warn "Could not install cron (not critical)." + info "Backup cron installed." +fi + +# ── Done ────────────────────────────────────────────────────────────────────── +echo "" +echo -e "${GREEN}============================================================${NC}" +echo -e "${GREEN} Deploy complete!${NC}" +echo -e "${GREEN}============================================================${NC}" +echo "" +docker compose -f "$COMPOSE_FILE" ps +echo "" +EC2_IP=$(curl -s -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" \ + "http://169.254.169.254/latest/meta-data/public-ipv4" 2>/dev/null \ + || echo "") +echo " Minecraft: ${EC2_IP}:${MINECRAFT_PORT:-42069}" +echo " Grafana: http://${EC2_IP}:3004" +echo " MindServer: http://${EC2_IP}:8080" +echo "" +echo " Logs: docker compose -f /app/${COMPOSE_FILE} logs -f" +echo "" diff --git a/aws/ec2-go.sh b/aws/ec2-go.sh new file mode 100644 index 000000000..985d7bc62 --- /dev/null +++ b/aws/ec2-go.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# ============================================================================= +# aws/ec2-go.sh — One-command Mindcraft deploy +# ============================================================================= +# Auto-detects whether you're ON EC2 or remote (Mac/Linux). +# On EC2: runs everything locally (no SSH needed) +# Remote: SSHs into EC2 to run commands +# +# Usage: +# bash aws/ec2-go.sh # Pull latest code + restart containers +# bash aws/ec2-go.sh --build # Pull + rebuild Docker images +# bash aws/ec2-go.sh --secrets # Re-pull SSM secrets + restart +# bash aws/ec2-go.sh --full # Full: secrets + build + restart +# ============================================================================= +set -euo pipefail + +GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; CYAN='\033[0;36m'; NC='\033[0m' +info() { echo -e "${GREEN}[INFO]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; } +step() { echo -e "\n${CYAN}=== $* ===${NC}"; } + +# ── Parse args ──────────────────────────────────────────────────────────────── +COMPOSE_FILE="docker-compose.aws.yml" +APP_DIR="/app" +DO_BUILD=false +DO_SECRETS=false + +for arg in "$@"; do + case "$arg" in + --build) DO_BUILD=true ;; + --secrets) DO_SECRETS=true ;; + --full) DO_BUILD=true; DO_SECRETS=true ;; + --help|-h) + echo "Usage: ec2-go.sh [--build] [--secrets] [--full]" + echo " --build Rebuild Docker images" + echo " --secrets Re-pull secrets from SSM to .env and keys.json" + echo " --full Both --build and --secrets" + exit 0 ;; + *) warn "Unknown arg: $arg" ;; + esac +done + +# ── Detect: are we ON EC2 or remote? ───────────────────────────────────────── +# Check 3 ways: IMDSv2 (token-based), IMDSv1, or hostname pattern ip-* +IS_EC2=false +IMDS_TOKEN=$(curl -sf -m 2 -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60" 2>/dev/null || true) +if [[ -n "$IMDS_TOKEN" ]]; then + # IMDSv2 works + IS_EC2=true +elif curl -sf -m 2 http://169.254.169.254/latest/meta-data/instance-id >/dev/null 2>&1; then + # IMDSv1 fallback + IS_EC2=true +elif hostname | grep -q '^ip-'; then + # EC2 default hostname pattern (ip-10-0-1-107 etc.) + IS_EC2=true +fi + +if $IS_EC2; then + info "Detected: running ON EC2 — executing locally" + # run_cmd just runs the command directly + run_cmd() { bash -c "$1"; } +else + info "Detected: running remotely — will SSH into EC2" + EC2_IP="${EC2_PUBLIC_IP:?Set EC2_PUBLIC_IP in .env or environment}" + EC2_KEY="${EC2_KEY_FILE:-$HOME/.ssh/mindcraft-ec2.pem}" + EC2_USER="ubuntu" + + if [[ ! -f "$EC2_KEY" ]]; then + error "SSH key not found: ${EC2_KEY} + Set EC2_KEY_FILE or copy your .pem to ~/.ssh/mindcraft-ec2.pem + Or run this script directly on EC2 (it auto-detects)." + fi + + SSH_OPTS="-i ${EC2_KEY} -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10" + SSH_CMD="ssh ${SSH_OPTS} ${EC2_USER}@${EC2_IP}" + + # Test SSH + if ! $SSH_CMD "echo ok" >/dev/null 2>&1; then + error "Cannot SSH to ${EC2_IP}. Is the instance running?" + fi + info "SSH connected to ${EC2_IP}" + + # run_cmd sends the command over SSH + run_cmd() { $SSH_CMD bash -c "$1"; } +fi + +# ── Step 1: Git pull ───────────────────────────────────────────────────────── +step "1/4 Git Pull" +run_cmd ' +cd /app +if [ -d .git ]; then + git fetch origin main 2>&1 || echo "[WARN] git fetch failed — using local files" + git reset --hard origin/main 2>&1 || echo "[WARN] git reset failed" + echo "[OK] Code updated from origin/main" +else + echo "[WARN] /app is not a git repo — skipping pull" +fi +' + +# ── Step 2: Re-pull secrets from SSM (optional) ────────────────────────────── +if $DO_SECRETS; then + step "2/4 Pull Secrets from SSM" + run_cmd ' +cd /app +TOKEN=$(curl -sf -m 2 -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60" 2>/dev/null || true) +if [ -n "$TOKEN" ]; then + REGION=$(curl -sf -m 5 -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region) +else + REGION=$(curl -sf -m 5 http://169.254.169.254/latest/meta-data/placement/region) +fi +if [ -z "$REGION" ]; then REGION="us-east-1"; echo "[WARN] Metadata unavailable, defaulting to us-east-1"; fi + +get_param() { + aws ssm get-parameter \ + --region "$REGION" \ + --name "/mindcraft/$1" \ + --with-decryption \ + --query "Parameter.Value" \ + --output text 2>/dev/null || echo "" +} + +echo "Pulling secrets from SSM /mindcraft/*..." +GEMINI_API_KEY=$(get_param GEMINI_API_KEY) +XAI_API_KEY=$(get_param XAI_API_KEY) +ANTHROPIC_API_KEY=$(get_param ANTHROPIC_API_KEY) +DISCORD_BOT_TOKEN=$(get_param DISCORD_BOT_TOKEN) +BOT_DM_CHANNEL=$(get_param BOT_DM_CHANNEL) +BACKUP_CHAT_CHANNEL=$(get_param BACKUP_CHAT_CHANNEL) +DISCORD_ADMIN_IDS=$(get_param DISCORD_ADMIN_IDS) +TAILSCALE_AUTHKEY=$(get_param TAILSCALE_AUTHKEY) +LITELLM_MASTER_KEY=$(get_param LITELLM_MASTER_KEY) +EC2_PUBLIC_IP=$(get_param EC2_PUBLIC_IP) +GITHUB_TOKEN=$(get_param GITHUB_TOKEN) + +# Strip embedded newlines from SSM values — a multiline value would break +# the .env file format and could inject extra key=value pairs. +strip_nl() { printf '%s' "$1" | tr -d '\n\r'; } +GEMINI_API_KEY=$(strip_nl "$GEMINI_API_KEY") +XAI_API_KEY=$(strip_nl "$XAI_API_KEY") +ANTHROPIC_API_KEY=$(strip_nl "$ANTHROPIC_API_KEY") +DISCORD_BOT_TOKEN=$(strip_nl "$DISCORD_BOT_TOKEN") +BOT_DM_CHANNEL=$(strip_nl "$BOT_DM_CHANNEL") +BACKUP_CHAT_CHANNEL=$(strip_nl "$BACKUP_CHAT_CHANNEL") +DISCORD_ADMIN_IDS=$(strip_nl "$DISCORD_ADMIN_IDS") +TAILSCALE_AUTHKEY=$(strip_nl "$TAILSCALE_AUTHKEY") +LITELLM_MASTER_KEY=$(strip_nl "$LITELLM_MASTER_KEY") +EC2_PUBLIC_IP=$(strip_nl "$EC2_PUBLIC_IP") +GITHUB_TOKEN=$(strip_nl "$GITHUB_TOKEN") + +cat > /app/keys.json < /app/.env <&1 && echo '' && docker compose -f ${COMPOSE_FILE} ps" + +# ── Step 4: Verify bots ────────────────────────────────────────────────────── +step "4/4 Bot Verification" +info "Waiting 15s for bots to connect..." +sleep 15 + +run_cmd ' +cd /app +LOGS=$(docker compose -f docker-compose.aws.yml logs --tail 30 mindcraft 2>&1) +for bot in "CloudGrok" "LocalAndy"; do + if echo "$LOGS" | grep -q "$bot"; then + echo "[OK] $bot appears in logs" + else + echo "[WARN] $bot not found in recent logs (may still be starting)" + fi +done +echo "" +echo "=== Recent logs ===" +echo "$LOGS" | tail -15 +' + +# ── Done ────────────────────────────────────────────────────────────────────── +echo "" +echo -e "${GREEN}========================================${NC}" +echo -e "${GREEN} Deploy complete!${NC}" +echo -e "${GREEN}========================================${NC}" +echo "" +if $IS_EC2; then + TOKEN=$(curl -sf -m 2 -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 60" 2>/dev/null || true) + if [[ -n "$TOKEN" ]]; then + EC2_IP=$(curl -sf -m 2 -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/public-ipv4 2>/dev/null || echo "this-host") + else + EC2_IP=$(curl -sf -m 2 http://169.254.169.254/latest/meta-data/public-ipv4 2>/dev/null || echo "this-host") + fi +fi +echo " Minecraft: ${EC2_IP:-localhost}:${MINECRAFT_PORT:-42069}" +echo " MindServer: http://${EC2_IP:-localhost}:8080" +echo " Grafana: http://${EC2_IP:-localhost}:3004" +echo "" +echo " Logs: docker compose -f /app/docker-compose.aws.yml logs -f mindcraft" +echo "" diff --git a/aws/env-toggle.sh b/aws/env-toggle.sh new file mode 100644 index 000000000..59c4124b5 --- /dev/null +++ b/aws/env-toggle.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# ============================================================================= +# aws/env-toggle.sh — Ensure only one environment runs Mindcraft at a time +# ============================================================================= +# Usage: +# bash aws/env-toggle.sh --aws # Start AWS, stop local +# bash aws/env-toggle.sh --local # Stop AWS workloads, start local +# bash aws/env-toggle.sh --auto # Check EC2 state and toggle accordingly +# bash aws/env-toggle.sh --status # Just show what's running +# +# NOTE: Docker is not accessible from WSL on this system. +# Run local docker commands from Windows CMD/PowerShell. +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CONFIG_FILE="${SCRIPT_DIR}/config.env" + +GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'; NC='\033[0m' +info() { echo -e "${GREEN}[env-toggle]${NC} $*"; } +warn() { echo -e "${YELLOW}[env-toggle]${NC} $*"; } +error() { echo -e "${RED}[env-toggle]${NC} $*"; exit 1; } + +# ── Load config ─────────────────────────────────────────────────────────────── +[[ -f "$CONFIG_FILE" ]] || error "config.env not found. Run aws/setup.sh first." +# shellcheck source=/dev/null +source "$CONFIG_FILE" + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +check_ec2_state() { + aws ec2 describe-instances \ + --region "$REGION" \ + --instance-ids "$INSTANCE_ID" \ + --query 'Reservations[0].Instances[0].State.Name' \ + --output text 2>/dev/null || echo "unknown" +} + +start_aws() { + local state + state=$(check_ec2_state) + if [[ "$state" == "running" ]]; then + info "EC2 already running (${INSTANCE_ID})" + return + fi + info "Starting EC2 instance ${INSTANCE_ID}..." + aws ec2 start-instances --region "$REGION" --instance-ids "$INSTANCE_ID" >/dev/null + aws ec2 wait instance-running --region "$REGION" --instance-ids "$INSTANCE_ID" + + # Get new public IP (changes on stop/start unless Elastic IP) + EC2_IP=$(aws ec2 describe-instances \ + --region "$REGION" \ + --instance-ids "$INSTANCE_ID" \ + --query 'Reservations[0].Instances[0].PublicIpAddress' \ + --output text) + sed -i "s/^EC2_IP=.*/EC2_IP=${EC2_IP}/" "$CONFIG_FILE" + info "EC2 running at ${EC2_IP}" +} + +stop_aws() { + local state + state=$(check_ec2_state) + if [[ "$state" == "stopped" ]]; then + info "EC2 already stopped." + return + fi + info "Stopping Mindcraft containers on EC2..." + SSH_OPTS="-i ${KEY_FILE} -o StrictHostKeyChecking=no -o ConnectTimeout=5" + ssh ${SSH_OPTS} ubuntu@${EC2_IP} \ + "cd /app && docker compose -f docker-compose.aws.yml stop" 2>/dev/null || true + + info "Stopping EC2 instance ${INSTANCE_ID}..." + aws ec2 stop-instances --region "$REGION" --instance-ids "$INSTANCE_ID" >/dev/null + info "EC2 stopping (saves money while local is active)" +} + +stop_local() { + warn "To stop local Mindcraft containers, run from Windows CMD:" + echo "" + echo " docker compose -f docker-compose.yml stop mindcraft discord-bot" + echo "" + warn "Docker is not accessible from WSL. Run the above in CMD or PowerShell." +} + +start_local() { + warn "To start local Mindcraft containers, run from Windows CMD:" + echo "" + echo " docker compose -f docker-compose.yml up -d" + echo "" + warn "Docker is not accessible from WSL. Run the above in CMD or PowerShell." +} + +show_status() { + local state + state=$(check_ec2_state) + echo "" + info "AWS EC2: ${state} (${INSTANCE_ID})" + if [[ "$state" == "running" ]]; then + echo " IP: ${EC2_IP}" + echo " Minecraft: ${EC2_IP}:${MINECRAFT_PORT:-42069}" + echo " Grafana: http://${EC2_IP}:3004" + echo " MindServer: http://${EC2_IP}:8080" + fi + echo "" + info "Local Docker: check from Windows CMD with 'docker compose ps'" + echo "" +} + +# ── Main ────────────────────────────────────────────────────────────────────── +case "${1:-}" in + --aws) + info "Switching to AWS environment..." + start_aws + stop_local + info "AWS is active. Local containers should be stopped." + ;; + --local) + info "Switching to local environment..." + stop_aws + start_local + info "AWS stopped. Start local containers from Windows CMD." + ;; + --auto) + state=$(check_ec2_state) + if [[ "$state" == "running" ]]; then + info "EC2 is running → ensuring local is stopped" + stop_local + else + info "EC2 is ${state} → local environment should be active" + start_local + fi + ;; + --status) + show_status + ;; + *) + echo "Usage: bash aws/env-toggle.sh [--aws | --local | --auto | --status]" + echo "" + echo " --aws Start AWS EC2, remind to stop local" + echo " --local Stop AWS EC2, remind to start local" + echo " --auto Check EC2 state and advise accordingly" + echo " --status Show what's running where" + exit 1 + ;; +esac diff --git a/aws/restore.sh b/aws/restore.sh new file mode 100644 index 000000000..ca8772989 --- /dev/null +++ b/aws/restore.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# ============================================================================= +# aws/restore.sh — Restore Minecraft world and bot memory from S3 +# ============================================================================= +# Run from local: bash aws/restore.sh +# WARNING: This overwrites local data on EC2. Use with caution. +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CONFIG_FILE="${SCRIPT_DIR}/config.env" + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m' +info() { echo -e "${GREEN}[RESTORE]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; } + +# ── Detect if running locally (SSH to EC2) or on EC2 directly ──────────────── +if [[ -f "$CONFIG_FILE" ]]; then + # Running locally — SSH to EC2 and run restore there + # shellcheck source=/dev/null + source "$CONFIG_FILE" + [[ -n "${EC2_IP:-}" ]] || error "EC2_IP not set in config.env" + + echo "" + warn "WARNING: This will OVERWRITE the current world and bot memory on EC2 with S3 data." + read -r -p "Are you sure? Type 'yes' to confirm: " CONFIRM + [[ "$CONFIRM" == "yes" ]] || { echo "Aborted."; exit 0; } + + SSH_OPTS="-i ${KEY_FILE} -o StrictHostKeyChecking=no" + info "Running restore on EC2 (${EC2_IP}) via SSH..." + ssh ${SSH_OPTS} ubuntu@${EC2_IP} "bash /app/aws/restore.sh --confirmed" + exit 0 +fi + +# ── Running ON EC2 ──────────────────────────────────────────────────────────── +[[ "${1:-}" == "--confirmed" ]] || { echo "Run this from your local machine via: bash aws/restore.sh"; exit 1; } + +REGION=$(curl -s http://169.254.169.254/latest/meta-data/placement/region 2>/dev/null \ + || echo "${AWS_DEFAULT_REGION:-us-east-1}") +S3_BUCKET=$(aws ssm get-parameter \ + --region "$REGION" \ + --name "/mindcraft/S3_BUCKET" \ + --with-decryption \ + --query 'Parameter.Value' \ + --output text 2>/dev/null \ + || grep S3_BUCKET /app/.env | cut -d= -f2 || "") + +[[ -n "$S3_BUCKET" ]] || error "S3_BUCKET not found" + +APP_DIR="/app" + +# ── Stop all services ───────────────────────────────────────────────────────── +info "Stopping all containers..." +docker compose -f "${APP_DIR}/docker-compose.aws.yml" stop minecraft mindcraft + +# ── Restore world from S3 ───────────────────────────────────────────────────── +info "Restoring minecraft-data from s3://${S3_BUCKET}/minecraft-data/ ..." +mkdir -p "${APP_DIR}/minecraft-data" +aws s3 sync \ + "s3://${S3_BUCKET}/minecraft-data/" \ + "${APP_DIR}/minecraft-data" \ + --sse AES256 \ + --region "$REGION" \ + --delete + +# ── Restore bot memory from S3 ──────────────────────────────────────────────── +info "Restoring bot memory from s3://${S3_BUCKET}/bots/ ..." +aws s3 sync \ + "s3://${S3_BUCKET}/bots/" \ + "${APP_DIR}/bots/" \ + --sse AES256 \ + --region "$REGION" + +# ── Restart services ────────────────────────────────────────────────────────── +info "Restarting containers..." +docker compose -f "${APP_DIR}/docker-compose.aws.yml" up -d minecraft mindcraft + +info "Restore complete." diff --git a/aws/s3-policy.json b/aws/s3-policy.json new file mode 100644 index 000000000..213de2dca --- /dev/null +++ b/aws/s3-policy.json @@ -0,0 +1,51 @@ +{ + "_comment": "Template — actual values are filled in by aws/setup.sh. Do not use directly.", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "DenyHTTP", + "Effect": "Deny", + "Principal": "*", + "Action": "s3:*", + "Resource": [ + "arn:aws:s3:::BUCKET_NAME", + "arn:aws:s3:::BUCKET_NAME/*" + ], + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + } + }, + { + "Sid": "AllowEC2Role", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::ACCOUNT_ID:role/mindcraft-ec2-role" + }, + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::BUCKET_NAME", + "arn:aws:s3:::BUCKET_NAME/*" + ] + }, + { + "Sid": "AllowAdminIAM", + "Effect": "Allow", + "Principal": { + "AWS": "CALLER_ARN" + }, + "Action": "s3:*", + "Resource": [ + "arn:aws:s3:::BUCKET_NAME", + "arn:aws:s3:::BUCKET_NAME/*" + ] + } + ] +} diff --git a/aws/setup-ollama-proxy.sh b/aws/setup-ollama-proxy.sh new file mode 100644 index 000000000..e702a0771 --- /dev/null +++ b/aws/setup-ollama-proxy.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# ============================================================================= +# aws/setup-ollama-proxy.sh — Set up socat proxy for Ollama via Tailscale +# ============================================================================= +# Runs on EC2 host. Creates a systemd service that proxies localhost:11435 +# to the local Ollama instance via Tailscale (set OLLAMA_TAILSCALE_IP env var). +# +# Why: Docker containers (even with network_mode: host) have issues routing +# data through Tailscale's TUN interface. This proxy runs as a native host +# process, which can use Tailscale routing without issues. +# +# Usage: +# sudo bash /app/aws/setup-ollama-proxy.sh +# ============================================================================= +set -euo pipefail + +GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; NC='\033[0m' +info() { echo -e "${GREEN}[INFO]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; } + +if [ -z "${OLLAMA_TAILSCALE_IP:-}" ]; then + error "OLLAMA_TAILSCALE_IP is required. Set it to your local machine's Tailscale IP (e.g. 100.x.x.x)" +fi +OLLAMA_REMOTE="${OLLAMA_TAILSCALE_IP}:11434" +PROXY_PORT="11435" +SERVICE_NAME="ollama-proxy" + +# ── Install socat if needed ────────────────────────────────────────────────── +if ! command -v socat &>/dev/null; then + info "Installing socat..." + apt-get update -qq && apt-get install -y -qq socat +else + info "socat already installed." +fi + +# ── Create systemd service ─────────────────────────────────────────────────── +info "Creating systemd service: ${SERVICE_NAME}" +cat > /etc/systemd/system/${SERVICE_NAME}.service < ${OLLAMA_REMOTE}) +After=network.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/usr/bin/socat TCP4-LISTEN:${PROXY_PORT},bind=127.0.0.1,reuseaddr,fork TCP4:${OLLAMA_REMOTE} +Restart=always +RestartSec=5 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target +EOF + +# ── Kill any stale socat processes on the proxy port ───────────────────────── +if pgrep -f "socat.*${PROXY_PORT}" &>/dev/null; then + info "Killing stale socat processes on port ${PROXY_PORT}..." + pkill -f "socat.*${PROXY_PORT}" || true + sleep 1 +fi + +# ── Enable and start ───────────────────────────────────────────────────────── +systemctl daemon-reload +systemctl enable "${SERVICE_NAME}" +systemctl restart "${SERVICE_NAME}" +sleep 2 + +# ── Verify ─────────────────────────────────────────────────────────────────── +if systemctl is-active --quiet "${SERVICE_NAME}"; then + info "Service ${SERVICE_NAME} is running." +else + warn "Service ${SERVICE_NAME} failed to start. Check: journalctl -u ${SERVICE_NAME}" +fi + +# Quick connectivity test +info "Testing proxy connectivity..." +if curl -s --max-time 10 "http://127.0.0.1:${PROXY_PORT}/api/tags" | grep -q "models"; then + info "Ollama reachable through proxy at localhost:${PROXY_PORT}" +else + warn "Could not reach Ollama through proxy. Is Ollama running on your local machine?" + warn "Is Tailscale connected? Check: tailscale status (in the tailscale container)" +fi + +# ── Restart mindcraft to pick up the new profile URL ───────────────────────── +info "Restarting mindcraft container..." +cd /app +docker compose -f docker-compose.aws.yml up -d --no-deps --force-recreate mindcraft + +echo "" +info "Done! LocalAndy will now connect to Ollama via localhost:${PROXY_PORT} -> Tailscale -> ${OLLAMA_REMOTE}" +echo "" diff --git a/aws/setup.sh b/aws/setup.sh new file mode 100644 index 000000000..e7e0abe95 --- /dev/null +++ b/aws/setup.sh @@ -0,0 +1,472 @@ +#!/usr/bin/env bash +# ============================================================================= +# aws/setup.sh — Mindcraft AWS Infrastructure Setup +# ============================================================================= +# Creates: VPC, Security Group, S3 bucket, IAM role, SSM parameters, EC2 instance +# Run once from WSL: bash aws/setup.sh +# ============================================================================= +set -euo pipefail + +# ── Config ──────────────────────────────────────────────────────────────────── +REGION="${AWS_REGION:-us-east-1}" +INSTANCE_TYPE="t3.large" +AMI_NAME_FILTER="ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*" +KEY_NAME="mindcraft-ec2" +KEY_FILE="$(dirname "$0")/mindcraft-ec2.pem" +MINECRAFT_PORT="${MINECRAFT_PORT:-42069}" +CONFIG_FILE="$(dirname "$0")/config.env" +APP_DIR="/app" +STACK_NAME="mindcraft" +# S3 bucket name must be globally unique; we append account ID +BUCKET_PREFIX="mindcraft-world-backups" + +# ── Colors ──────────────────────────────────────────────────────────────────── +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m' +info() { echo -e "${GREEN}[INFO]${NC} $*"; } +warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } +error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; } + +# ── Prerequisites ───────────────────────────────────────────────────────────── +info "Checking prerequisites..." + +command -v aws >/dev/null 2>&1 || error "AWS CLI not found. Install: https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html + Quick install: + curl 'https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip' -o /tmp/awscliv2.zip + unzip /tmp/awscliv2.zip -d /tmp + sudo /tmp/aws/install" + +command -v jq >/dev/null 2>&1 || error "jq not found. Install: sudo apt-get install -y jq" +command -v ssh >/dev/null 2>&1 || error "ssh not found." + +# Verify credentials +CALLER=$(aws sts get-caller-identity 2>/dev/null) \ + || error "AWS credentials not configured. Run: aws configure + You'll need: + - AWS Access Key ID (from IAM → Security credentials) + - AWS Secret Access Key (same page) + - Default region: ${REGION} + - Output format: json" + +ACCOUNT_ID=$(echo "$CALLER" | jq -r '.Account') +CALLER_ARN=$(echo "$CALLER" | jq -r '.Arn') +BUCKET_NAME="${BUCKET_PREFIX}-${ACCOUNT_ID}" + +info "AWS account: ${ACCOUNT_ID}" +info "Caller ARN: ${CALLER_ARN}" +info "Region: ${REGION}" +info "S3 bucket: ${BUCKET_NAME}" + +# ── Admin IP for SSH/admin access ──────────────────────────────────────────── +DETECTED_IP=$(curl -s https://checkip.amazonaws.com 2>/dev/null || curl -s https://api.ipify.org 2>/dev/null || echo "") + +if [[ -n "$DETECTED_IP" ]]; then + echo "" + read -r -p "Your current IP appears to be ${DETECTED_IP}. Use this to restrict admin ports? [Y/n] " USE_DETECTED + if [[ "${USE_DETECTED,,}" != "n" ]]; then + ADMIN_IP="${DETECTED_IP}" + else + read -r -p "Enter your IP address (for SSH/Grafana/UI access): " ADMIN_IP + fi +else + read -r -p "Enter your IP address (for SSH/Grafana/UI access): " ADMIN_IP +fi + +ADMIN_CIDR="${ADMIN_IP}/32" +info "Admin CIDR: ${ADMIN_CIDR}" + +# ── SSM Secrets collection ──────────────────────────────────────────────────── +echo "" +info "Collecting secrets for SSM Parameter Store (stored encrypted, never in git)..." +echo " Press Enter to skip any key you don't use." + +collect_secret() { + local name="$1" prompt="$2" + local input="" char + printf " %s: " "$prompt" > /dev/tty + while IFS= read -r -s -n1 char < /dev/tty; do + if [[ -z "$char" ]]; then # Enter + break + elif [[ "$char" == $'\177' ]] || [[ "$char" == $'\b' ]]; then # Backspace + if [[ -n "$input" ]]; then + input="${input%?}" + printf '\b \b' > /dev/tty + fi + else + input+="$char" + printf '*' > /dev/tty # Show * per character (not captured by $()) + fi + done + printf '\n' > /dev/tty + printf '%s' "$input" +} + +GEMINI_API_KEY=$(collect_secret "GEMINI_API_KEY" "GEMINI_API_KEY") +XAI_API_KEY=$(collect_secret "XAI_API_KEY" "XAI_API_KEY (also used as OPENAI_API_KEY)") +ANTHROPIC_API_KEY=$(collect_secret "ANTHROPIC_API_KEY" "ANTHROPIC_API_KEY") +DISCORD_BOT_TOKEN=$(collect_secret "DISCORD_BOT_TOKEN" "DISCORD_BOT_TOKEN") +BOT_DM_CHANNEL=$(collect_secret "BOT_DM_CHANNEL" "BOT_DM_CHANNEL (Discord channel ID)") +BACKUP_CHAT_CHANNEL=$(collect_secret "BACKUP_CHAT_CHANNEL" "BACKUP_CHAT_CHANNEL (Discord channel ID)") + +echo "" + +# ============================================================================= +# 1. VPC +# ============================================================================= +info "Creating VPC..." +VPC_ID=$(aws ec2 create-vpc \ + --region "$REGION" \ + --cidr-block 10.0.0.0/16 \ + --tag-specifications "ResourceType=vpc,Tags=[{Key=Name,Value=${STACK_NAME}-vpc}]" \ + --query 'Vpc.VpcId' --output text) +aws ec2 modify-vpc-attribute --region "$REGION" --vpc-id "$VPC_ID" --enable-dns-support +aws ec2 modify-vpc-attribute --region "$REGION" --vpc-id "$VPC_ID" --enable-dns-hostnames +info "VPC: ${VPC_ID}" + +# Subnet +SUBNET_ID=$(aws ec2 create-subnet \ + --region "$REGION" \ + --vpc-id "$VPC_ID" \ + --cidr-block 10.0.1.0/24 \ + --availability-zone "${REGION}a" \ + --tag-specifications "ResourceType=subnet,Tags=[{Key=Name,Value=${STACK_NAME}-subnet}]" \ + --query 'Subnet.SubnetId' --output text) +aws ec2 modify-subnet-attribute --region "$REGION" --subnet-id "$SUBNET_ID" --map-public-ip-on-launch +info "Subnet: ${SUBNET_ID}" + +# Internet Gateway +IGW_ID=$(aws ec2 create-internet-gateway \ + --region "$REGION" \ + --tag-specifications "ResourceType=internet-gateway,Tags=[{Key=Name,Value=${STACK_NAME}-igw}]" \ + --query 'InternetGateway.InternetGatewayId' --output text) +aws ec2 attach-internet-gateway --region "$REGION" --internet-gateway-id "$IGW_ID" --vpc-id "$VPC_ID" +info "IGW: ${IGW_ID}" + +# Route table +RTB_ID=$(aws ec2 create-route-table \ + --region "$REGION" \ + --vpc-id "$VPC_ID" \ + --tag-specifications "ResourceType=route-table,Tags=[{Key=Name,Value=${STACK_NAME}-rtb}]" \ + --query 'RouteTable.RouteTableId' --output text) +aws ec2 create-route --region "$REGION" --route-table-id "$RTB_ID" --destination-cidr-block 0.0.0.0/0 --gateway-id "$IGW_ID" >/dev/null +aws ec2 associate-route-table --region "$REGION" --route-table-id "$RTB_ID" --subnet-id "$SUBNET_ID" >/dev/null +info "Route table: ${RTB_ID}" + +# ============================================================================= +# 2. Security Group +# ============================================================================= +info "Creating security group..." +SG_ID=$(aws ec2 create-security-group \ + --region "$REGION" \ + --group-name "${STACK_NAME}-sg" \ + --description "Mindcraft server security group" \ + --vpc-id "$VPC_ID" \ + --query 'GroupId' --output text) + +# Minecraft — restrict to trusted IPs (set MINECRAFT_CIDR to allow wider access) +MINECRAFT_CIDR="${MINECRAFT_CIDR:-${ADMIN_CIDR}}" +aws ec2 authorize-security-group-ingress --region "$REGION" --group-id "$SG_ID" \ + --ip-permissions "IpProtocol=tcp,FromPort=${MINECRAFT_PORT},ToPort=${MINECRAFT_PORT},IpRanges=[{CidrIp=${MINECRAFT_CIDR},Description='Minecraft'}]" >/dev/null + +# Admin ports — restricted to admin IP only +for PORT_DESC in "22:SSH" "3004:Grafana" "8080:MindServerUI" "9090:Prometheus"; do + PORT="${PORT_DESC%%:*}"; DESC="${PORT_DESC##*:}" + aws ec2 authorize-security-group-ingress --region "$REGION" --group-id "$SG_ID" \ + --ip-permissions "IpProtocol=tcp,FromPort=${PORT},ToPort=${PORT},IpRanges=[{CidrIp=${ADMIN_CIDR},Description='${DESC} - admin only'}]" >/dev/null +done + +# Outbound — HTTPS only (LLM APIs, Docker Hub, GitHub, SSM, S3) +aws ec2 authorize-security-group-egress --region "$REGION" --group-id "$SG_ID" \ + --ip-permissions "IpProtocol=tcp,FromPort=443,ToPort=443,IpRanges=[{CidrIp=0.0.0.0/0,Description='HTTPS outbound (LLM APIs, Docker, S3)'}]" 2>/dev/null || true + +# Outbound — DNS (required for container name resolution) +aws ec2 authorize-security-group-egress --region "$REGION" --group-id "$SG_ID" \ + --ip-permissions "IpProtocol=udp,FromPort=53,ToPort=53,IpRanges=[{CidrIp=0.0.0.0/0,Description='DNS resolution'}]" 2>/dev/null || true +aws ec2 authorize-security-group-egress --region "$REGION" --group-id "$SG_ID" \ + --ip-permissions "IpProtocol=tcp,FromPort=53,ToPort=53,IpRanges=[{CidrIp=0.0.0.0/0,Description='DNS resolution (TCP)'}]" 2>/dev/null || true + +aws ec2 create-tags --region "$REGION" --resources "$SG_ID" \ + --tags "Key=Name,Value=${STACK_NAME}-sg" +info "Security group: ${SG_ID}" + +# ============================================================================= +# 3. S3 Bucket +# ============================================================================= +info "Creating S3 bucket: ${BUCKET_NAME}..." + +# Create bucket (us-east-1 doesn't use --create-bucket-configuration) +if [[ "$REGION" == "us-east-1" ]]; then + aws s3api create-bucket --region "$REGION" --bucket "$BUCKET_NAME" >/dev/null +else + aws s3api create-bucket --region "$REGION" --bucket "$BUCKET_NAME" \ + --create-bucket-configuration "LocationConstraint=${REGION}" >/dev/null +fi + +# Block ALL public access +aws s3api put-public-access-block --bucket "$BUCKET_NAME" \ + --public-access-block-configuration \ + "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" + +# Versioning ON +aws s3api put-bucket-versioning --bucket "$BUCKET_NAME" \ + --versioning-configuration Status=Enabled + +# SSE-S3 encryption (AES-256) +aws s3api put-bucket-encryption --bucket "$BUCKET_NAME" \ + --server-side-encryption-configuration '{ + "Rules": [{ + "ApplyServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}, + "BucketKeyEnabled": true + }] + }' + +# Lifecycle: keep 30 versions, expire noncurrent after 90 days +aws s3api put-bucket-lifecycle-configuration --bucket "$BUCKET_NAME" \ + --lifecycle-configuration '{ + "Rules": [{ + "ID": "keep-30-versions", + "Status": "Enabled", + "Filter": {"Prefix": ""}, + "NoncurrentVersionExpiration": {"NoncurrentDays": 90}, + "NoncurrentVersionTransitions": [], + "AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 7} + }] + }' + +info "S3 bucket configured." + +# ============================================================================= +# 4. IAM Role for EC2 +# ============================================================================= +info "Creating IAM role: ${STACK_NAME}-ec2-role..." + +ROLE_NAME="${STACK_NAME}-ec2-role" +POLICY_NAME="${STACK_NAME}-ec2-policy" +INSTANCE_PROFILE_NAME="${STACK_NAME}-ec2-profile" + +# Trust policy +aws iam create-role \ + --role-name "$ROLE_NAME" \ + --assume-role-policy-document '{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole" + }] + }' \ + --description "Mindcraft EC2 instance role" >/dev/null + +# Permissions policy: S3 + SSM +ROLE_ARN="arn:aws:iam::${ACCOUNT_ID}:role/${ROLE_NAME}" + +aws iam put-role-policy \ + --role-name "$ROLE_NAME" \ + --policy-name "$POLICY_NAME" \ + --policy-document "{ + \"Version\": \"2012-10-17\", + \"Statement\": [ + { + \"Sid\": \"S3BucketAccess\", + \"Effect\": \"Allow\", + \"Action\": [\"s3:GetObject\",\"s3:PutObject\",\"s3:DeleteObject\",\"s3:ListBucket\",\"s3:GetBucketLocation\"], + \"Resource\": [ + \"arn:aws:s3:::${BUCKET_NAME}\", + \"arn:aws:s3:::${BUCKET_NAME}/*\" + ] + }, + { + \"Sid\": \"SSMParameterAccess\", + \"Effect\": \"Allow\", + \"Action\": [\"ssm:GetParameter\",\"ssm:GetParameters\",\"ssm:GetParametersByPath\"], + \"Resource\": \"arn:aws:ssm:${REGION}:${ACCOUNT_ID}:parameter/mindcraft/*\" + } + ] + }" + +# Instance profile +aws iam create-instance-profile \ + --instance-profile-name "$INSTANCE_PROFILE_NAME" >/dev/null +aws iam add-role-to-instance-profile \ + --instance-profile-name "$INSTANCE_PROFILE_NAME" \ + --role-name "$ROLE_NAME" + +info "IAM role: ${ROLE_ARN}" + +# Wait for role to propagate +info "Waiting for IAM role propagation (10s)..." +sleep 10 + +# ============================================================================= +# 5. S3 Bucket Policy (EC2 role + admin IAM only) +# ============================================================================= +info "Applying S3 bucket policy..." +aws s3api put-bucket-policy --bucket "$BUCKET_NAME" \ + --policy "{ + \"Version\": \"2012-10-17\", + \"Statement\": [ + { + \"Sid\": \"DenyHTTP\", + \"Effect\": \"Deny\", + \"Principal\": \"*\", + \"Action\": \"s3:*\", + \"Resource\": [ + \"arn:aws:s3:::${BUCKET_NAME}\", + \"arn:aws:s3:::${BUCKET_NAME}/*\" + ], + \"Condition\": {\"Bool\": {\"aws:SecureTransport\": \"false\"}} + }, + { + \"Sid\": \"AllowEC2Role\", + \"Effect\": \"Allow\", + \"Principal\": {\"AWS\": \"${ROLE_ARN}\"}, + \"Action\": [\"s3:GetObject\",\"s3:PutObject\",\"s3:DeleteObject\",\"s3:ListBucket\",\"s3:GetBucketLocation\"], + \"Resource\": [ + \"arn:aws:s3:::${BUCKET_NAME}\", + \"arn:aws:s3:::${BUCKET_NAME}/*\" + ] + }, + { + \"Sid\": \"AllowAdminIAM\", + \"Effect\": \"Allow\", + \"Principal\": {\"AWS\": \"${CALLER_ARN}\"}, + \"Action\": \"s3:*\", + \"Resource\": [ + \"arn:aws:s3:::${BUCKET_NAME}\", + \"arn:aws:s3:::${BUCKET_NAME}/*\" + ] + } + ] + }" +info "Bucket policy applied." + +# ============================================================================= +# 6. SSM Parameters +# ============================================================================= +info "Storing secrets in SSM Parameter Store..." + +put_param() { + local name="$1" value="$2" + if [[ -n "$value" ]]; then + aws ssm put-parameter --region "$REGION" \ + --name "/mindcraft/${name}" \ + --value "$value" \ + --type SecureString \ + --overwrite >/dev/null + info " Stored /mindcraft/${name}" + else + warn " Skipped /mindcraft/${name} (empty)" + fi +} + +put_param "GEMINI_API_KEY" "$GEMINI_API_KEY" +put_param "XAI_API_KEY" "$XAI_API_KEY" +put_param "ANTHROPIC_API_KEY" "$ANTHROPIC_API_KEY" +put_param "DISCORD_BOT_TOKEN" "$DISCORD_BOT_TOKEN" +put_param "BOT_DM_CHANNEL" "$BOT_DM_CHANNEL" +put_param "BACKUP_CHAT_CHANNEL" "$BACKUP_CHAT_CHANNEL" +put_param "S3_BUCKET" "$BUCKET_NAME" + +# ============================================================================= +# 7. EC2 Key Pair +# ============================================================================= +info "Creating EC2 key pair: ${KEY_NAME}..." + +if aws ec2 describe-key-pairs --region "$REGION" --key-names "$KEY_NAME" >/dev/null 2>&1; then + warn "Key pair '${KEY_NAME}' already exists. Delete it first if you want a new one:" + warn " aws ec2 delete-key-pair --region ${REGION} --key-name ${KEY_NAME}" +else + aws ec2 create-key-pair \ + --region "$REGION" \ + --key-name "$KEY_NAME" \ + --query 'KeyMaterial' \ + --output text > "$KEY_FILE" + chmod 600 "$KEY_FILE" + info "Private key saved to: ${KEY_FILE}" +fi + +# ============================================================================= +# 8. EC2 Instance +# ============================================================================= +info "Finding latest Ubuntu 24.04 AMI..." +AMI_ID=$(aws ec2 describe-images \ + --region "$REGION" \ + --owners 099720109477 \ + --filters "Name=name,Values=${AMI_NAME_FILTER}" \ + "Name=state,Values=available" \ + "Name=architecture,Values=x86_64" \ + --query 'sort_by(Images, &CreationDate)[-1].ImageId' \ + --output text) +info "AMI: ${AMI_ID}" + +info "Launching EC2 instance (${INSTANCE_TYPE})..." +INSTANCE_ID=$(aws ec2 run-instances \ + --region "$REGION" \ + --image-id "$AMI_ID" \ + --instance-type "$INSTANCE_TYPE" \ + --key-name "$KEY_NAME" \ + --subnet-id "$SUBNET_ID" \ + --security-group-ids "$SG_ID" \ + --iam-instance-profile "Name=${INSTANCE_PROFILE_NAME}" \ + --block-device-mappings '[{"DeviceName":"/dev/sda1","Ebs":{"VolumeSize":30,"VolumeType":"gp3","DeleteOnTermination":true}}]' \ + --user-data "file://$(dirname "$0")/user-data.sh" \ + --tag-specifications \ + "ResourceType=instance,Tags=[{Key=Name,Value=${STACK_NAME}-server}]" \ + "ResourceType=volume,Tags=[{Key=Name,Value=${STACK_NAME}-root}]" \ + --query 'Instances[0].InstanceId' \ + --output text) + +info "Instance launched: ${INSTANCE_ID}" +info "Waiting for instance to reach running state..." +aws ec2 wait instance-running --region "$REGION" --instance-ids "$INSTANCE_ID" + +EC2_IP=$(aws ec2 describe-instances \ + --region "$REGION" \ + --instance-ids "$INSTANCE_ID" \ + --query 'Reservations[0].Instances[0].PublicIpAddress' \ + --output text) + +# ============================================================================= +# 9. Write config.env +# ============================================================================= +cat > "$CONFIG_FILE" </dev/null || warn "Could not terminate instance (may not exist)" + info "Waiting for termination..." + aws ec2 wait instance-terminated --region "$REGION" --instance-ids "$INSTANCE_ID" 2>/dev/null || true +fi + +# ── Delete key pair ─────────────────────────────────────────────────────────── +if [[ -n "${KEY_NAME:-}" ]]; then + info "Deleting key pair: ${KEY_NAME}..." + aws ec2 delete-key-pair --region "$REGION" --key-name "$KEY_NAME" 2>/dev/null || warn "Key pair not found" +fi + +# ── Delete IAM ──────────────────────────────────────────────────────────────── +if [[ -n "${INSTANCE_PROFILE_NAME:-}" ]]; then + info "Removing IAM instance profile: ${INSTANCE_PROFILE_NAME}..." + aws iam remove-role-from-instance-profile \ + --instance-profile-name "$INSTANCE_PROFILE_NAME" \ + --role-name "${ROLE_NAME}" 2>/dev/null || true + aws iam delete-instance-profile \ + --instance-profile-name "$INSTANCE_PROFILE_NAME" 2>/dev/null || warn "Instance profile not found" +fi + +if [[ -n "${ROLE_NAME:-}" ]]; then + info "Deleting IAM role: ${ROLE_NAME}..." + # Delete inline policies first + POLICIES=$(aws iam list-role-policies --role-name "$ROLE_NAME" --query 'PolicyNames' --output text 2>/dev/null || echo "") + for p in $POLICIES; do + aws iam delete-role-policy --role-name "$ROLE_NAME" --policy-name "$p" 2>/dev/null || true + done + aws iam delete-role --role-name "$ROLE_NAME" 2>/dev/null || warn "Role not found" +fi + +# ── Delete SSM parameters ───────────────────────────────────────────────────── +info "Deleting SSM parameters at /mindcraft/..." +PARAMS=$(aws ssm describe-parameters \ + --region "$REGION" \ + --parameter-filters "Key=Path,Values=/mindcraft" \ + --query 'Parameters[].Name' \ + --output text 2>/dev/null || echo "") +for p in $PARAMS; do + aws ssm delete-parameter --region "$REGION" --name "$p" 2>/dev/null || true + info " Deleted ${p}" +done + +# ── Delete security group ───────────────────────────────────────────────────── +if [[ -n "${SG_ID:-}" ]]; then + info "Deleting security group: ${SG_ID}..." + # Wait a moment for EC2 to fully detach + sleep 10 + aws ec2 delete-security-group --region "$REGION" --group-id "$SG_ID" 2>/dev/null || warn "SG not found or still in use" +fi + +# ── Delete VPC components ───────────────────────────────────────────────────── +if [[ -n "${SUBNET_ID:-}" ]]; then + info "Deleting subnet: ${SUBNET_ID}..." + aws ec2 delete-subnet --region "$REGION" --subnet-id "$SUBNET_ID" 2>/dev/null || warn "Subnet not found" +fi + +if [[ -n "${IGW_ID:-}" && -n "${VPC_ID:-}" ]]; then + info "Detaching and deleting IGW: ${IGW_ID}..." + aws ec2 detach-internet-gateway --region "$REGION" --internet-gateway-id "$IGW_ID" --vpc-id "$VPC_ID" 2>/dev/null || true + aws ec2 delete-internet-gateway --region "$REGION" --internet-gateway-id "$IGW_ID" 2>/dev/null || warn "IGW not found" +fi + +if [[ -n "${VPC_ID:-}" ]]; then + # Delete route tables (non-main) + RTB_IDS=$(aws ec2 describe-route-tables \ + --region "$REGION" \ + --filters "Name=vpc-id,Values=${VPC_ID}" \ + --query 'RouteTables[?Associations[0].Main!=`true`].RouteTableId' \ + --output text 2>/dev/null || echo "") + for rtb in $RTB_IDS; do + aws ec2 delete-route-table --region "$REGION" --route-table-id "$rtb" 2>/dev/null || true + done + + info "Deleting VPC: ${VPC_ID}..." + aws ec2 delete-vpc --region "$REGION" --vpc-id "$VPC_ID" 2>/dev/null || warn "VPC not found or has dependencies" +fi + +# ── S3 bucket (optional) ────────────────────────────────────────────────────── +if [[ "${DELETE_S3,,}" == "y" && -n "${BUCKET_NAME:-}" ]]; then + warn "Deleting S3 bucket and ALL contents: ${BUCKET_NAME}..." + # Remove all versions and delete markers first + aws s3api delete-objects \ + --bucket "$BUCKET_NAME" \ + --delete "$(aws s3api list-object-versions \ + --bucket "$BUCKET_NAME" \ + --query '{Objects: Versions[].{Key:Key,VersionId:VersionId}}' \ + --output json 2>/dev/null)" >/dev/null 2>&1 || true + aws s3 rm "s3://${BUCKET_NAME}" --recursive 2>/dev/null || true + aws s3api delete-bucket --bucket "$BUCKET_NAME" --region "$REGION" 2>/dev/null || warn "Bucket not found" + info "S3 bucket deleted." +else + info "S3 bucket preserved: ${BUCKET_NAME} (your backups are safe)" +fi + +# ── Clean up local config ───────────────────────────────────────────────────── +if [[ -f "$CONFIG_FILE" ]]; then + rm -f "$CONFIG_FILE" + info "Removed config.env" +fi +if [[ -f "${SCRIPT_DIR}/mindcraft-ec2.pem" ]]; then + rm -f "${SCRIPT_DIR}/mindcraft-ec2.pem" + info "Removed mindcraft-ec2.pem" +fi + +echo "" +echo -e "${GREEN}Teardown complete.${NC}" diff --git a/aws/user-data.sh b/aws/user-data.sh new file mode 100644 index 000000000..af46e9baa --- /dev/null +++ b/aws/user-data.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# ============================================================================= +# aws/user-data.sh — EC2 First-Boot Bootstrap Script +# ============================================================================= +# Runs as root on first boot via EC2 user-data. +# Installs Docker, Docker Compose plugin, AWS CLI v2. +# The actual app deployment is handled by aws/deploy.sh from your local machine. +# ============================================================================= +set -euo pipefail +exec > /var/log/user-data.log 2>&1 + +echo "=== Mindcraft EC2 Bootstrap: $(date) ===" + +# ── System update ───────────────────────────────────────────────────────────── +apt-get update -y +apt-get upgrade -y +apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + unzip \ + htop \ + git \ + jq \ + cron + +# ── Docker ──────────────────────────────────────────────────────────────────── +install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + -o /etc/apt/keyrings/docker.asc +chmod a+r /etc/apt/keyrings/docker.asc + +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \ + https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ + > /etc/apt/sources.list.d/docker.list + +apt-get update -y +apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + +systemctl enable docker +systemctl start docker + +# Allow ubuntu user to run docker without sudo +usermod -aG docker ubuntu + +# ── AWS CLI v2 ──────────────────────────────────────────────────────────────── +curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip +unzip -q /tmp/awscliv2.zip -d /tmp +/tmp/aws/install +rm -rf /tmp/aws /tmp/awscliv2.zip + +# ── App directory ───────────────────────────────────────────────────────────── +mkdir -p /app +chown ubuntu:ubuntu /app + +# ── Cron: backup every 6 hours ──────────────────────────────────────────────── +# The actual backup script is deployed by aws/deploy.sh +# Cron job added after deploy.sh runs for the first time via: +# sudo crontab -u ubuntu /app/aws/cron.tab +# Placeholder created here so the file exists +cat > /app/aws-cron.tab <<'CRON' +# Mindcraft world backup — every 6 hours +0 */6 * * * /app/aws/backup.sh >> /var/log/mindcraft-backup.log 2>&1 +CRON +chown ubuntu:ubuntu /app/aws-cron.tab + +# ── Ready marker ────────────────────────────────────────────────────────────── +touch /var/lib/cloud/instance/mindcraft-bootstrap-done +echo "=== Bootstrap complete: $(date) ===" +echo "Waiting for aws/deploy.sh to push application files..." diff --git a/docker-compose.aws.yml b/docker-compose.aws.yml new file mode 100644 index 000000000..50386c936 --- /dev/null +++ b/docker-compose.aws.yml @@ -0,0 +1,280 @@ +services: + # ── Minecraft Server (Paper + Cross-Play) ────────────────────────────────── + # Paper server with ViaVersion suite (Java backward compat) and + # Geyser (Bedrock cross-play on UDP 19132). + # Migrated from vanilla 2025-02-27. Backup: minecraft-data-backup-pre-paper-* + minecraft: + image: itzg/minecraft-server + container_name: minecraft-server + # ⚠️ SECURITY: ONLINE_MODE=FALSE (required for mineflayer) disables Mojang auth. + # Anyone who knows the EC2 IP can join. Restrict the external port at the AWS + # Security Group level to trusted IPs only (your own IPs / player IPs). + ports: + - "${MINECRAFT_PORT:-42069}:25565" # Java Edition (override via MINECRAFT_PORT in .env) + - "19132:19132/udp" # Bedrock Edition (Geyser) + environment: + EULA: "TRUE" + TYPE: "PAPER" + VERSION: "LATEST" # Latest MC version — ensure mineflayer supports it + MEMORY: "2G" # 2G JVM heap — Aikar GC keeps it efficient + USE_AIKAR_FLAGS: "true" # Optimized GC flags for Paper (reduces GC pauses) + DIFFICULTY: "normal" + MODE: "survival" + ENABLE_COMMAND_BLOCK: "true" + VIEW_DISTANCE: "6" + SIMULATION_DISTANCE: "4" + ONLINE_MODE: "FALSE" # Required for mineflayer bots (offline auth) + ENFORCE_SECURE_PROFILE: "FALSE" # Allow unsigned chat from mineflayer bots (1.19.1+ signed-chat requirement) + ENABLE_RCON: "true" + RCON_PASSWORD: "${RCON_PASSWORD:?Set RCON_PASSWORD in .env}" + # ── Whitelist Security ── + ENFORCE_WHITELIST: "TRUE" # Enable whitelist (only whitelisted players can join) + # WHITELIST env var intentionally omitted: it queries Playerdb (Mojang API) to resolve + # usernames, which fails for offline-mode bot accounts. Instead, whitelist.json is + # pre-generated with correct offline UUIDs (OfflinePlayer: MD5) and mounted below. + # ── Plugin auto-download from Modrinth ── + # ViaVersion suite: lets older Java clients connect + # Geyser: lets Bedrock clients connect (Floodgate not needed w/ ONLINE_MODE=FALSE) + MODRINTH_PROJECTS: "viaversion,viabackwards,viarewind,geyser:beta" + # Geyser uses :beta because no stable release exists on Modrinth yet + volumes: + - ./minecraft-data:/data + - ./whitelist.json:/data/whitelist.json:ro # Pre-built offline UUIDs; see whitelist.json at repo root + restart: unless-stopped + stdin_open: true + tty: true + healthcheck: + test: ["CMD", "mc-health", "--host", "localhost", "--port", "25565"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 120s # Paper + plugins need more startup time + deploy: + resources: + limits: + cpus: "1.5" + memory: 3500M # 2G heap + 1.5G for Geyser native/metaspace/GC + + # ── Mindcraft Agents ────────────────────────────────────────────────────── + mindcraft: + build: . + container_name: mindcraft-agents + network_mode: host + command: sh -c "Xvfb :99 -screen 0 1024x768x24 &>/dev/null & export DISPLAY=:99 && sleep 2 && exec node main.js" + environment: + PROFILES: '["./profiles/cloud-persistent.json", "./profiles/claude-explorer.json"]' + SETTINGS_JSON: '{"auto_open_ui": false, "host": "localhost", "mindserver_host_public": true, "mindserver_url": "", "allow_vision": true, "render_bot_view": true}' + LIBGL_ALWAYS_SOFTWARE: "1" + CHROMADB_URL: "http://localhost:8000" + GEMINI_API_KEY: "${GEMINI_API_KEY}" + XAI_API_KEY: "${XAI_API_KEY}" + ANTHROPIC_API_KEY: "${ANTHROPIC_API_KEY}" + volumes: + - ./settings.js:/app/settings.js + - ./profiles:/app/profiles + - ./bots:/app/bots + depends_on: + - minecraft + - chromadb + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.0" + memory: 2560M + + # ── Discord Bot ─────────────────────────────────────────────────────────── + discord-bot: + image: app-mindcraft + container_name: discord-bot + working_dir: /app + command: node discord-bot.js + volumes: + - ./discord-bot.js:/app/discord-bot.js + - ./package.json:/app/package.json + - ./src:/app/src + - ./profiles:/app/profiles + environment: + DISCORD_BOT_TOKEN: "${DISCORD_BOT_TOKEN}" + BOT_DM_CHANNEL: "${BOT_DM_CHANNEL}" + BACKUP_CHAT_CHANNEL: "${BACKUP_CHAT_CHANNEL}" + DISCORD_ADMIN_IDS: "${DISCORD_ADMIN_IDS}" + GOOGLE_API_KEY: "${GEMINI_API_KEY}" + MINDSERVER_HOST: "host.docker.internal" + PUBLIC_HOST: "${EC2_PUBLIC_IP}" + extra_hosts: + - "host.docker.internal:host-gateway" + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + + # ── ChromaDB (vector memory for ensemble) ──────────────────────────────── + chromadb: + image: chromadb/chroma:0.5.20 + container_name: chromadb + # Internal only — no host port binding. Reachable within Docker network as chromadb:8000. + expose: + - "8000" + volumes: + - ./chromadb-data:/chroma/chroma + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + + # ── Prometheus ──────────────────────────────────────────────────────────── + prometheus: + image: prom/prometheus:latest + container_name: prometheus + volumes: + - ./prometheus-aws.yml:/etc/prometheus/prometheus.yml + # Internal only — Grafana scrapes via Docker network. Not exposed to host. + expose: + - "9090" + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + + # ── Grafana ─────────────────────────────────────────────────────────────── + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - "3004:3000" + volumes: + - grafana-data:/var/lib/grafana + - ./grafana-provisioning/datasources.yml:/etc/grafana/provisioning/datasources/datasources.yml + - ./grafana-provisioning/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml + - ./grafana-provisioning/dashboard-json:/etc/grafana/provisioning/dashboards/json + - ./grafana-provisioning/alerting:/etc/grafana/provisioning/alerting + environment: + GF_SECURITY_ADMIN_PASSWORD: "${GF_ADMIN_PASSWORD:?Set GF_ADMIN_PASSWORD in .env}" + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + + # ── Node Exporter ───────────────────────────────────────────────────────── + node-exporter: + image: prom/node-exporter:latest + container_name: node-exporter + # Internal only — Prometheus scrapes via Docker network. Not exposed to host. + expose: + - "9100" + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.10" + memory: 64M + + # ── cAdvisor ────────────────────────────────────────────────────────────── + cadvisor: + image: gcr.io/cadvisor/cadvisor:latest + container_name: cadvisor + # Internal only — Prometheus scrapes via Docker network. Not exposed to host. + expose: + - "8080" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + + # ── World Backup → S3 ──────────────────────────────────────────────────── + backup: + image: offen/docker-volume-backup:v2 + container_name: minecraft-backup + environment: + AWS_S3_BUCKET_NAME: "${AWS_S3_BUCKET:-mindcraft-world-backups}" + AWS_DEFAULT_REGION: "us-east-1" + # AWS credentials intentionally omitted — use EC2 instance IAM role instead. + # Attach an IAM role with s3:PutObject/s3:DeleteObject on the bucket. + # Static keys would be readable by any container with docker.sock access. + BACKUP_CRON_EXPRESSION: "0 3 * * *" # Daily at 3 AM UTC + BACKUP_RETENTION_DAYS: "7" + BACKUP_FILENAME: "minecraft-world-%Y-%m-%dT%H-%M-%S.tar.gz" + volumes: + - ./minecraft-data:/backup/minecraft-data:ro + # docker.sock (ro) is required so the backup tool can pause containers for + # a consistent snapshot. Mitigated by: no static AWS keys in this container, + # IAM role scoped to the backup bucket only, and image pinned to v2. + - /var/run/docker.sock:/var/run/docker.sock:ro + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.10" + memory: 64M + + # ── LiteLLM — Unified LLM proxy ────────────────────────────────────────── + litellm: + image: ghcr.io/berriai/litellm:main-latest + container_name: litellm-proxy + ports: + - "4000:4000" + volumes: + - ./services/litellm/litellm_config.yaml:/app/config.yaml + environment: + GEMINI_API_KEY: "${GEMINI_API_KEY}" + XAI_API_KEY: "${XAI_API_KEY}" + ANTHROPIC_API_KEY: "${ANTHROPIC_API_KEY}" + VLLM_BASE_URL: "${VLLM_BASE_URL:-http://host.docker.internal:8000/v1}" + LITELLM_MASTER_KEY: "${LITELLM_MASTER_KEY}" + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + + # ── Tailscale VPN — connects EC2 to local GPU ────────────────────────────── + # network_mode: host puts tailscale0 in the HOST network namespace so that + # Docker bridge containers (mindcraft) can route to Tailscale IPs via the + # host gateway. Without this, the tunnel only exists in the container's + # namespace and mindcraft cannot reach 100.x.x.x addresses. + tailscale: + image: tailscale/tailscale:latest + container_name: tailscale + network_mode: host + environment: + TS_AUTHKEY: "${TAILSCALE_AUTHKEY}" + TS_EXTRA_ARGS: "" + TS_STATE_DIR: /var/lib/tailscale + TS_HOSTNAME: "mindcraft-ec2" + TS_USERSPACE: "false" + volumes: + - tailscale-state:/var/lib/tailscale + - /dev/net/tun:/dev/net/tun + cap_add: + - NET_ADMIN + - SYS_MODULE + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.10" + memory: 64M + + # viaproxy disabled — needs >256MB JVM heap, not enough RAM on t3.large + # To re-enable: uncomment and set memory limit to 512M + +volumes: + grafana-data: + tailscale-state: diff --git a/docker-compose.yml b/docker-compose.yml index eec423298..f16ad3b89 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,27 +1,220 @@ services: + + # ── GPU Monitoring (only starts with --profile local or --profile monitoring) ── + nvidia-gpu-exporter: + image: nvidia-gpu-exporter + container_name: nvidia-gpu-exporter + runtime: nvidia + environment: + - NVIDIA_VISIBLE_DEVICES=all + ports: + - "9835:9835" + restart: unless-stopped + profiles: + - local + - monitoring + + # ── Minecraft Server (Paper + Cross-Play) ──────────────────────────────────── + # Paper server with ViaVersion suite (Java backward compat) and + # Geyser (Bedrock cross-play on UDP 19132). + minecraft: + image: itzg/minecraft-server + container_name: minecraft-server + ports: + - "${MINECRAFT_PORT:-42069}:25565" # Java Edition (override via MINECRAFT_PORT in .env) + - "19132:19132/udp" # Bedrock Edition (Geyser) + environment: + EULA: "TRUE" + TYPE: "PAPER" + VERSION: "LATEST" # Latest MC version — ensure mineflayer supports it + MEMORY: "4G" + USE_AIKAR_FLAGS: "true" # Optimized GC flags for Paper (reduces GC pauses) + DIFFICULTY: "normal" + MODE: "survival" + ENABLE_COMMAND_BLOCK: "true" + VIEW_DISTANCE: "10" + ONLINE_MODE: "FALSE" # Required for mineflayer bots (offline auth) + ENFORCE_SECURE_PROFILE: "FALSE" # Allow unsigned chat from mineflayer bots (1.19.1+ signed-chat requirement) + ENABLE_RCON: "true" # Enables save-off/save-on for experiment backups + RCON_PASSWORD: "${RCON_PASSWORD:?Set RCON_PASSWORD in .env}" # Override via .env for production + # ── Whitelist Security ── + ENFORCE_WHITELIST: "TRUE" # Enable whitelist (only whitelisted players can join) + # WHITELIST env var intentionally omitted: it queries Playerdb (Mojang API) to resolve + # usernames, which fails for offline-mode bot accounts. Instead, whitelist.json is + # pre-generated with correct offline UUIDs (OfflinePlayer: MD5) and mounted below. + # ── Plugin auto-download from Modrinth ── + # ViaVersion suite: lets older Java clients connect + # Geyser: lets Bedrock clients connect (Floodgate not needed w/ ONLINE_MODE=FALSE) + MODRINTH_PROJECTS: "viaversion,viabackwards,viarewind,geyser:beta" + # Geyser uses :beta because no stable release exists on Modrinth yet + volumes: + - ./minecraft-data:/data + - ./whitelist.json:/data/whitelist.json:ro # Pre-built offline UUIDs; see whitelist.json at repo root + restart: unless-stopped + stdin_open: true + tty: true + logging: + driver: json-file + options: + max-size: "50m" + max-file: "3" + healthcheck: + test: ["CMD", "mc-health", "--host", "localhost", "--port", "25565"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 90s + + # ── Mindcraft Agent(s) (always starts — PROFILES env controls which bots load) ─ + # Set PROFILES before calling docker compose, or use start.ps1. + # Examples: + # PROFILES='["./profiles/local-research.json"]' → LocalResearch_1 only + # PROFILES='["./profiles/cloud-persistent.json"]' → CloudPersistent_1 only + # PROFILES='["./profiles/local-research.json","./profiles/cloud-persistent.json"]' → both mindcraft: build: . + container_name: mindcraft-agents environment: - SETTINGS_JSON: | - {"auto_open_ui": false} + SETTINGS_JSON: '{"auto_open_ui": false, "mindserver_host_public": true, "host": "minecraft-server"}' + PROFILES: "${PROFILES:-}" + GEMINI_API_KEY: "${GEMINI_API_KEY:-}" + XAI_API_KEY: "${XAI_API_KEY:-}" + ANTHROPIC_API_KEY: "${ANTHROPIC_API_KEY:-}" + OPENAI_API_KEY: "${OPENAI_API_KEY:-}" + VLLM_BASE_URL: "${VLLM_BASE_URL:-http://host.docker.internal:8000/v1}" volumes: - ./settings.js:/app/settings.js - ./keys.json:/app/keys.json - ./profiles:/app/profiles - ./bots:/app/bots ports: - - "3000-3003:3000-3003" # see the view from the camera mounted on your bot head: http://localhost:3000/ - - 8080 # Mindserver port + - "3000-3003:3000-3003" # Bot camera views: http://localhost:3000/ + - "8080:8080" # MindServer UI: http://localhost:8080/ + extra_hosts: + - "host.docker.internal:host-gateway" # Reach Ollama/vLLM running on Windows host + depends_on: + minecraft: + condition: service_healthy + restart: unless-stopped + logging: + driver: json-file + options: + max-size: "100m" + max-file: "5" + healthcheck: + test: ["CMD-SHELL", "node -e \"fetch('http://localhost:8080/').then(r=>{process.exit(r.ok?0:1)}).catch(()=>process.exit(1))\""] + interval: 15s + timeout: 10s + retries: 8 + start_period: 120s + + # ── Discord Bot (only starts with --profile cloud or --profile discord) ──────── + discord-bot: + image: node:22-slim + container_name: discord-bot + working_dir: /app + command: sh -c "npm install --production 2>/dev/null; node discord-bot.js" + volumes: + - ./discord-bot.js:/app/discord-bot.js + - ./package.json:/app/package.json + - ./src:/app/src + - ./profiles:/app/profiles + environment: + DISCORD_BOT_TOKEN: "${DISCORD_BOT_TOKEN}" + BOT_DM_CHANNEL: "${BOT_DM_CHANNEL}" + BACKUP_CHAT_CHANNEL: "${BACKUP_CHAT_CHANNEL}" + DISCORD_ADMIN_IDS: "${DISCORD_ADMIN_IDS}" + MINDSERVER_HOST: "${MINDSERVER_HOST:-mindcraft-agents}" + depends_on: + mindcraft: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "node -e \"require('net').createConnection({port:8080,host:'mindcraft-agents',timeout:3000}).on('connect',()=>process.exit(0)).on('error',()=>process.exit(1))\""] + interval: 60s + timeout: 5s + retries: 3 + start_period: 30s + logging: + driver: json-file + options: + max-size: "20m" + max-file: "3" + profiles: + - cloud + - discord + + # ── LiteLLM Proxy (only starts with --profile litellm) ─────────────────────── + # Unified OpenAI-compatible proxy for Ollama and cloud models. + # Start: docker compose --profile litellm up -d litellm + # Health: http://localhost:4000/health + litellm: + image: ghcr.io/berriai/litellm:main-latest + container_name: litellm-proxy + ports: + - "4000:4000" + volumes: + - ./services/litellm/litellm_config.yaml:/app/config.yaml + environment: + GEMINI_API_KEY: "${GEMINI_API_KEY:-}" + XAI_API_KEY: "${XAI_API_KEY:-}" + ANTHROPIC_API_KEY: "${ANTHROPIC_API_KEY:-}" + VLLM_BASE_URL: "${VLLM_BASE_URL:-http://host.docker.internal:8000/v1}" + LITELLM_MASTER_KEY: "${LITELLM_MASTER_KEY:?Set LITELLM_MASTER_KEY in .env}" extra_hosts: - - "host.docker.internal:host-gateway" + - "host.docker.internal:host-gateway" + command: --config /app/config.yaml + restart: unless-stopped + logging: + driver: json-file + options: + max-size: "30m" + max-file: "3" + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:4000/health || exit 1"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 45s + profiles: + - litellm - viaproxy: #use this service to connect to an unsupported minecraft server versions. more info: ./services/viaproxy/README.md + # ── ViaProxy (only starts with --profile viaproxy) ─────────────────────────── + viaproxy: image: ghcr.io/viaversion/viaproxy:latest volumes: - ./services/viaproxy:/app/run ports: - "25568:25568" - profiles: - - viaproxy stdin_open: true tty: true + profiles: + - viaproxy + + # ── vLLM (deferred — using WSL2 or Ollama instead) ─────────────────────────── + # Start vLLM: wsl -d Ubuntu-22.04 -- bash services/vllm/start.sh --background + # Endpoint: http://host.docker.internal:8000/v1 + # vllm: + # image: vllm/vllm-openai:latest + # container_name: vllm-server + # ports: + # - "8000:8000" + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: 1 + # capabilities: [gpu] + # command: > + # --model google/gemma-3-12b-it + # --max-model-len 4096 + # --gpu-memory-utilization 0.90 + # --enforce-eager + # --dtype half + # restart: unless-stopped + # profiles: + # - vllm + +volumes: + vllm-models: diff --git a/grafana-provisioning/alerting/.gitkeep b/grafana-provisioning/alerting/.gitkeep new file mode 100644 index 000000000..1d9b9a2a3 --- /dev/null +++ b/grafana-provisioning/alerting/.gitkeep @@ -0,0 +1 @@ +# Alerting rules directory — add rule YAML files here diff --git a/grafana-provisioning/alerting/rules.yml b/grafana-provisioning/alerting/rules.yml new file mode 100644 index 000000000..a1c3b28ab --- /dev/null +++ b/grafana-provisioning/alerting/rules.yml @@ -0,0 +1,7 @@ +apiVersion: 1 + +# Set deleteRules to remove stale alert rules that no longer apply. +# The gpu-exporter-down rule fires because there is no GPU exporter running. +deleteRules: + - orgId: 1 + uid: gpu-exporter-down diff --git a/grafana-provisioning/dashboard-json/.gitkeep b/grafana-provisioning/dashboard-json/.gitkeep new file mode 100644 index 000000000..44a145090 --- /dev/null +++ b/grafana-provisioning/dashboard-json/.gitkeep @@ -0,0 +1 @@ +# Dashboard JSON files directory — add Grafana dashboard JSON exports here diff --git a/grafana-provisioning/dashboards.yml b/grafana-provisioning/dashboards.yml new file mode 100644 index 000000000..a3b3e08c5 --- /dev/null +++ b/grafana-provisioning/dashboards.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + options: + path: /etc/grafana/provisioning/dashboards/json diff --git a/grafana-provisioning/datasources.yml b/grafana-provisioning/datasources.yml new file mode 100644 index 000000000..83ad3390c --- /dev/null +++ b/grafana-provisioning/datasources.yml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true diff --git a/prometheus-aws.yml b/prometheus-aws.yml new file mode 100644 index 000000000..569bd3c97 --- /dev/null +++ b/prometheus-aws.yml @@ -0,0 +1,16 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + - job_name: node-exporter + static_configs: + - targets: ["node-exporter:9100"] + + - job_name: cadvisor + static_configs: + - targets: ["cadvisor:8080"] diff --git a/services/viaproxy/README.md b/services/viaproxy/README.md index 5ad445d6b..5f20c5df3 100644 --- a/services/viaproxy/README.md +++ b/services/viaproxy/README.md @@ -1,3 +1,5 @@ +# ViaProxy Setup + Use this service to connect your bot to an unsupported minecraft server versions. Run: @@ -8,7 +10,7 @@ docker-compose --profile viaproxy up After first start it will create config file `services/viaproxy/viaproxy.yml`. -Edit this file, and change your desired target `target-address`, +Edit this file, and change your desired target `target-address`, then point your `settings.js` `host` and `port` to viaproxy endpoint: @@ -17,17 +19,18 @@ then point your `settings.js` `host` and `port` to viaproxy endpoint: "port": 25568, ``` -This easily works with "offline" servers. +This easily works with "offline" servers. Connecting to "online" servers via viaproxy involves more effort:\ First start the ViaProxy container, then open another terminal in the mindcraft directory.\ Run `docker attach mindcraft-viaproxy-1` in the new terminal to attach to the container.\ After attaching, you can use the `account` command to manage user accounts: - - `account list` List all accounts in the list - - `account add microsoft` Add a microsoft account (run the command and follow the instructions) - - `account select ` Select the account to be used (run `account list` to see the ids) - - `account remove ` Remove an account (run `account list` to see the ids) - - `account deselect` Deselect the current account (go back to offline mode) + +- `account list` List all accounts in the list +- `account add microsoft` Add a microsoft account (run the command and follow the instructions) +- `account select ` Select the account to be used (run `account list` to see the ids) +- `account remove ` Remove an account (run `account list` to see the ids) +- `account deselect` Deselect the current account (go back to offline mode) > [!WARNING] > If you login with a microsoft account, the access token is stored in the `saves.json` file.\ @@ -36,6 +39,8 @@ After attaching, you can use the `account` command to manage user accounts: When you're done setting up your account (don't forget to select it), use `CTRL-P` then `CTRL-Q` to detach from the container. If you want to persist these changes, you can configure them in the `services/viaproxy/viaproxy.yml`. + 1. Change `auth-method` to `account` 2. Change `minecraft-account-index` to the id of your account + diff --git a/start.ps1 b/start.ps1 new file mode 100644 index 000000000..9804bd1f6 --- /dev/null +++ b/start.ps1 @@ -0,0 +1,19 @@ +<# +.SYNOPSIS + Clean launch: both bots, no syntax errors +#> +param( + [switch]$Detach +) + +Write-Host "Launching both bots..." -ForegroundColor Cyan +docker compose --profile both up -d + +Write-Host "Both bots starting..." -ForegroundColor Green +Write-Host "Check Minecraft: localhost:25565" -ForegroundColor Green +Write-Host "Discord: MindcraftBot should go online" -ForegroundColor Green +Write-Host "Logs: docker compose logs -f mindcraft" -ForegroundColor Green + +if ($Detach) { + Write-Host "Detached - running in background" -ForegroundColor Green +}