Add verbose comments on the current setup

Author: joshhvulcan

.github/workflows/build_test.yaml

@@ -58,7 +58,25 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      # Add deploy keys and clone repositories
+      # ============================================================================
+      # SSH Multi-Key Setup for Private Repository Access
+      # ============================================================================
+      # The Docker build needs to clone from multiple private GitHub repositories:
+      # - allenai/rslearn
+      # - allenai/olmoearth_pretrain
+      # - allenai/olmoearth_run
+      #
+      # PROBLEM: GitHub deploy keys are scoped to a single repository. We need
+      # separate deploy keys for each repo. However, when SSH connects to github.com
+      # with multiple keys loaded in the agent, it tries them in order. If the first
+      # key authenticates successfully but lacks access to a specific repository,
+      # GitHub returns "Repository not found" and SSH stops trying other keys.
+      #
+      # SOLUTION: We use SSH host aliases to route each repository to its specific key.
+      # webfactory/ssh-agent sets up the SSH agent on the runner, but Docker builds
+      # run in an isolated container that doesn't have access to the runner's SSH
+      # config or git URL rewrites. So we must recreate this setup inside the container.
+
       - name: Setup SSH agent with deploy keys
         uses: webfactory/[email protected]
         with:
@@ -77,6 +95,10 @@ jobs:
           echo "=== Git Config URL Rewrites ==="
           git config --global --get-regexp 'url\..*\.insteadof' || echo "No git URL rewrites found"
 
+      # Write SSH keys to files for use inside Docker container
+      # The SSH agent runs on the runner but we need actual key files inside the
+      # Docker container to create SSH config with host aliases. We write the keys
+      # to .docker-ssh/ which gets copied into the container during build.
       - name: Prepare SSH keys for Docker build
         run: |
           mkdir -p .docker-ssh

Dockerfile

@@ -44,7 +44,48 @@ COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
 
 COPY . /opt/rslearn_projects/
 
-# Set up SSH multi-key support for GitHub Actions (conditional based on build arg)
+# ============================================================================
+# SSH Multi-Key Configuration for GitHub Actions
+# ============================================================================
+# This section handles SSH authentication for multiple private GitHub repositories
+# when building in CI/CD environments that use separate deploy keys per repository.
+#
+# CONTEXT:
+# - GitHub deploy keys are scoped to a single repository for security
+# - We need to clone from multiple repos: rslearn, olmoearth_pretrain, olmoearth_run
+# - Each repository has its own deploy key stored as a GitHub Actions secret
+#
+# THE PROBLEM:
+# When SSH tries to connect to github.com with multiple keys in the agent:
+#   1. SSH tries the first key
+#   2. If it authenticates successfully, SSH stops trying other keys
+#   3. If that key lacks access to a specific repo, GitHub returns "Repository not found"
+#   4. The connection fails even though other keys in the agent might have access
+#
+# This happens because SSH authenticates at the HOST level (github.com), not the
+# repository level. Once any key authenticates with github.com, SSH considers the
+# connection successful and doesn't try other keys.
+#
+# THE SOLUTION:
+# We use SSH host aliases to make SSH think each repository is on a different host:
+#   - git@github-olmoearth-pretrain:allenai/olmoearth_pretrain.git
+#   - git@github-olmoearth-run:allenai/olmoearth_run.git
+#
+# Each alias points to github.com but specifies which SSH key to use via IdentityFile.
+# With IdentitiesOnly=yes, SSH only tries the specified key for that alias.
+#
+# IMPLEMENTATION:
+# The GitHub Actions workflow writes deploy keys to .docker-ssh/ and sets
+# USE_SSH_KEYS_FROM_BUILD=true. This code then:
+#   1. Copies the key files to ~/.ssh/
+#   2. Creates SSH config with host aliases mapped to specific keys
+#   3. Rewrites requirements*.txt files to use the host aliases
+#
+# LOCAL DEVELOPMENT:
+# Local developers typically have a single SSH key with access to all repositories,
+# so this complexity is unnecessary. When USE_SSH_KEYS_FROM_BUILD=false (default),
+# this entire setup is skipped and standard SSH agent forwarding is used.
+
 ARG USE_SSH_KEYS_FROM_BUILD=false
 RUN if [ "$USE_SSH_KEYS_FROM_BUILD" = "true" ] && [ -d /opt/rslearn_projects/.docker-ssh ]; then \
       echo "Setting up SSH keys from build context..." && \