MDArena/no-github-access.yaml at main · weitse-hsu/MDArena · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
# Blocks GitHub - and the third-party services that re-serve GitHub content
# verbatim - during MDArena evaluation runs, so an agent cannot look up a
# task's verifier or reference solution mid-run.
#
# `extra_hosts` writes entries into the container's /etc/hosts. Mapping these
# hostnames to 127.0.0.1 makes them resolve to localhost, where nothing is
# listening, so connections fail. Despite the field name, this adds no access -
# it is DNS override, not a firewall.
#
# What this cannot do, stated plainly:
#   - /etc/hosts has no wildcard syntax. Every hostname must be listed
#     literally, so a subdomain or mirror not named below is not covered, and
#     new repo-to-text proxies appear regularly.
#   - An agent that connects by IP address bypasses it entirely.
#   - It cannot be scoped to the MDArena repository alone. HTTPS exposes only
#     the hostname, via SNI, and never the request path, so
#     `github.com/<anything-else>` is indistinguishable from MDArena's own URL
#     at this layer. Blocking the repository means blocking the host.
#
# Treat this as raising the cost of the obvious lookup, not as a guarantee. It
# pairs well with a post-hoc scan of the agent trajectories under ./jobs for
# retrieval attempts, which catches whatever this list misses.
#
# Usage: harbor run ... --extra-docker-compose no-github-access.yaml
#
# Two run-time interactions to know about before using this:
#
#   - `49_slco2a1_physical_validation` is the only task whose instruction.md
#     points the agent at GitHub (it names the physical_validation repository).
#     Under this overlay that install route fails. The task stays solvable
#     because `physical-validation` is on PyPI and PyPI stays reachable, but
#     the agent has to work that out unaided. See TODOS.md item 8.
#
#   - `-a codex` does not survive this overlay. On a glibc image such as
#     mdarena:latest, Harbor installs the Codex CLI by bootstrapping nvm from
#     raw.githubusercontent.com, so the agent fails to install before the task
#     starts. Harbor skips that install when `command -v codex` already
#     succeeds, so baking the Codex CLI into the image is the fix; until then,
#     run Codex evaluations without this overlay. The Claude Code CLI is
#     unaffected - it installs from downloads.claude.ai or npm.

services:
  main:
    extra_hosts:
      # GitHub itself.
      - "github.com:127.0.0.1"
      - "www.github.com:127.0.0.1"
      - "api.github.com:127.0.0.1"
      - "codeload.github.com:127.0.0.1"
      - "gist.github.com:127.0.0.1"
      - "github.dev:127.0.0.1"

      # GitHub's own content hosts, which serve raw file and gist bodies.
      - "raw.githubusercontent.com:127.0.0.1"
      - "objects.githubusercontent.com:127.0.0.1"
      - "gist.githubusercontent.com:127.0.0.1"
      - "media.githubusercontent.com:127.0.0.1"

      # CDNs that re-serve arbitrary files from any public GitHub repository by
      # path, e.g. cdn.jsdelivr.net/gh/<owner>/<repo>@<ref>/<path>. Blocking
      # github.com without these leaves the file contents reachable.
      - "jsdelivr.net:127.0.0.1"
      - "cdn.jsdelivr.net:127.0.0.1"
      - "fastly.jsdelivr.net:127.0.0.1"
      - "gcore.jsdelivr.net:127.0.0.1"
      - "testingcf.jsdelivr.net:127.0.0.1"
      - "raw.githack.com:127.0.0.1"
      - "rawcdn.githack.com:127.0.0.1"
      - "statically.io:127.0.0.1"
      - "cdn.statically.io:127.0.0.1"

      # Repo-to-text services. Swapping github.com for one of these domains
      # returns the whole repository as a single LLM-friendly document, which
      # makes them a more convenient route to a verifier than github.com.
      - "uithub.com:127.0.0.1"
      - "gitingest.com:127.0.0.1"
      - "deepwiki.com:127.0.0.1"
      - "github1s.com:127.0.0.1"

      # Generic URL-fetch proxies, which would relay any of the above.
      - "r.jina.ai:127.0.0.1"

      # Code search over public repositories: enough to surface a verifier's
      # contents without fetching the repository at all.
      - "grep.app:127.0.0.1"
      - "sourcegraph.com:127.0.0.1"
      - "searchcode.com:127.0.0.1"