Skip to content

Commit ef562e1

Browse files
maxamillionmaxamillion
authored
Merge pull request #31 from tyraziel/k3s-mirror-enablement
Add docker.io mirror support for internal k3s cluster
2 parents 5531551 + b2435d1 commit ef562e1

5 files changed

Lines changed: 298 additions & 1 deletion

File tree

‎architecture/gateway.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ All configuration is via CLI flags with environment variable fallbacks. The `--d
128128
| `--sandbox-ssh-port` | `OPENSHELL_SANDBOX_SSH_PORT` | `2222` | SSH listen port inside sandbox pods |
129129
| `--ssh-handshake-secret` | `OPENSHELL_SSH_HANDSHAKE_SECRET` | None | Shared HMAC-SHA256 secret for gateway-to-sandbox handshake |
130130
| `--ssh-handshake-skew-secs` | `OPENSHELL_SSH_HANDSHAKE_SKEW_SECS` | `300` | Allowed clock skew (seconds) for SSH handshake timestamps |
131+
| — | `OPENSHELL_CONTAINER_REGISTRY` | None | Default container registry for image pulls. When set, all images that don't specify an explicit registry are pulled from this endpoint instead of `docker.io`. Useful when Docker Hub is blocked by corporate firewalls, air-gapped networks, or rate limiting. |
131132

132133
## Shared State
133134

‎crates/openshell-bootstrap/src/docker.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -842,6 +842,10 @@ pub async fn ensure_container(
842842
if let Some(endpoint) = registry_endpoint {
843843
env_vars.push(format!("REGISTRY_ENDPOINT={endpoint}"));
844844
}
845+
// Default container registry when docker.io is unavailable.
846+
if let Some(registry) = env_non_empty("OPENSHELL_CONTAINER_REGISTRY") {
847+
env_vars.push(format!("OPENSHELL_CONTAINER_REGISTRY={registry}"));
848+
}
845849
if let Some(password) = effective_password {
846850
// Default to __token__ when only a password/token is provided.
847851
let username = effective_username.unwrap_or_else(|| "__token__".to_string());

‎deploy/docker/cluster-entrypoint.sh‎

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,18 @@ REGEOF
282282
REGEOF
283283
fi
284284

285+
# Default container registry when docker.io is unavailable.
286+
if [ -n "${OPENSHELL_CONTAINER_REGISTRY:-}" ]; then
287+
echo "Adding default container registry: ${OPENSHELL_CONTAINER_REGISTRY}"
288+
cat >>"$REGISTRIES_YAML" <<REGEOF
289+
"docker.io":
290+
endpoint:
291+
- "${OPENSHELL_CONTAINER_REGISTRY}"
292+
REGEOF
293+
else
294+
echo "Info: OPENSHELL_CONTAINER_REGISTRY not set; unqualified image pulls will use docker.io directly"
295+
fi
296+
285297
if [ -n "${REGISTRY_USERNAME:-}" ] && [ -n "${REGISTRY_PASSWORD:-}" ]; then
286298
cat >>"$REGISTRIES_YAML" <<REGEOF
287299
@@ -318,7 +330,22 @@ REGEOF
318330
fi
319331
fi
320332
else
321-
echo "Warning: REGISTRY_HOST not set; skipping registry config"
333+
echo "Warning: REGISTRY_HOST not set; skipping OpenShell component registry mirror config"
334+
fi
335+
336+
# Default container registry — even without a component registry
337+
# configured, k3s still needs to pull system images (e.g.
338+
# rancher/mirrored-pause) and will fail if docker.io is unreachable.
339+
if [ -n "${OPENSHELL_CONTAINER_REGISTRY:-}" ] && [ ! -f "$REGISTRIES_YAML" ]; then
340+
echo "Configuring default container registry: ${OPENSHELL_CONTAINER_REGISTRY}"
341+
cat >"$REGISTRIES_YAML" <<REGEOF
342+
mirrors:
343+
"docker.io":
344+
endpoint:
345+
- "${OPENSHELL_CONTAINER_REGISTRY}"
346+
REGEOF
347+
elif [ ! -f "$REGISTRIES_YAML" ]; then
348+
echo "Info: OPENSHELL_CONTAINER_REGISTRY not set; unqualified image pulls will use docker.io directly"
322349
fi
323350

324351
# Copy bundled Helm chart tarballs to the k3s static charts directory.

‎docs/reference/support-matrix.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ To override the default image references, set the following environment variable
5454
| ------------------------------ | --------------------------------------------------- |
5555
| `OPENSHELL_CLUSTER_IMAGE` | Override the cluster image reference. |
5656
| `OPENSHELL_COMMUNITY_REGISTRY` | Override the registry for community sandbox images. |
57+
| `OPENSHELL_CONTAINER_REGISTRY` | Default container registry for image pulls. When set, images without an explicit registry are pulled from this endpoint instead of `docker.io`. |
5758

5859
## Kernel Requirements
5960

‎e2e/cluster-entrypoint/registries_test.sh‎

Lines changed: 264 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,264 @@
1+
#!/bin/sh
2+
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Tests for the registries.yaml generation logic in cluster-entrypoint.sh.
6+
#
7+
# Extracts the registry configuration block from the entrypoint and runs it
8+
# in isolation with various combinations of environment variables, then
9+
# asserts on the resulting registries.yaml content and log output.
10+
#
11+
# Usage: sh e2e/cluster-entrypoint/registries_test.sh
12+
13+
set -eu
14+
15+
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
16+
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
17+
ENTRYPOINT="$REPO_ROOT/deploy/docker/cluster-entrypoint.sh"
18+
19+
_PASS=0
20+
_FAIL=0
21+
22+
# ---------------------------------------------------------------------------
23+
# Assertions
24+
# ---------------------------------------------------------------------------
25+
26+
pass() {
27+
_PASS=$((_PASS + 1))
28+
printf ' PASS: %s\n' "$1"
29+
}
30+
31+
fail() {
32+
_FAIL=$((_FAIL + 1))
33+
printf ' FAIL: %s\n' "$1" >&2
34+
if [ -n "${2:-}" ]; then
35+
printf ' %s\n' "$2" >&2
36+
fi
37+
}
38+
39+
assert_file_exists() {
40+
if [ -f "$1" ]; then
41+
pass "$2"
42+
else
43+
fail "$2" "file not found: $1"
44+
fi
45+
}
46+
47+
assert_file_not_exists() {
48+
if [ ! -f "$1" ]; then
49+
pass "$2"
50+
else
51+
fail "$2" "file unexpectedly exists: $1"
52+
fi
53+
}
54+
55+
assert_file_contains() {
56+
if grep -qF "$2" "$1" 2>/dev/null; then
57+
pass "$3"
58+
else
59+
fail "$3" "expected '$2' in $1"
60+
fi
61+
}
62+
63+
assert_file_not_contains() {
64+
if ! grep -qF "$2" "$1" 2>/dev/null; then
65+
pass "$3"
66+
else
67+
fail "$3" "unexpected '$2' found in $1"
68+
fi
69+
}
70+
71+
assert_output_contains() {
72+
if printf '%s' "$1" | grep -qF "$2"; then
73+
pass "$3"
74+
else
75+
fail "$3" "expected '$2' in output"
76+
fi
77+
}
78+
79+
print_summary() {
80+
printf '\n=== Results: %d passed, %d failed ===\n' "$_PASS" "$_FAIL"
81+
[ "$_FAIL" -eq 0 ]
82+
}
83+
84+
# ---------------------------------------------------------------------------
85+
# Test harness
86+
# ---------------------------------------------------------------------------
87+
# Extracts the yaml_quote helper and the registries.yaml generation block
88+
# from the real cluster-entrypoint.sh and runs them in isolation. This
89+
# avoids executing the full entrypoint (DNS, iptables, k3s, etc.) while
90+
# ensuring tests always exercise the actual production code.
91+
92+
WORK_DIR=""
93+
94+
setup() {
95+
WORK_DIR="$(mktemp -d)"
96+
export REGISTRIES_YAML="$WORK_DIR/registries.yaml"
97+
98+
# Clear all registry-related env vars
99+
unset REGISTRY_HOST 2>/dev/null || true
100+
unset REGISTRY_ENDPOINT 2>/dev/null || true
101+
unset REGISTRY_INSECURE 2>/dev/null || true
102+
unset REGISTRY_USERNAME 2>/dev/null || true
103+
unset REGISTRY_PASSWORD 2>/dev/null || true
104+
unset COMMUNITY_REGISTRY_HOST 2>/dev/null || true
105+
unset COMMUNITY_REGISTRY_USERNAME 2>/dev/null || true
106+
unset COMMUNITY_REGISTRY_PASSWORD 2>/dev/null || true
107+
unset OPENSHELL_CONTAINER_REGISTRY 2>/dev/null || true
108+
}
109+
110+
teardown() {
111+
echo " Info: test artifacts left in $WORK_DIR"
112+
}
113+
114+
# Run the registries.yaml generation logic in isolation.
115+
# Extracts the yaml_quote function and the registry config block from the
116+
# real cluster-entrypoint.sh, then runs them in a subshell.
117+
# Captures stdout+stderr for log message assertions.
118+
run_registry_config() {
119+
{
120+
echo '#!/bin/sh'
121+
echo 'set -e'
122+
# Extract yaml_quote helper
123+
sed -n '/^yaml_quote()/,/^}/p' "$ENTRYPOINT"
124+
# Extract registry config block (between the two section markers),
125+
# stripping the REGISTRIES_YAML= assignment so our test path is used.
126+
sed -n '/^# Generate k3s private registry configuration/,/^# Copy bundled Helm chart tarballs/p' "$ENTRYPOINT" \
127+
| sed '1d' | sed '$d' \
128+
| sed '/^REGISTRIES_YAML="/d'
129+
} >"$WORK_DIR/generate.sh"
130+
chmod +x "$WORK_DIR/generate.sh"
131+
sh "$WORK_DIR/generate.sh" 2>&1
132+
}
133+
134+
# ---------------------------------------------------------------------------
135+
# Tests
136+
# ---------------------------------------------------------------------------
137+
138+
test_registry_host_and_container_registry() {
139+
printf 'TEST: REGISTRY_HOST set + OPENSHELL_CONTAINER_REGISTRY set\n'
140+
setup
141+
142+
export REGISTRY_HOST="ghcr.io"
143+
export OPENSHELL_CONTAINER_REGISTRY="https://mirror.gcr.io"
144+
145+
OUTPUT=$(run_registry_config)
146+
147+
assert_file_exists "$REGISTRIES_YAML" "registries.yaml created"
148+
assert_file_contains "$REGISTRIES_YAML" '"ghcr.io"' "contains ghcr.io registry"
149+
assert_file_contains "$REGISTRIES_YAML" '"docker.io"' "contains docker.io registry"
150+
assert_file_contains "$REGISTRIES_YAML" 'https://mirror.gcr.io' "contains mirror.gcr.io endpoint"
151+
assert_output_contains "$OUTPUT" "Adding default container registry" "logs container registry addition"
152+
153+
teardown
154+
}
155+
156+
test_registry_host_without_container_registry() {
157+
printf 'TEST: REGISTRY_HOST set + OPENSHELL_CONTAINER_REGISTRY unset\n'
158+
setup
159+
160+
export REGISTRY_HOST="ghcr.io"
161+
162+
OUTPUT=$(run_registry_config)
163+
164+
assert_file_exists "$REGISTRIES_YAML" "registries.yaml created"
165+
assert_file_contains "$REGISTRIES_YAML" '"ghcr.io"' "contains ghcr.io registry"
166+
assert_file_not_contains "$REGISTRIES_YAML" '"docker.io"' "does not contain docker.io registry"
167+
assert_output_contains "$OUTPUT" "unqualified image pulls will use docker.io directly" "logs info about direct docker.io pulls"
168+
169+
teardown
170+
}
171+
172+
test_no_registry_host_with_container_registry() {
173+
printf 'TEST: REGISTRY_HOST unset + OPENSHELL_CONTAINER_REGISTRY set\n'
174+
setup
175+
176+
export OPENSHELL_CONTAINER_REGISTRY="https://mirror.gcr.io"
177+
178+
OUTPUT=$(run_registry_config)
179+
180+
assert_file_exists "$REGISTRIES_YAML" "registries.yaml created"
181+
assert_file_contains "$REGISTRIES_YAML" '"docker.io"' "contains docker.io registry"
182+
assert_file_contains "$REGISTRIES_YAML" 'https://mirror.gcr.io' "contains mirror.gcr.io endpoint"
183+
assert_file_not_contains "$REGISTRIES_YAML" '"ghcr.io"' "does not contain ghcr.io registry"
184+
assert_output_contains "$OUTPUT" "Configuring default container registry" "logs container registry configuration"
185+
assert_output_contains "$OUTPUT" "REGISTRY_HOST not set" "logs warning about missing REGISTRY_HOST"
186+
187+
teardown
188+
}
189+
190+
test_no_registry_host_no_container_registry() {
191+
printf 'TEST: REGISTRY_HOST unset + OPENSHELL_CONTAINER_REGISTRY unset\n'
192+
setup
193+
194+
OUTPUT=$(run_registry_config)
195+
196+
assert_file_not_exists "$REGISTRIES_YAML" "registries.yaml not created"
197+
assert_output_contains "$OUTPUT" "REGISTRY_HOST not set" "logs warning about missing REGISTRY_HOST"
198+
assert_output_contains "$OUTPUT" "unqualified image pulls will use docker.io directly" "logs info about direct docker.io pulls"
199+
200+
teardown
201+
}
202+
203+
test_container_registry_with_auth() {
204+
printf 'TEST: REGISTRY_HOST + OPENSHELL_CONTAINER_REGISTRY + auth credentials\n'
205+
setup
206+
207+
export REGISTRY_HOST="ghcr.io"
208+
export REGISTRY_USERNAME="__token__"
209+
export REGISTRY_PASSWORD="ghp_faketoken123"
210+
export OPENSHELL_CONTAINER_REGISTRY="https://mirror.gcr.io"
211+
212+
OUTPUT=$(run_registry_config)
213+
214+
assert_file_exists "$REGISTRIES_YAML" "registries.yaml created"
215+
assert_file_contains "$REGISTRIES_YAML" '"docker.io"' "contains docker.io registry"
216+
assert_file_contains "$REGISTRIES_YAML" '"ghcr.io"' "contains ghcr.io registry"
217+
assert_file_contains "$REGISTRIES_YAML" "configs:" "contains configs section"
218+
assert_file_contains "$REGISTRIES_YAML" "__token__" "contains auth username"
219+
220+
teardown
221+
}
222+
223+
test_container_registry_valid_yaml_structure() {
224+
printf 'TEST: generated registries.yaml has valid YAML mirror structure\n'
225+
setup
226+
227+
export REGISTRY_HOST="ghcr.io"
228+
export OPENSHELL_CONTAINER_REGISTRY="https://mirror.gcr.io"
229+
230+
run_registry_config >/dev/null
231+
232+
# Verify docker.io entry is indented under mirrors: (not a top-level key)
233+
if grep -q '^ "docker.io":' "$REGISTRIES_YAML"; then
234+
pass "docker.io entry is correctly indented under mirrors:"
235+
else
236+
fail "docker.io entry is correctly indented under mirrors:" \
237+
"expected 2-space indent, got: $(grep 'docker.io' "$REGISTRIES_YAML")"
238+
fi
239+
240+
# Verify there is only one mirrors: top-level key
241+
mirrors_count=$(grep -c '^mirrors:' "$REGISTRIES_YAML")
242+
if [ "$mirrors_count" -eq 1 ]; then
243+
pass "exactly one mirrors: top-level key"
244+
else
245+
fail "exactly one mirrors: top-level key" "found $mirrors_count"
246+
fi
247+
248+
teardown
249+
}
250+
251+
# ---------------------------------------------------------------------------
252+
# Runner
253+
# ---------------------------------------------------------------------------
254+
255+
printf '=== cluster-entrypoint registries.yaml generation tests ===\n\n'
256+
257+
test_registry_host_and_container_registry; echo ""
258+
test_registry_host_without_container_registry; echo ""
259+
test_no_registry_host_with_container_registry; echo ""
260+
test_no_registry_host_no_container_registry; echo ""
261+
test_container_registry_with_auth; echo ""
262+
test_container_registry_valid_yaml_structure
263+
264+
print_summary

0 commit comments

Comments
 (0)