Skip to content

Commit 8fed49b

Browse files
EdwardCookeEdwardCooke
authored
Merge pull request #32 from cyclops-k8s/ec-mirrors
Add mirrors to proxy hosts and configure containerd bug fix
2 parents 8d84af5 + fb0a56c commit 8fed49b

File tree

23 files changed

+534
-31
lines changed

23 files changed

+534
-31
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,4 @@ override.tf.json
5050
terraform.rc
5151

5252
# End of https://www.toptal.com/developers/gitignore/api/ansible,terraform
53+
!test.tfvars

‎example-hooks/install-calico/post-cluster-init/install-calico.yaml‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,7 @@
1111
executable: /usr/bin/bash
1212

1313
- name: Wait for calico to be ready
14-
loop:
15-
- {namespace: kube-system, app: calico-node}
16-
loop_control:
17-
label: "{{ k8s_app.app }}"
18-
loop_var: k8s_app
1914
register: calico_ready
2015
retries: 12
2116
until: calico_ready.rc == 0
22-
ansible.builtin.command: kubectl --kubeconfig /etc/kubernetes/admin.conf wait --for=condition=Ready pod -l k8s-app={{ k8s_app.app }} --timeout=5s --namespace {{ k8s_app.namespace }}
17+
ansible.builtin.command: kubectl --kubeconfig /etc/kubernetes/admin.conf wait --for=condition=Ready pod -l k8s-app=calico-node --timeout=5s --namespace kube-system

‎example-hooks/proxy-on-control-planes/pre_control_planes/proxy-on-control-planes.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@
5656
sysctl_file: /etc/sysctl.d/99-non-local-bind.conf
5757
value: '1'
5858
state: present
59-
reload: yes
59+
reload: true
6060

6161
- name: Post proxy configuration hooks
6262
ansible.builtin.include_tasks: "{{ item }}"

‎example-hooks/registry-mirrors/post-proxies/add-containerd-mirrors.yaml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# execute the containerd roles
2+
- name: Setup mirrors
3+
when: registry_mirrors | default([]) | length > 0
4+
block:
5+
- name: Include containerd role
6+
vars:
7+
containerd_registry_mirrors: "{{ proxy_mirrors | default([]) }}"
8+
ansible.builtin.include_role:
9+
name: containerd
10+
11+
# unhold containerd.io package if it was held previously
12+
- name: Unhold containerd.io package so it can be updated during normal patching cycles
13+
ansible.builtin.dpkg_selections:
14+
name: containerd.io
15+
selection: install
16+
17+
- name: Install Docker engine
18+
ansible.builtin.package:
19+
name:
20+
- docker-ce
21+
- docker-ce-cli
22+
- containerd.io
23+
- docker-buildx-plugin
24+
- docker-compose-plugin
25+
state: present
26+
27+
- name: Create mirror configuration directory
28+
ansible.builtin.file:
29+
path: "{{ registry_mirror_config_path }}"
30+
state: directory
31+
owner: root
32+
group: root
33+
mode: '0755'
34+
35+
- name: Create mirror script directory
36+
ansible.builtin.file:
37+
path: "{{ registry_mirror_service_path | default(registry_mirror_config_path) }}"
38+
state: directory
39+
owner: root
40+
group: root
41+
mode: '0755'
42+
43+
- name: Copy the service.sh file to the mirror scripts directory
44+
ansible.builtin.copy:
45+
src: "service.sh"
46+
dest: "{{ registry_mirror_service_path | default(registry_mirror_config_path) }}/service.sh"
47+
owner: root
48+
group: root
49+
mode: '0755'
50+
51+
- name: Create mirrors
52+
loop_control:
53+
label: "{{ mirror.registry }}"
54+
loop_var: mirror
55+
loop: "{{ registry_mirrors }}"
56+
ansible.builtin.include_tasks: create-mirror-container.yaml

‎example-hooks/registry-mirrors/post-proxies/create-mirror-container.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
- name: Setting sanitized mirror name fact
2+
ansible.builtin.set_fact:
3+
sanitized_mirror_name: "{{ mirror.registry | regex_replace('[^a-zA-Z0-9\\.]', '-') }}"
4+
5+
- name: Create mirror configuration file
6+
ansible.builtin.template:
7+
src: "mirror-config.yaml.j2"
8+
dest: "{{ mirror.config_path | default(registry_mirror_config_path + '/' + sanitized_mirror_name + '.yaml') }}"
9+
owner: root
10+
group: root
11+
mode: '0644'
12+
13+
- name: Create systemd unit file for mirror registry
14+
ansible.builtin.template:
15+
src: "mirror-systemd.unit.j2"
16+
dest: "/etc/systemd/system/mirror-{{ sanitized_mirror_name }}.service"
17+
owner: root
18+
group: root
19+
mode: '0644'
20+
21+
- name: Reload systemd to pick up new mirror service
22+
ansible.builtin.systemd:
23+
daemon_reload: true
24+
25+
- name: Enable and start mirror registry service
26+
ansible.builtin.systemd:
27+
name: "mirror-{{ sanitized_mirror_name }}.service"
28+
enabled: true
29+
state: started

‎example-hooks/registry-mirrors/post-proxies/files/service.sh‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
usage()
6+
{
7+
echo "Usage: $0
8+
9+
-c | --config Path to the configuration file.
10+
-d | --data-path Path to the data directory.
11+
-g | --gid GID to run the container as.
12+
-i | --image Docker image to use.
13+
-n | --name Name of the container.
14+
-p | --port Port to expose the container on.
15+
-s | --state State of the container (start|stop).
16+
-u | --uid UID to run the container as.
17+
"
18+
exit 1
19+
}
20+
21+
# Store the command line arguments as a variable
22+
PARSED_ARGUMENTS=$(getopt -a -n "$0" -o c:d:g:i:n:p:s:u: --long config:,data-path:,gid:,image:,name:,port:,state:,uid: -- "$@")
23+
VALID_ARGUMENTS=$?
24+
25+
# Make sure some arguments were passed in
26+
if [ "$VALID_ARGUMENTS" != "0" ]
27+
then
28+
usage
29+
fi
30+
31+
eval set -- "$PARSED_ARGUMENTS"
32+
33+
CONFIG="${CONFIG:-}"
34+
PORT="${PORT:-}"
35+
IMAGE="${IMAGE:-}"
36+
MIRROR_UID="${MIRROR_UID:-}"
37+
MIRROR_GID="${MIRROR_GID:-}"
38+
DATA_PATH="${DATA_PATH:-}"
39+
STATE="${STATE:-}"
40+
NAME="${NAME:-}"
41+
42+
while :
43+
do
44+
case "$1" in
45+
-c | --config) CONFIG="$2"; shift 2 ;;
46+
-d | --data-path) DATA_PATH="$2"; shift 2 ;;
47+
-g | --gid) MIRROR_GID="$2"; shift 2 ;;
48+
-i | --image) IMAGE="$2"; shift 2 ;;
49+
-n | --name) NAME="$2"; shift 2 ;;
50+
-p | --port) PORT="$2"; shift 2 ;;
51+
-s | --state) STATE="$2"; shift 2 ;;
52+
-u | --uid) MIRROR_UID="$2"; shift 2 ;;
53+
-h | --help) usage;;
54+
--) shift; break ;;
55+
*) usage;;
56+
esac
57+
done
58+
59+
if [ -z "${PORT}" ] || \
60+
[ -z "${IMAGE}" ] || \
61+
[ -z "${MIRROR_UID}" ] || \
62+
[ -z "${MIRROR_GID}" ] || \
63+
[ -z "${DATA_PATH}" ] || \
64+
[ -z "${CONFIG}" ] || \
65+
[ -z "${STATE}" ] || \
66+
[ -z "${NAME}" ]
67+
then
68+
usage
69+
fi
70+
71+
if ! [[ "${MIRROR_UID}" =~ ^[0-9]+$ ]]
72+
then
73+
echo "UID must be a number"
74+
exit 1
75+
fi
76+
77+
if ! [[ "${MIRROR_GID}" =~ ^[0-9]+$ ]]
78+
then
79+
echo "GID must be a number"
80+
exit 1
81+
fi
82+
83+
if ! [[ "${PORT}" =~ ^[0-9]+$ ]]
84+
then
85+
echo "Port must be a number"
86+
exit 1
87+
fi
88+
89+
if [ -f "${CONFIG}" ]
90+
then
91+
echo "The config file specified with --config does not exist: ${CONFIG}"
92+
exit 1
93+
fi
94+
95+
if [ ! -d "${DATA_PATH}" ]
96+
then
97+
echo "Data path ${DATA_PATH} does not exist"
98+
mkdir -p "${DATA_PATH}"
99+
echo "Created data path ${DATA_PATH}"
100+
echo "Setting permissions on data path ${DATA_PATH} to ${MIRROR_UID}:${MIRROR_GID}"
101+
chown -R "${MIRROR_UID}:${MIRROR_GID}" "${DATA_PATH}"
102+
fi
103+
104+
if [ "${STATE}" == "start" ]
105+
then
106+
/usr/bin/docker stop "${NAME}" || true
107+
/usr/bin/docker rm "${NAME}" || true
108+
echo "******************* Starting mirror container ${NAME} on port ${PORT} *******************"
109+
110+
docker run -d --name "${NAME}" \
111+
-e OTEL_TRACES_EXPORTER=none \
112+
-v "${DATA_PATH}:/var/lib/registry" \
113+
-v "${CONFIG}:/etc/distribution/config.yml:ro" \
114+
-p "127.0.0.1:${PORT}:5000" \
115+
--user "${MIRROR_UID}:${MIRROR_GID}" \
116+
--read-only \
117+
"${IMAGE}"
118+
119+
echo "******************* Waiting for mirror container ${NAME} on port ${PORT} to start *******************"
120+
# wait for the container to be ready
121+
until curl "http://127.0.0.1:${PORT}/v2/" -f --silent --max-time 2 >/dev/null 2>&1
122+
do
123+
sleep 1
124+
done
125+
echo "******************* Started mirror container ${NAME} on port ${PORT} *******************"
126+
127+
systemd-notify --ready --status="Mirror ${NAME} is running on port ${PORT}"
128+
129+
# Start a background process to send watchdog notifications as long as we can curl the container
130+
bash -c "
131+
while true
132+
do
133+
if curl \"http://localhost:${PORT}/v2/\" -f --silent --max-time 2 >/dev/null 2>&1
134+
then
135+
systemd-notify WATCHDOG=1 || echo \"Failed to send watchdog notification for mirror ${NAME}\"
136+
else
137+
echo \"Mirror ${NAME} on ${PORT} is not responding\"
138+
fi
139+
sleep 10
140+
done
141+
" &
142+
HEALTHCHECK_PID=$!
143+
144+
# Trap to clean up background health check on exit
145+
trap 'kill ${HEALTHCHECK_PID} 2>/dev/null || true' EXIT TERM INT
146+
147+
docker logs -f --since 30s "${NAME}" # This should run for the lifetime of the container and send the logs to journald
148+
elif [ "${STATE}" == "stop" ]
149+
then
150+
docker stop "${NAME}" || true
151+
docker rm "${NAME}" || true
152+
else
153+
echo "Invalid state: ${STATE}"
154+
exit 1
155+
fi

‎example-hooks/registry-mirrors/post-proxies/templates/haproxy.cfg.j2‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
global
2+
log /dev/log local0
3+
log /dev/log local1 notice
4+
stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
5+
user haproxy
6+
group haproxy
7+
daemon
8+
stats timeout 30s
9+
nbthread 4
10+
11+
# Default SSL material locations
12+
ca-base /etc/ssl/certs
13+
crt-base /etc/ssl/private
14+
15+
# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
16+
ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
17+
ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
18+
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
19+
20+
listen stats
21+
bind :1936
22+
mode http
23+
log global
24+
stats enable
25+
stats refresh 30s
26+
stats show-node
27+
stats uri /haproxy?stats
28+
http-request use-service prometheus-exporter if { path /metrics }
29+
timeout connect 5000
30+
timeout client 50000
31+
timeout server 50000
32+
33+
defaults
34+
log global
35+
mode http
36+
option dontlognull
37+
38+
listen controlplane-in
39+
bind {{ kubernetes_proxy_bind_address }}:{{ kubernetes_proxy_port }}
40+
mode tcp
41+
timeout client 5000
42+
timeout connect 5000
43+
timeout server 5000
44+
option httpchk
45+
http-check send meth GET uri /readyz
46+
{% for host in groups['control_planes'] %}
47+
server {{ hostvars[host]['ansible_host'] }} {{ hostvars[host]['kubernetes_api_server_bind_address'] | default(query('community.dns.lookup', hostvars[host]['ansible_host'])[0]) }}:{{ hostvars[host]['kubernetes_api_server_port'] }} check check-ssl inter {{ kubernetes_control_plane_check_interval | default('250ms') }} verify none
48+
{% endfor %}
49+
50+
listen registry-router
51+
bind :{{ registry_mirror_port | default(5000) }}
52+
mode http
53+
54+
{% for mirror in registry_mirrors %}
55+
acl ns_{{ mirror.registry | regex_replace('[^a-zA-Z0-9]', '-') }} urlp(ns) -m str {{ mirror.registry }}
56+
{% endfor %}
57+
58+
# Reject requests without valid ns parameter
59+
http-request deny deny_status 400 unless {% for mirror in registry_mirrors %}ns_{{ mirror.registry | regex_replace('[^a-zA-Z0-9]', '-') }}{% if not loop.last %} or {% endif %}{% endfor %}
60+
61+
{% for mirror in registry_mirrors %}
62+
server {{ mirror.registry | regex_replace('[^a-zA-Z0-9]', '-') }} 127.0.0.1:{{ mirror.port }} check port {{ mirror.port }}
63+
{% endfor %}
64+
65+
{% for mirror in registry_mirrors %}
66+
use-server {{ mirror.registry | regex_replace('[^a-zA-Z0-9]', '-') }} if ns_{{ mirror.registry | regex_replace('[^a-zA-Z0-9]', '-') }}
67+
{% endfor %}
68+
69+
timeout connect 5000
70+
timeout client 50000
71+
timeout server 50000

‎example-hooks/registry-mirrors/post-proxies/templates/mirror-config.yaml.j2‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
version: 0.1
2+
log:
3+
fields:
4+
service: registry
5+
6+
storage:
7+
filesystem:
8+
rootdirectory: /var/lib/registry
9+
10+
http:
11+
addr: :5000
12+
headers:
13+
X-Content-Type-Options: [nosniff]
14+
15+
delete:
16+
enabled: true
17+
18+
proxy:
19+
remoteurl: "{{ mirror.remote_url | default('https://' + mirror.registry) }}"
20+
{% if mirror.username is defined and mirror.password is defined %}
21+
username: {{ mirror.username | to_yaml }}
22+
password: {{ mirror.password | to_yaml }}
23+
{% endif %}
24+
ttl: {{ mirror.ttl | default('336h0m0s') }}

‎example-hooks/registry-mirrors/post-proxies/templates/mirror-systemd.unit.j2‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
[Unit]
2+
Description=Distribution registry
3+
After=docker.service
4+
Requires=docker.service
5+
6+
[Service]
7+
NotifyAccess=all
8+
Type=notify
9+
WatchdogSec=30s
10+
Environment="CONFIG={{ mirror.config_path | default(registry_mirror_config_path + '/' + sanitized_mirror_name + '.yaml') }}"
11+
Environment="DATA_PATH={{ mirror.data_path }}"
12+
Environment="IMAGE={{ mirror.image | default('registry:3') }}"
13+
Environment="MIRROR_GID={{ mirror.group_id | default(1000) }}"
14+
Environment="MIRROR_UID={{ mirror.user_id | default(1000) }}"
15+
Environment="PORT={{ mirror.port }}"
16+
Environment="NAME=mirror-{{ sanitized_mirror_name }}"
17+
Environment="OTEL_TRACES_EXPORTER=none"
18+
Restart=always
19+
ExecStart={{ registry_mirror_service_path | default(registry_mirror_config_path) }}/service.sh --state "start"
20+
ExecStop={{ registry_mirror_service_path | default(registry_mirror_config_path) }}/service.sh --state "stop"
21+
22+
[Install]
23+
WantedBy=multi-user.target

0 commit comments

Comments
 (0)