Skip to content

Commit 329e8b1

Browse files
iakatiakat
committed
Add containers
1 parent ad9703c commit 329e8b1

21 files changed

+2976
-10
lines changed

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
containers/data/

‎Gemfile‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,20 @@
11
source "https://rubygems.org"
22

3-
gem 'analysand', '~> 3.0.2', git: 'https://github.com/yipdw/analysand.git'
3+
gem 'analysand', '~> 3.1.0'
44
gem 'addressable'
55
gem 'cinch', '~> 2.2.0'
66
gem 'celluloid'
77
gem 'celluloid-redis'
88
gem 'listen', '~> 2.0'
9-
gem 'net-http-persistent'
9+
gem 'net-http-persistent', '~> 2.9'
1010

1111
# Psych 2.0.0 as shipped with Ruby 2.0 doesn't include Psych.safe_load
1212
gem 'psych', '~> 2.0', '>= 2.0.1'
1313

14-
gem 'redis'
15-
gem 'hiredis'
14+
15+
gem "redis", '~> 3.0', :require => ['redis', 'redis/connection/hiredis']
16+
gem 'hiredis', '~> 0.5'
17+
gem 'hiredis-client'
1618
gem 'trollop'
1719
gem 'uuidtools'
1820
gem 'twitter', '~> 5.5.1'
@@ -31,8 +33,8 @@ end
3133

3234
group :dashboard do
3335
gem 'json'
34-
gem 'reel'
35-
gem 'webmachine', :git => 'https://github.com/seancribbs/webmachine-ruby.git'
36+
gem 'reel', '~> 0.4.0'
37+
gem 'webmachine', '~> 1.2.2'
3638
gem 'webmachine-sprockets', :git => 'https://github.com/ArchiveTeam/webmachine-sprockets.git'
3739
gem 'erubis'
3840
end

‎containers/.env.example‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
COUCHDB_USER="admin"
2+
COUCHDB_PASSWORD="password"
3+
REDIS_PASSWORD="password"
4+
ARCHIVEBOT_COUCHDB_URL="http://$COUCHDB_USER:$COUCHDB_PASSWORD@couchdb:5984/archivebot"
5+
ARCHIVEBOT_IRC_URL="ircs://irc.hackint.org:6697"
6+
ARCHIVEBOT_REDIS_URL="redis://:$REDIS_PASSWORD@valkey:6379/0?password=$REDIS_PASSWORD"
7+
ARCHIVEBOT_PIPE_REDIS_URL="redis://:$REDIS_PASSWORD@autossh:6379/0?password=$REDIS_PASSWORD"
8+
ARCHIVEBOT_ZEROMQ_URL="tcp://updates-listener:12345"
9+
ARCHIVEBOT_ZEROMQ_BIND_URL="tcp://0.0.0.0:12345"
10+
ARCHIVEBOT_DASHBOARD_URL="http://0.0.0.0:4567"
11+
ARCHIVEBOT_IRC_CHANNEL="#notarchivebot"
12+
ARCHIVEBOT_IRC_NICK="notarchivebot"
13+
ARCHIVEBOT_PIPE_AUTOSSH_TARGET="[email protected] -p 922"
14+
ARCHIVEBOT_PIPE_NAME="pipe1"

‎containers/backend.Dockerfile‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
FROM debian:bullseye-slim
2+
ENV LC_ALL=C
3+
RUN apt-get update && \
4+
DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io install \
5+
tini curl sudo gnupg ca-certificates apt-utils build-essential ruby ruby-dev bundler python3 python3-websockets git libzmq5 libzmq3-dev libssl-dev && \
6+
echo 'deb http://deb.debian.org/debian bullseye-backports main' >/etc/apt/sources.list.d/backports.list && \
7+
apt-get update && \
8+
DEBIAN_FRONTEND=noninteractive DEBIAN_PRIORITY=critical apt-get -qqy --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold -o Dpkg::Options::=--force-unsafe-io -t bullseye-backports install zstd && \
9+
git clone https://gitea.arpa.li/JustAnotherArchivist/little-things /tmp/JAAs-little-things && \
10+
cd /tmp/JAAs-little-things && \
11+
chmod +x /tmp/JAAs-little-things/* && \
12+
mv /tmp/JAAs-little-things/* /usr/local/bin/ && \
13+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
14+
15+
WORKDIR /home/archivebot/ArchiveBot
16+
17+
COPY Gemfile /home/archivebot/ArchiveBot/Gemfile
18+
COPY plumbing/Gemfile /home/archivebot/ArchiveBot/plumbing/Gemfile
19+
ENV GEM_HOME=/home/archivebot/.gems
20+
RUN bundle install && \
21+
cd plumbing && \
22+
bundle install && \
23+
gem install bundler -v 1.15.1
24+
COPY . /home/archivebot/ArchiveBot
25+
RUN rm /home/archivebot/ArchiveBot/Gemfile.lock && \
26+
rm /home/archivebot/ArchiveBot/plumbing/Gemfile.lock
27+
RUN cd /home/archivebot/ArchiveBot/ && \
28+
bundle install --path /home/archivebot/.gems
29+
30+
RUN groupadd -r archivebot && useradd -r -m -g archivebot archivebot && \
31+
chown -R archivebot:archivebot /home/archivebot/ &\
32+
chmod -R 0755 /home/archivebot/ &\
33+
wait
34+
# USER archivebot
35+
WORKDIR /home/archivebot/ArchiveBot
36+
ENV PATH="/home/archivebot/.gems/ruby/2.7.0/bin:${PATH}" \
37+
PYTHONUNBUFFERED=1
38+
ENTRYPOINT ["/usr/bin/tini", "--", "/home/archivebot/ArchiveBot/entrypoint.sh"]
39+
CMD ["help"]

‎containers/backend.docker-compose.yaml‎

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
# deploying an archivebot...
2+
version: '3.8'
3+
networks:
4+
# publicly accessible network
5+
frontend:
6+
name: archivebot-frontend
7+
# stuff that needs to talk to zeromq.
8+
zeromq:
9+
name: archivebot-zeromq
10+
# redis, couchdb, and the backend services
11+
redis:
12+
name: archivebot-redis
13+
couchdb:
14+
name: archivebot-couchdb
15+
services:
16+
valkey:
17+
build:
18+
context: ..
19+
dockerfile: containers/valkey.Dockerfile
20+
volumes:
21+
- ./data/backend/valkey:/var/lib/valkey
22+
environment:
23+
- VALKEY_PASSWORD=${REDIS_PASSWORD}
24+
networks:
25+
- redis
26+
deploy:
27+
resources:
28+
limits:
29+
cpus: '1'
30+
memory: 1024M
31+
reservations:
32+
cpus: '0.05'
33+
memory: 64M
34+
couchdb:
35+
build:
36+
context: ..
37+
dockerfile: containers/couchdb.Dockerfile
38+
volumes:
39+
- ./data/backend/couchdb:/opt/couchdb/data
40+
environment:
41+
- COUCHDB_USER=${COUCHDB_USER}
42+
- COUCHDB_PASSWORD=${COUCHDB_PASSWORD}
43+
networks:
44+
- couchdb
45+
deploy:
46+
resources:
47+
limits:
48+
cpus: '1'
49+
memory: 1024M
50+
reservations:
51+
cpus: '0.05'
52+
memory: 64M
53+
ircbot:
54+
build:
55+
context: ..
56+
dockerfile: containers/backend.Dockerfile
57+
environment:
58+
- ARCHIVEBOT_IRC_URL=${ARCHIVEBOT_IRC_URL}
59+
- ARCHIVEBOT_IRC_NICK=${ARCHIVEBOT_IRC_NICK}
60+
- ARCHIVEBOT_IRC_CHANNEL=${ARCHIVEBOT_IRC_CHANNEL}
61+
- ARCHIVEBOT_IRC_PASSWORD=${ARCHIVEBOT_IRC_PASSWORD}
62+
- ARCHIVEBOT_COUCHDB_URL=${ARCHIVEBOT_COUCHDB_URL}
63+
- ARCHIVEBOT_REDIS_URL=${ARCHIVEBOT_REDIS_URL}
64+
command: ["ircbot"]
65+
networks:
66+
- couchdb
67+
- redis
68+
- zeromq
69+
deploy:
70+
resources:
71+
limits:
72+
cpus: '1'
73+
memory: 1024M
74+
reservations:
75+
cpus: '0.05'
76+
memory: 64M
77+
updates-listener:
78+
build:
79+
context: ..
80+
dockerfile: containers/backend.Dockerfile
81+
command: ["updates-listener"]
82+
environment:
83+
- ARCHIVEBOT_ZEROMQ_BIND_URL=${ARCHIVEBOT_ZEROMQ_BIND_URL}
84+
- ARCHIVEBOT_REDIS_URL=${ARCHIVEBOT_REDIS_URL}
85+
- ARCHIVEBOT_COUCHDB_URL=${ARCHIVEBOT_COUCHDB_URL}
86+
networks:
87+
- couchdb
88+
- zeromq
89+
- redis
90+
deploy:
91+
resources:
92+
limits:
93+
cpus: '1'
94+
memory: 1024M
95+
reservations:
96+
cpus: '0.05'
97+
memory: 64M
98+
dashboard:
99+
build:
100+
context: ..
101+
dockerfile: containers/backend.Dockerfile
102+
command: ["dashboard"]
103+
environment:
104+
- ARCHIVEBOT_DASHBOARD_URL=${ARCHIVEBOT_DASHBOARD_URL}
105+
- ARCHIVEBOT_REDIS_URL=${ARCHIVEBOT_REDIS_URL}
106+
- ARCHIVEBOT_COUCHDB_URL=${ARCHIVEBOT_COUCHDB_URL}
107+
networks:
108+
- couchdb
109+
- frontend
110+
- zeromq
111+
- redis
112+
ports:
113+
- "4567:4567"
114+
deploy:
115+
resources:
116+
limits:
117+
cpus: '1'
118+
memory: 1024M
119+
reservations:
120+
cpus: '0.05'
121+
memory: 64M
122+
websocket:
123+
image: ghcr.io/iakat/archivebot-dashboard-repeater
124+
environment:
125+
- UPSTREAM=${ARCHIVEBOT_ZEROMQ_URL}
126+
ports:
127+
- "4568:4568"
128+
networks:
129+
- zeromq
130+
- frontend
131+
deploy:
132+
resources:
133+
limits:
134+
cpus: '1'
135+
memory: 1024M
136+
reservations:
137+
cpus: '0.05'
138+
memory: 64M
139+
cogs:
140+
build:
141+
context: ..
142+
dockerfile: containers/backend.Dockerfile
143+
command: ["cogs"]
144+
environment:
145+
- ARCHIVEBOT_REDIS_URL=${ARCHIVEBOT_REDIS_URL}
146+
- ARCHIVEBOT_COUCHDB_URL=${ARCHIVEBOT_COUCHDB_URL}
147+
networks:
148+
- redis
149+
- couchdb
150+
deploy:
151+
resources:
152+
limits:
153+
cpus: '1'
154+
memory: 1024M
155+
reservations:
156+
cpus: '0.05'
157+
memory: 64M
158+
analyzer:
159+
build:
160+
context: ..
161+
dockerfile: containers/backend.Dockerfile
162+
command: ["analyzer"]
163+
environment:
164+
- ARCHIVEBOT_REDIS_URL=${ARCHIVEBOT_REDIS_URL}
165+
- ARCHIVEBOT_COUCHDB_URL=${ARCHIVEBOT_COUCHDB_URL}
166+
networks:
167+
- redis
168+
- couchdb
169+
deploy:
170+
resources:
171+
limits:
172+
cpus: '1'
173+
memory: 1024M
174+
reservations:
175+
cpus: '0.05'
176+
memory: 64M
177+
trimmer:
178+
build:
179+
context: ..
180+
dockerfile: containers/backend.Dockerfile
181+
command: ["trimmer"]
182+
environment:
183+
- ARCHIVEBOT_REDIS_URL=${ARCHIVEBOT_REDIS_URL}
184+
- ARCHIVEBOT_COUCHDB_URL=${ARCHIVEBOT_COUCHDB_URL}
185+
networks:
186+
- redis
187+
- couchdb
188+
deploy:
189+
resources:
190+
limits:
191+
cpus: '1'
192+
memory: 1024M
193+
reservations:
194+
cpus: '0.05'
195+
memory: 64M
196+
openssh:
197+
networks:
198+
- redis
199+
build:
200+
context: ..
201+
dockerfile: containers/openssh.Dockerfile
202+
ports:
203+
- "922:22"
204+
command: ["openssh"]
205+
deploy:
206+
resources:
207+
limits:
208+
cpus: '0.1'
209+
memory: 64M
210+
reservations:
211+
cpus: '0.01'
212+
memory: 16M
213+
volumes:
214+
- ./data/backend/openssh:/home/pipeline/.ssh

‎containers/couchdb.Dockerfile‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
FROM couchdb
2+
EXPOSE 5984
3+
4+
# we init it: start it in bg, wait for it to be ready, then create the db, and some items.
5+
COPY db/design_docs /design_docs
6+
# start couchdb in the background
7+
RUN set -ex && \
8+
echo """#!/usr/bin/env bash \n\
9+
set -ex \n\
10+
COUCHDB=http://\$COUCHDB_USER:\$[email protected]:5984 \n\
11+
/docker-entrypoint.sh \$@ & \n\
12+
sleep 5 \n\
13+
while [ \$(curl -s -o /dev/null -w \"%{http_code}\" \$COUCHDB/_all_dbs) -ne 200 ]; do \n\
14+
sleep 1 \n\
15+
done \n\
16+
# check if database exists, if not create it \n\
17+
18+
if [ \$(curl -s -o /dev/null -w \"%{http_code}\" \$COUCHDB/archivebot) -ne 200 ]; then \n\
19+
cd /design_docs \n\
20+
grep -v _rev archive_urls.json > /tmp/archive_urls.json \n\
21+
grep -v _rev ignore_patterns.json > /tmp/ignore_patterns.json \n\
22+
grep -v _rev jobs.json > /tmp/jobs.json \n\
23+
grep -v _rev user_agents.json > /tmp/user_agents.json \n\
24+
curl -X PUT \$COUCHDB/_users \n\
25+
curl -X PUT \$COUCHDB/_replicator \n\
26+
curl -X PUT \$COUCHDB/_global_changes \n\
27+
curl -X PUT \$COUCHDB/archivebot \n\
28+
curl -X PUT \$COUCHDB/archivebot_logs \n\
29+
curl -X PUT \$COUCHDB/archivebot/_design/archive_urls -d @/tmp/archive_urls.json \n\
30+
curl -X PUT \$COUCHDB/archivebot/_design/ignore_patterns -d @/tmp/ignore_patterns.json \n\
31+
curl -X PUT \$COUCHDB/archivebot/_design/jobs -d @/tmp/jobs.json \n\
32+
curl -X PUT \$COUCHDB/archivebot/_design/user_agents -d @/tmp/user_agents.json\n\
33+
touch /_archivebot_done_db \n\
34+
fi \n\
35+
sync \n\
36+
wait \n\
37+
""" > /_after_entrypoint.sh && \
38+
chmod +x /_after_entrypoint.sh && \
39+
cat /_after_entrypoint.sh && \
40+
[ -f /docker-entrypoint.sh ] && [ -f /_after_entrypoint.sh ] || exit 1
41+
42+
# RUN COUCHDB_USER=admin COUCHDB_PASSWORD=password /docker-entrypoint.sh /_after_entrypoint.sh "/opt/couchdb/bin/couchdb" & \
43+
# # when /_archivebot_done_db exists, we know the db is ready, kill ir
44+
# while [ ! -f /_archivebot_done_db ]; do sleep 1; done && \
45+
# kill $(pgrep -f "/opt/couchdb/bin/couchdb") && \
46+
# rm /_archivebot_done_db
47+
48+
ENTRYPOINT ["/usr/bin/tini", "--", "/docker-entrypoint.sh", "/_after_entrypoint.sh"]
49+
CMD ["/opt/couchdb/bin/couchdb"]

‎containers/openssh.Dockerfile‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
FROM alpine:3.20
2+
RUN apk add --no-cache --virtual=.run-deps \
3+
openssh tini autossh bash inotify-tools curl ca-certificates && \
4+
addgroup pipeline && \
5+
adduser -D -G pipeline pipeline -s /bin/false && \
6+
mkdir -p /home/pipeline/.ssh && \
7+
chown pipeline:pipeline /home/pipeline/.ssh && \
8+
passwd -u pipeline && \
9+
# Add matchgroup pipeline to /etc/ssh/sshd_config
10+
# Only allow port valkey:6379 to be forwarded
11+
cat <<EOF >>/etc/ssh/sshd_config
12+
Match Group pipeline
13+
PasswordAuthentication no
14+
AllowTcpForwarding yes
15+
X11Forwarding no
16+
PermitTunnel no
17+
GatewayPorts no
18+
AllowStreamLocalForwarding no
19+
AllowAgentForwarding no
20+
PermitOpen valkey:6379
21+
ForceCommand echo 'This account can only be used for port forwarding'
22+
AuthorizedKeysFile /home/%u/.ssh/authorized_keys
23+
EOF
24+
COPY containers/openssh.entrypoint.sh /_ssh_entrypoint.sh
25+
ENTRYPOINT ["/sbin/tini", "--", "/_ssh_entrypoint.sh"]
26+
VOLUME /etc/ssh/sshd_config.d
27+
EXPOSE 22

0 commit comments

Comments
 (0)