Skip to content

Commit 8e258e6

Browse files
tsibleytsibley
committed
Merge branch 'trs/aws-batch-overlays'
2 parents 7061a9e + 9b6592f commit 8e258e6

9 files changed

+307
-3
lines changed

Diff for: ‎Dockerfile

+8-2
Original file line numberDiff line numberDiff line change
@@ -417,13 +417,16 @@ COPY --from=builder-build-platform --chown=nextstrain:nextstrain /nextstrain /n
417417
COPY --from=builder-target-platform --chown=nextstrain:nextstrain /nextstrain /nextstrain
418418

419419
# Add our entrypoints and helpers
420-
COPY entrypoint entrypoint-aws-batch drop-privs create-envd delete-envd /sbin/
421-
RUN chmod a+rx /sbin/entrypoint* /sbin/drop-privs /sbin/{create,delete}-envd
420+
COPY entrypoint entrypoint-aws-batch drop-privs create-envd delete-envd chdir-workdir /sbin/
421+
RUN chmod a+rx /sbin/entrypoint* /sbin/drop-privs /sbin/{create,delete}-envd /sbin/chdir-workdir
422422

423423
# Make /nextstrain a global HOME, writable by any UID (like /tmp)
424424
RUN chmod a+rwXt /nextstrain
425425
ENV HOME=/nextstrain
426426

427+
# Run the final setup as our target user for permissions reasons.
428+
USER nextstrain:nextstrain
429+
427430
# No nesting of runtimes, please. Use the ambient runtime inside this runtime.
428431
ENV NEXTSTRAIN_HOME=/nextstrain
429432
RUN nextstrain check-setup --set-default ambient \
@@ -435,6 +438,9 @@ RUN nextstrain check-setup --set-default ambient \
435438
WORKDIR /nextstrain/build
436439
RUN chown nextstrain:nextstrain /nextstrain/build
437440

441+
# Switch back to root. The entrypoint will drop to nextstrain:nextstrain as
442+
# necessary when a container starts.
443+
USER root
438444
ENTRYPOINT ["/sbin/entrypoint"]
439445

440446
# Finally, add metadata at the end so it doesn't bust cached layers.

Diff for: ‎chdir-workdir

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
if [[ -n "${NEXTSTRAIN_WORKDIR:-}" ]]; then
5+
mkdir --parents "$NEXTSTRAIN_WORKDIR"
6+
cd "$NEXTSTRAIN_WORKDIR"
7+
fi
8+
9+
exec "$@"

Diff for: ‎entrypoint

+1
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,6 @@ else
2323
create-envd \
2424
envdir /nextstrain/env.d \
2525
delete-envd \
26+
chdir-workdir \
2627
"$@"
2728
fi

Diff for: ‎entrypoint-aws-batch

+13-1
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,24 @@
22
set -euo pipefail
33

44
# Show what we're running for the benefit of the logs.
5-
set -x
5+
if [[ "${NEXTSTRAIN_AWS_BATCH_VERBOSE:=1}" != 0 ]]; then
6+
set -x
7+
fi
68

79
# Download the working dir.
810
case "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" in
911
s3://*.zip)
1012
aws s3 cp --no-progress "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" "$PWD.zip"
13+
14+
for dir in /nextstrain/{augur,auspice,fauna}; do
15+
relative_dir="$(realpath "$dir" --relative-to="$PWD")"/
16+
17+
if zipinfo -1 "$PWD.zip" "$relative_dir" &>/dev/null; then
18+
echo "removing $dir because workdir ZIP contains $relative_dir overlay"
19+
rm -rf "$dir"
20+
fi
21+
done
22+
1123
unzip -: -o "$PWD.zip"
1224
;;
1325
s3://*)

Diff for: ‎tests/aws-batch-verbose.t

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env cram
2+
3+
Setup.
4+
5+
$ : "${IMAGE:=localhost:5000/nextstrain/base:latest}"
6+
$ (docker image inspect "$IMAGE" || docker image pull "$IMAGE") &>/dev/null
7+
8+
A workdir URL is required and not setting one here causes the entrypoint to
9+
error, but that's enough for testing verbose mode.
10+
11+
$ export NEXTSTRAIN_AWS_BATCH_WORKDIR_URL=
12+
13+
Verbose mode is default.
14+
15+
$ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_WORKDIR_URL "$IMAGE" \
16+
> /sbin/entrypoint-aws-batch true
17+
+ case "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" in
18+
+ echo 'entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>'
19+
entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>
20+
+ exit 1
21+
[1]
22+
23+
Verbose mode is anything not zero.
24+
25+
$ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE=yes} "$IMAGE" \
26+
> /sbin/entrypoint-aws-batch true
27+
+ case "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" in
28+
+ echo 'entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>'
29+
entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>
30+
+ exit 1
31+
[1]
32+
33+
$ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE=} "$IMAGE" \
34+
> /sbin/entrypoint-aws-batch true
35+
+ case "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL" in
36+
+ echo 'entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>'
37+
entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>
38+
+ exit 1
39+
[1]
40+
41+
Verbose mode can be turned off.
42+
43+
$ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE=0} "$IMAGE" \
44+
> /sbin/entrypoint-aws-batch true
45+
entrypoint-aws-batch: No handler for NEXTSTRAIN_AWS_BATCH_WORKDIR_URL <>
46+
[1]

Diff for: ‎tests/aws-batch-workdir-url.t

+192
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
#!/usr/bin/env cram
2+
3+
Setup.
4+
5+
$ [[ -n "$AWS_ACCESS_KEY_ID" && -n "$AWS_SECRET_ACCESS_KEY" ]] || exit 80
6+
7+
$ : "${IMAGE:=localhost:5000/nextstrain/base:latest}"
8+
$ (docker image inspect "$IMAGE" || docker image pull "$IMAGE") &>/dev/null
9+
10+
$ export NEXTSTRAIN_AWS_BATCH_VERBOSE=0
11+
12+
Workdir ZIP archive is downloaded and extracted.
13+
14+
$ export NEXTSTRAIN_AWS_BATCH_WORKDIR_URL="s3://nextstrain-tmp/$(python3 -c 'import uuid; print(uuid.uuid4())').zip"
15+
16+
$ aws s3 cp --quiet "$TESTDIR/data/workdir-without-overlays.zip" "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL"
17+
18+
$ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE} --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN} "$IMAGE" \
19+
> /sbin/entrypoint-aws-batch bash -euo pipefail -xc 'ls -l'
20+
download: s3://nextstrain-tmp/*.zip to ../build.zip (glob)
21+
Archive: /nextstrain/build.zip
22+
extracting: reticulating
23+
extracting: splines
24+
+ ls -l
25+
total 0
26+
-rw-rw-r-- 1 nextstrain nextstrain 0 Mar 10 21:46 reticulating
27+
-rw-rw-r-- 1 nextstrain nextstrain 0 Mar 10 21:46 splines
28+
upload: ../build.zip to s3://nextstrain-tmp/*.zip (glob)
29+
30+
/nextstrain/{augur,auspice} are removed when the workdir ZIP contains overlays.
31+
32+
$ export NEXTSTRAIN_AWS_BATCH_WORKDIR_URL="s3://nextstrain-tmp/$(python3 -c 'import uuid; print(uuid.uuid4())').zip"
33+
34+
$ aws s3 cp --quiet "$TESTDIR/data/workdir-with-augur-auspice-overlays.zip" "$NEXTSTRAIN_AWS_BATCH_WORKDIR_URL"
35+
36+
$ docker run --rm --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE} --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN} "$IMAGE" \
37+
> /sbin/entrypoint-aws-batch bash -euo pipefail -xc 'ls -lR . ../augur ../auspice'
38+
download: s3://nextstrain-tmp/*.zip to ../build.zip (glob)
39+
removing /nextstrain/augur because workdir ZIP contains ../augur/ overlay
40+
removing /nextstrain/auspice because workdir ZIP contains ../auspice/ overlay
41+
Archive: /nextstrain/build.zip
42+
extracting: reticulating
43+
extracting: splines
44+
creating: ../augur/
45+
creating: ../augur/a/
46+
creating: ../augur/a/b/
47+
creating: ../augur/a/b/c/
48+
extracting: ../augur/a/b/c/world.txt
49+
extracting: ../augur/a/b/c/hello.txt
50+
creating: ../augur/augur/
51+
extracting: ../augur/augur/__init__.py
52+
extracting: ../augur/README.md
53+
creating: ../auspice/
54+
+ ls -lR . ../augur ../auspice
55+
.:
56+
total 0
57+
-rw-rw-r-- 1 nextstrain nextstrain 0 Mar 10 21:46 reticulating
58+
-rw-rw-r-- 1 nextstrain nextstrain 0 Mar 10 21:46 splines
59+
60+
../augur:
61+
total 12
62+
-rw-rw-r-- 1 nextstrain nextstrain 22 Mar 10 21:45 README.md
63+
drwxrwxr-x 3 nextstrain nextstrain 4096 Mar 10 21:32 a
64+
drwxrwxr-x 2 nextstrain nextstrain 4096 Mar 10 21:46 augur
65+
66+
../augur/a:
67+
total 4
68+
drwxrwxr-x 3 nextstrain nextstrain 4096 Mar 10 21:32 b
69+
70+
../augur/a/b:
71+
total 4
72+
drwxrwxr-x 2 nextstrain nextstrain 4096 Mar 10 21:33 c
73+
74+
../augur/a/b/c:
75+
total 8
76+
-rw-rw-r-- 1 nextstrain nextstrain 6 Mar 10 21:32 hello.txt
77+
-rw-rw-r-- 1 nextstrain nextstrain 6 Mar 10 21:32 world.txt
78+
79+
../augur/augur:
80+
total 4
81+
-rw-rw-r-- 1 nextstrain nextstrain 34 Mar 10 21:46 __init__.py
82+
83+
../auspice:
84+
total 0
85+
upload: ../build.zip to s3://nextstrain-tmp/*.zip (glob)
86+
87+
…even when the workdir is not /nextstrain/build, e.g. when it's a sibling.
88+
89+
$ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/abc --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE} --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN} "$IMAGE" \
90+
> /sbin/entrypoint-aws-batch bash -euo pipefail -xc 'ls -lR . ../augur ../auspice'
91+
download: s3://nextstrain-tmp/*.zip to ../abc.zip (glob)
92+
removing /nextstrain/augur because workdir ZIP contains ../augur/ overlay
93+
removing /nextstrain/auspice because workdir ZIP contains ../auspice/ overlay
94+
Archive: /nextstrain/abc.zip
95+
extracting: reticulating
96+
extracting: splines
97+
creating: ../augur/
98+
creating: ../augur/a/
99+
creating: ../augur/a/b/
100+
creating: ../augur/a/b/c/
101+
extracting: ../augur/a/b/c/world.txt
102+
extracting: ../augur/a/b/c/hello.txt
103+
creating: ../augur/augur/
104+
extracting: ../augur/augur/__init__.py
105+
extracting: ../augur/README.md
106+
creating: ../auspice/
107+
+ ls -lR . ../augur ../auspice
108+
.:
109+
total 0
110+
-rw-rw-r-- 1 nextstrain nextstrain 0 Mar 10 21:46 reticulating
111+
-rw-rw-r-- 1 nextstrain nextstrain 0 Mar 10 21:46 splines
112+
113+
../augur:
114+
total 12
115+
-rw-rw-r-- 1 nextstrain nextstrain 22 Mar 10 21:45 README.md
116+
drwxrwxr-x 3 nextstrain nextstrain 4096 Mar 10 21:32 a
117+
drwxrwxr-x 2 nextstrain nextstrain 4096 Mar 10 21:46 augur
118+
119+
../augur/a:
120+
total 4
121+
drwxrwxr-x 3 nextstrain nextstrain 4096 Mar 10 21:32 b
122+
123+
../augur/a/b:
124+
total 4
125+
drwxrwxr-x 2 nextstrain nextstrain 4096 Mar 10 21:33 c
126+
127+
../augur/a/b/c:
128+
total 8
129+
-rw-rw-r-- 1 nextstrain nextstrain 6 Mar 10 21:32 hello.txt
130+
-rw-rw-r-- 1 nextstrain nextstrain 6 Mar 10 21:32 world.txt
131+
132+
../augur/augur:
133+
total 4
134+
-rw-rw-r-- 1 nextstrain nextstrain 34 Mar 10 21:46 __init__.py
135+
136+
../auspice:
137+
total 0
138+
upload: ../abc.zip to s3://nextstrain-tmp/*.zip (glob)
139+
140+
…but not when the workdir is somewhere completely different.
141+
142+
$ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/x/y/z --env=NEXTSTRAIN_AWS_BATCH_{WORKDIR_URL,VERBOSE} --env=AWS_{ACCESS_KEY_ID,SECRET_ACCESS_KEY,SESSION_TOKEN} "$IMAGE" \
143+
> /sbin/entrypoint-aws-batch bash -euo pipefail -xc 'ls -lR . ../augur ../auspice; realpath ../augur ../auspice'
144+
download: s3://nextstrain-tmp/*.zip to ../z.zip (glob)
145+
Archive: /nextstrain/x/y/z.zip
146+
extracting: reticulating
147+
extracting: splines
148+
creating: ../augur/
149+
creating: ../augur/a/
150+
creating: ../augur/a/b/
151+
creating: ../augur/a/b/c/
152+
extracting: ../augur/a/b/c/world.txt
153+
extracting: ../augur/a/b/c/hello.txt
154+
creating: ../augur/augur/
155+
extracting: ../augur/augur/__init__.py
156+
extracting: ../augur/README.md
157+
creating: ../auspice/
158+
+ ls -lR . ../augur ../auspice
159+
.:
160+
total 0
161+
-rw-rw-r-- 1 nextstrain nextstrain 0 Mar 10 21:46 reticulating
162+
-rw-rw-r-- 1 nextstrain nextstrain 0 Mar 10 21:46 splines
163+
164+
../augur:
165+
total 12
166+
-rw-rw-r-- 1 nextstrain nextstrain 22 Mar 10 21:45 README.md
167+
drwxrwxr-x 3 nextstrain nextstrain 4096 Mar 10 21:32 a
168+
drwxrwxr-x 2 nextstrain nextstrain 4096 Mar 10 21:46 augur
169+
170+
../augur/a:
171+
total 4
172+
drwxrwxr-x 3 nextstrain nextstrain 4096 Mar 10 21:32 b
173+
174+
../augur/a/b:
175+
total 4
176+
drwxrwxr-x 2 nextstrain nextstrain 4096 Mar 10 21:33 c
177+
178+
../augur/a/b/c:
179+
total 8
180+
-rw-rw-r-- 1 nextstrain nextstrain 6 Mar 10 21:32 hello.txt
181+
-rw-rw-r-- 1 nextstrain nextstrain 6 Mar 10 21:32 world.txt
182+
183+
../augur/augur:
184+
total 4
185+
-rw-rw-r-- 1 nextstrain nextstrain 34 Mar 10 21:46 __init__.py
186+
187+
../auspice:
188+
total 0
189+
+ realpath ../augur ../auspice
190+
/nextstrain/x/y/augur
191+
/nextstrain/x/y/auspice
192+
upload: ../z.zip to s3://nextstrain-tmp/*.zip (glob)

Diff for: ‎tests/data/workdir-with-augur-auspice-overlays.zip

1.51 KB
Binary file not shown.

Diff for: ‎tests/data/workdir-without-overlays.zip

292 Bytes
Binary file not shown.

Diff for: ‎tests/workdir.t

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/usr/bin/env cram
2+
3+
Setup.
4+
5+
$ : "${IMAGE:=localhost:5000/nextstrain/base:latest}"
6+
$ (docker image inspect "$IMAGE" || docker image pull "$IMAGE") &>/dev/null
7+
8+
NEXTSTRAIN_WORKDIR changes initial working directory.
9+
10+
$ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/augur "$IMAGE" \
11+
> bash -eu -c 'echo "$PWD"'
12+
/nextstrain/augur
13+
14+
$ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/augur -u 1234:5678 "$IMAGE" \
15+
> bash -eu -c 'echo "$PWD"'
16+
/nextstrain/augur
17+
18+
Missing directories are created, like with the `--workdir` option of `docker run`.
19+
20+
$ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/a/b/c "$IMAGE" \
21+
> bash -eu -c 'ls -ld "$PWD"'
22+
drwxr-xr-x * nextstrain nextstrain * /nextstrain/a/b/c (glob)
23+
24+
$ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nextstrain/a/b/c -u 1234:5678 "$IMAGE" \
25+
> bash -eu -c 'ls -ld "$PWD"'
26+
drwxr-xr-x * 1234 5678 * /nextstrain/a/b/c (glob)
27+
28+
…but permissions still apply, as the `mkdir` happens after drop-privs.
29+
30+
$ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nope "$IMAGE" \
31+
> bash -eu -c 'echo "$PWD"'
32+
mkdir: cannot create directory ‘/nope’: Permission denied
33+
[1]
34+
35+
$ docker run --rm --env=NEXTSTRAIN_WORKDIR=/nope -u 1234:5678 "$IMAGE" \
36+
> bash -eu -c 'echo "$PWD"'
37+
mkdir: cannot create directory ‘/nope’: Permission denied
38+
[1]

0 commit comments

Comments
 (0)