add tasks with skopeo; cleanup Dockerfile

jackdbd · Oct 10, 2024 · a8b3d21 · a8b3d21
1 parent 7ed22ec
commit a8b3d21
Show file tree

Hide file tree

Showing 2 changed files with 91 additions and 65 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -14,23 +14,32 @@ RUN if [ -z "${APP_NAME}" ] ; then echo "APP_NAME not set!" ; exit 1; fi
 ARG APP_VERSION
 RUN if [ -z "${APP_VERSION}" ] ; then echo "APP_VERSION not set!" ; exit 1; fi
 
-# If you need to use a version of pod-jackdbd-jsoup not available on the pod
-# registry, you will need to download it and place it where bb.edn declares it
-# (it would be {:pods {some-pod {:path "some-path"}}}). 
-# ARG JSOUP_POD_PATH="resources/pod/pod-jackdbd-jsoup"
-
 ARG CREATED_DATE
+RUN if [ -z "${CREATED_DATE}" ] ; then echo "CREATED_DATE not set!" ; exit 1; fi
 
 # https://github.com/babashka/babashka/releases
 ARG BB_VERSION="1.4.192"
 
-RUN apt update
-RUN apt install wget
-RUN wget --directory-prefix /tmp "https://github.com/babashka/babashka/releases/download/v${BB_VERSION}/babashka-${BB_VERSION}-linux-amd64-static.tar.gz"
-RUN tar xf "/tmp/babashka-${BB_VERSION}-linux-amd64-static.tar.gz" --directory=/tmp
-RUN mv /tmp/bb /usr/local/bin/bb
+# Case A: pod version on pod registry
+# Babashka will download the pod here when running any Babashka task, like
+# `bb run build:bb-uber`, because in the bb.edn file we specified a `:version`
+# of this pod which is available on pod registry.
+# ARG JSOUP_POD_VERSION
+# RUN if [ -z "${JSOUP_POD_VERSION}" ] ; then echo "JSOUP_POD_VERSION not set!" ; exit 1; fi
+# Case B: pod version NOT on pod registry
+# If you need to use a version of pod-jackdbd-jsoup which is not available on
+# pod registry, you will need to use `:path` in the bb.edn file, download the
+# pod it manually, and place it where bb.edn declares it.
+# (e.g. {:pods {com.github.jackdbd/jsoup {:path "/usr/src/app/pods/jsoup"}}}).
+
+# Use a single RUN instruction to create just one layer in the container image.
+RUN apt update && \
+    apt install wget && \
+    wget --directory-prefix /tmp "https://github.com/babashka/babashka/releases/download/v${BB_VERSION}/babashka-${BB_VERSION}-linux-amd64-static.tar.gz" && \
+    tar xf "/tmp/babashka-${BB_VERSION}-linux-amd64-static.tar.gz" --directory=/tmp && \
+    mv /tmp/bb /usr/local/bin/bb && \
+    mkdir -p ${APP_DIR}
 
-RUN mkdir -p ${APP_DIR}
 WORKDIR ${APP_DIR}
 
 # I think that resources (i.e. assets) and source code change frequently, while
@@ -43,8 +52,8 @@ COPY bb ${APP_DIR}/bb
 # COPY ${JSOUP_POD_PATH} ${APP_DIR}/${JSOUP_POD_PATH}
 COPY src ${APP_DIR}/src
 
-RUN bb run build:bb-uber
-RUN mv target/${APP_NAME}-${APP_VERSION}.jar "${APP_NAME}.jar"
+RUN bb run build:bb-uber && \
+    mv target/${APP_NAME}-${APP_VERSION}.jar "${APP_NAME}.jar"
 
 # === STAGE 2 ================================================================ #
 # Run the uberjar
@@ -53,28 +62,36 @@ RUN mv target/${APP_NAME}-${APP_VERSION}.jar "${APP_NAME}.jar"
 # skopeo list-tags docker://docker.io/babashka/babashka
 FROM docker.io/babashka/babashka:1.4.193-SNAPSHOT AS bb-uberjar-runner
 
-ARG NON_ROOT_USER=zaraki
-RUN groupadd --gid 1234 $NON_ROOT_USER && \
-    useradd --uid 1234 --gid 1234 --shell /bin/bash --create-home $NON_ROOT_USER
-
 ARG APP_DIR=/usr/src/app
+RUN if [ -z "${APP_DIR}" ] ; then echo "APP_DIR not set!" ; exit 1; fi
+
+ARG CREATED_DATE
+RUN if [ -z "${CREATED_DATE}" ] ; then echo "CREATED_DATE not set!" ; exit 1; fi
+
 ARG JSOUP_POD_VERSION
 RUN if [ -z "${JSOUP_POD_VERSION}" ] ; then echo "JSOUP_POD_VERSION not set!" ; exit 1; fi
-ARG CREATED_DATE
+
+ARG NON_ROOT_USER=zaraki
+RUN if [ -z "${NON_ROOT_USER}" ] ; then echo "NON_ROOT_USER not set!" ; exit 1; fi
+
+RUN groupadd --gid 1234 $NON_ROOT_USER && \
+    useradd --uid 1234 --gid 1234 --shell /bin/bash --create-home $NON_ROOT_USER
 
 USER $NON_ROOT_USER
 WORKDIR "/home/$NON_ROOT_USER"
 
 COPY --from=bb-uberjar-builder "${APP_DIR}/fosdem-dl.jar" fosdem-dl.jar
 
-# Babashka downloads pods to $HOME/.babashka/pods/repository, but in my bb.edn
-# I declared that this pod is at resources/pod/pod-jackdbd-jsoup
-# ARG JSOUP_POD_PATH="resources/pod/pod-jackdbd-jsoup"
-# ARG JSOUP_POD_BB_PATH="/home/${NON_ROOT_USER}/.babashka/pods/repository/com.github.jackdbd/pod.jackdbd.jsoup/$JSOUP_POD_VERSION/linux/x86_64/pod-jackdbd-jsoup"
+# Bake the jsoup pod into the container image.
+# NOTE: we could avoid baking the pod into the container image and save ~15 MB,
+# but this would mean that Babashka will have to download the pod at runtime
+# every single time the container starts.
 
-# We can either copy the pod from the previous stage...
-# COPY --from=bb-uberjar-builder "${APP_DIR}/${JSOUP_POD_PATH}" "${JSOUP_POD_PATH}"
-#...or let Babashka download it from the pod registry (if it's available there).
+# Option A: if the builder stage has already downloaded the pod, copied it,
+# created a non-root user and gave that user execution permissions on the pod,
+# we have nothing to do here.
+
+# Option B: we let Babashka re-download the pod from the pod registry.
 RUN bb -e "(require '[babashka.pods :as pods]) \
 (pods/load-pod 'com.github.jackdbd/jsoup \"${JSOUP_POD_VERSION}\")"
 
@@ -83,18 +100,22 @@ RUN bb -e "(require '[babashka.pods :as pods]) \
 # An alternative to this mess would be to set the BABASHKA_PODS_DIR environment
 # variable, I think.
 # https://github.com/babashka/pods?tab=readme-ov-file#where-does-the-pod-come-from
-# NOTE: I use a single RUN instruction to save one layer in the container image.
-# We could download the pod with one RUN instruction and then move it with
-# another one, but this would create an additional layer of roughly 26 MB.
-# TIP: You can use dive to inspect the layers of the container image.
 # RUN bb -e "(require '[babashka.pods :as pods]) \
 # (pods/load-pod 'com.github.jackdbd/jsoup \"${JSOUP_POD_VERSION}\")" && \
 #     mkdir -p $(dirname $JSOUP_POD_PATH) && \
 #     mv $JSOUP_POD_BB_PATH $JSOUP_POD_PATH && \
 #     rm -rf "/home/${NON_ROOT_USER}/.babashka"
 
+# https://github.com/opencontainers/image-spec/blob/main/annotations.md
 LABEL org.opencontainers.image.created=${CREATED_DATE}
+LABEL org.opencontainers.image.description="CLI to download videos and slides from FOSDEM websites"
+# https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/
+LABEL org.opencontainers.image.licenses="MIT"
+# This is required when pushing the container image from a computer to GitHub's Container Registry.
 LABEL org.opencontainers.image.source=https://github.com/jackdbd/fosdem-dl
+LABEL org.opencontainers.image.title=fosdem-dl
+LABEL org.opencontainers.image.url=https://github.com/jackdbd/fosdem-dl
+
 ENTRYPOINT ["bb", "fosdem-dl.jar"]
 CMD ["help"]
 # CMD ["talks", "--help"]

diff --git a/bb.edn b/bb.edn
@@ -6,6 +6,7 @@
  :tasks
  {:requires ([babashka.process :refer [process check]]
              [clojure.edn :as edn]
+             [clojure.java.shell :refer [sh]]
              [clojure.string :as str]
              [fosdem-dl.talks-cli :refer [talks-cli]]
              [fosdem-dl.tracks-cli :refer [tracks-cli]]
@@ -21,7 +22,7 @@
           (def image-tag (format "ghcr.io/%s/%s:%s" registry-namespace app-name app-version)))
 
   build:bb-uber
-  {:doc "Build the Babashka uberjar (see: https://book.babashka.org/#_uberjar)"
+  {:doc "Build Babashka uberjar (see: https://book.babashka.org/#_uberjar)"
    :depends [clean:artifacts]
    :task (let [debug? false
                cmd (if (= "true" (System/getenv "DEBUG_BB_UBERJAR"))
@@ -32,29 +33,28 @@
            (shell cmd))}
 
   build:uber
-  {:doc "Build the uberjar"
+  {:doc "Build uberjar"
    :depends [clean:artifacts]
    :task (clojure "-T:build uber")}
 
   build:binary
-  {:doc "Compile the CLI into a statically-linked binary with GraalVM native-image (Linux only)"
-   ;; :depends [build:bb-uber]
-   ;; :depends [build:uber]
+  {:doc "Compile statically-linked binary with GraalVM native-image (Linux only)"
+   :depends [build:uber]
    :task (shell "script/compile.sh")}
 
   cp
-  {:doc "Print the classpath"
+  {:doc "Print classpath"
    :task (tasks/print-classpath)}
 
   clean
-  {:doc "Remove compilation artifacts, pods and downloads (in parallel)"
+  {:doc "Remove compilation artifacts and pods (in parallel)"
    :task (run '-clean {:parallel true})}
 
   -clean
-  {:depends [clean:artifacts clean:downloads clean:pods]}
+  {:depends [clean:artifacts clean:pods]}
 
   clean:artifacts
-  {:doc "Remove compilation artifacts"
+  {:doc "Remove compilation artifacts (uberjars, binaries)"
    :task (clojure "-T:build clean")}
 
   clean:downloads
@@ -70,13 +70,13 @@
            (shell cmd))}
 
   cli
-  {:doc "fosdem-dl CLI (e.g. 'bb cli talks -y 2019 -t llvm', 'bb cli tracks -y 2020')"
+  {:doc "Run CLI as Babashka script (e.g. 'bb cli talks -y 2019 -t llvm', 'bb cli tracks -y 2020')"
    :task (let [list *command-line-args*
                cmd (conj list "./src/fosdem_dl/cli.clj")]
            (shell cmd))}
 
   cli:debug
-  {:doc "Launch the CLI with bb --debug -x fosdem-dl.cli/-main"
+  {:doc "Run CLI with bb --debug -x fosdem-dl.cli/-main"
    ;; The usage of clojure.java.io/resource assumes a classpath, so
    ;; when this is used in your uberscript, you still have to set a
    ;; classpath and bring the resources along.
@@ -88,28 +88,32 @@
            (shell cmd))}
 
   dep:upgrade
-  {:doc "Upgrade all dependencies"
+  {:doc "Upgrade all dependencies declared in deps.edn"
    :task (shell "neil dep upgrade")}
 
   container:build
-  {:doc "Build the container image"
+  {:doc "Build container image"
    :task (let [created-date (str (java.time.ZonedDateTime/now))
+               ;; On NixOS, the SOURCE_DATE_EPOCH environment variable is often
+               ;; set to UTC 1980-01-01 00:00:00 as part of the system’s
+               ;; commitment to reproducible builds. But if we want tools like
+               ;; `docker image ls` to display the correct created date, we need
+               ;; to set SOURCE_DATE_EPOCH when building the container image.
+               epoch (str/trim (:out (sh "date" "+%s")))
                options ["--file" "Dockerfile"
                         "--build-arg" (str "APP_NAME=" app-name)
                         "--build-arg" (str "APP_VERSION=" app-version)
-                        "--build-arg" "DEBUG_BB_UBERJAR=true"
-                        "--build-arg" "JSOUP_POD_VERSION=0.4.0"
                         "--build-arg" (str "CREATED_DATE=" created-date)
-                        "--debug"
-                        #_"--quiet"
+                        "--build-arg" (str "JSOUP_POD_VERSION=" "0.4.0")
+                        #_"--no-cache"
                         "--tag" image-tag]
                cmd (format "docker build ./ %s" (str/join " " options))]
-           (println cmd)
-           (shell cmd))}
+           (println (format "SOURCE_DATE_EPOCH=%s %s" epoch cmd))
+           (shell {:extra-env {"SOURCE_DATE_EPOCH" epoch}}  cmd))}
 
   container:cli
-  {:doc "Execute the fosdem-dl CLI in a Docker container"
-;;    :depends [container:build]
+  {:doc "Run CLI in a container"
+   ;; :depends [container:build]
    :task (let [cmd (str/join " " (concat [(format "docker run --rm %s" image-tag)] *command-line-args*))]
            (println cmd)
            (shell cmd))}
@@ -119,17 +123,27 @@
     ;; :depends [container:build]
    :task (shell (format "dive %s" image-tag))}
 
+  container:inspect
+  {:doc "Inspect container image layers with skopeo"
+   ;; :depends [container:build]
+   :task (shell (format "skopeo inspect docker://%s" image-tag))}
+
   container:push
-  {:doc "Push the container image to GitHub Container Registry"
+  {:doc "Push container image to GitHub Container Registry"
+;;    :depends [container:build container:scan]
    :task (let [cmd (format "docker push %s" image-tag)]
            (println cmd)
            (shell cmd))}
 
   container:scan
-  {:doc "Scan the container image with Trivy"
-      ;; :depends [container:build]
+  {:doc "Scan container image with Trivy"
+;;    :depends [container:build]
    :task (shell (format "trivy image --severity MEDIUM,HIGH,CRITICAL -f table %s" image-tag))}
 
+  container:tags
+  {:doc "List all tags for the container image with skopeo"
+   :task (shell (format "skopeo list-tags docker://ghcr.io/%s/%s" registry-namespace app-name))}
+
   -graph:gen
   {:task (clojure "-X:hiera" :layout :vertical)}
 
@@ -141,19 +155,10 @@
    :doc "Generate a graph of dependencies between namespaces and copy the image to resources/img"
    :task (shell "feh resources/img/namespaces.png")}
 
-  pod:upgrade
-  {:doc "Download the latest com.github.jackdbd/pod-jackdbd-jsoup available on GitHub Releases"
+  pod:download
+  {:doc "Download a version of pod-jackdbd-jsoup hosted on GitHub Releases (see POD_JACKDBD_JSOUP_VERSION environment variable)"
    :task (shell "./download_pod_jackdbd_jsoup.sh")}
 
   test
   {:doc "Run all tests"
-   :task (shell "bb test_runner.clj")}
-
-  ;; https://github.com/liquidz/babashka-test-action/?tab=readme-ov-file#local-usage
-  test:action
-  {:doc "Run all tests in a container, using https://github.com/liquidz/babashka-test-action/"
-   :task (let [working-dir (-> (process ["pwd"] {:out :string})
-                               check :out str/split-lines first)
-               cmd (format "docker run --rm -v %s:/tmp -w /tmp uochan/babashka-test 'src' 'test' '_test.clj$'" working-dir)]
-           (println cmd)
-           (shell cmd))}}}
+   :task (shell "bb test_runner.clj")}}}