Dry up the CI/CD configuration

.github/workflows/verify.yml

@@ -12,10 +12,10 @@ jobs:
         include:
           - fdb_ver: 6.2.30
             fdb_lib_url: https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb
-            fdb_docker_image: foundationdb/foundationdb:6.2.30
+
           - fdb_ver: 7.1.61
             fdb_lib_url: https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients_7.1.61-1_amd64.deb
-            fdb_docker_image: foundationdb/foundationdb:7.1.61
+
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
@@ -24,27 +24,57 @@ jobs:
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
-    - name: Build images with cache
+    - name: Compute Docker tag
+      id: docker
+      run: echo "tag=$(./scripts/docker_tag.sh)" >> $GITHUB_OUTPUT
+      env:
+        FDB_VER: ${{ matrix.fdb_ver }}
+
+    - name: Build image
       uses: docker/bake-action@v5
       env:
-        DOCKER_TAG: ${{ github.sha }}_fdb.${{ matrix.fdb_ver }}
+        DOCKER_TAG: ${{ steps.docker.outputs.tag }}
+        FDB_VER: ${{ matrix.fdb_ver }}
         FDB_LIB_URL: ${{ matrix.fdb_lib_url }}
-        GOLANGCI_LINT_VER: ${{ secrets.GOLANGCI_LINT_VER || 'v1.49.0' }}
-        SHELLCHECK_URL: ${{ secrets.SHELLCHECK_URL || 'https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz' }}
-        HADOLINT_URL: ${{ secrets.HADOLINT_URL || 'https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64' }}
-        JQ_URL: ${{ secrets.JQ_URL || 'https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64' }}
-        PANDOC_URL: ${{ secrets.PANDOC_URL || 'https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb' }}
-        GO_URL: ${{ secrets.GO_URL || 'https://go.dev/dl/go1.19.1.linux-amd64.tar.gz' }}
       with:
-        files: docker-bake.json
+        files: bake.hcl
         targets: build
+        load: true
         set: |
           *.cache-from=type=gha,scope=build-${{ matrix.fdb_ver }}
           *.cache-to=type=gha,mode=max,scope=build-${{ matrix.fdb_ver }}
 
+    - name: Restore cache
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/fql
+        key: go-${{ matrix.fdb_ver }}-${{ hashFiles('go.sum') }}
+        restore-keys: |
+          go-${{ matrix.fdb_ver }}-
+
+    - name: Cache FDB image
+      id: cache-fdb
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/fql/fdb-image.tar
+        key: fdb-image-${{ matrix.fdb_ver }}
+
+    - name: Load FDB image from cache
+      if: steps.cache-fdb.outputs.cache-hit == 'true'
+      run: docker load -i ~/.cache/fql/fdb-image.tar
+
+    - name: Pull and save FDB image
+      if: steps.cache-fdb.outputs.cache-hit != 'true'
+      run: |
+        mkdir -p ~/.cache/fql
+        docker pull foundationdb/foundationdb:${{ matrix.fdb_ver }}
+        docker save foundationdb/foundationdb:${{ matrix.fdb_ver }} -o ~/.cache/fql/fdb-image.tar
+
     - name: Verify
       run: ./build.sh --generate --verify
       env:
         FDB_VER: ${{ matrix.fdb_ver }}
-        FDB_LIB_URL: ${{ matrix.fdb_lib_url }}
-        FDB_DOCKER_IMAGE: ${{ matrix.fdb_docker_image }}
+        CACHE_VOLUME: ~/.cache/fql
+
+    - name: Fix cache permissions
+      run: sudo chown -R $(id -u):$(id -g) ~/.cache/fql

.gitignore

@@ -1,3 +1,2 @@
-.idea
 /fql
 *.cluster

CLAUDE.md

@@ -80,6 +80,7 @@ Query syntax uses directories (`/path/to/dir`), tuples `("elem1", 2, 0xFF)`, and
 
 The build system uses Docker Compose for consistent builds:
 - `build.sh` script provides unified interface
-- `docker-compose.yaml` defines build and FDB containers
+- `bake.hcl` defines Docker build configuration
+- `compose.yaml` defines runtime services (build, fql, fdb containers)
 - Use `--latest` flag for offline development
 - FDB container automatically managed for tests

bake.hcl

@@ -0,0 +1,76 @@
+# Docker Bake configuration for FQL.
+#
+# Variables can be overridden via environment variables.
+# The CI workflow passes these as env vars from the matrix.
+
+variable "DOCKER_TAG" {
+  default = "latest"
+}
+
+variable "FDB_VER" {
+  default = "6.2.30"
+}
+
+variable "FDB_LIB_URL" {
+  default = "https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb"
+}
+
+variable "GO_URL" {
+  default = "https://go.dev/dl/go1.19.1.linux-amd64.tar.gz"
+}
+
+variable "GOLANGCI_LINT_VER" {
+  default = "v1.49.0"
+}
+
+variable "SHELLCHECK_URL" {
+  default = "https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz"
+}
+
+variable "HADOLINT_URL" {
+  default = "https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64"
+}
+
+variable "JQ_URL" {
+  default = "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64"
+}
+
+variable "PANDOC_URL" {
+  default = "https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb"
+}
+
+# Shared build arguments used by both targets.
+function "build_args" {
+  params = []
+  result = {
+    FQL_VER           = DOCKER_TAG
+    FDB_LIB_URL       = FDB_LIB_URL
+    GO_URL            = GO_URL
+    GOLANGCI_LINT_VER = GOLANGCI_LINT_VER
+    SHELLCHECK_URL    = SHELLCHECK_URL
+    HADOLINT_URL      = HADOLINT_URL
+    JQ_URL            = JQ_URL
+    PANDOC_URL        = PANDOC_URL
+  }
+}
+
+group "default" {
+  targets = ["build", "fql"]
+}
+
+target "build" {
+  context    = "./docker"
+  dockerfile = "Dockerfile"
+  target     = "builder"
+  tags       = ["docker.io/janderland/fql-build:${DOCKER_TAG}"]
+  platforms  = ["linux/amd64"]
+  args       = build_args()
+}
+
+target "fql" {
+  context    = "."
+  dockerfile = "./docker/Dockerfile"
+  tags       = ["docker.io/janderland/fql:${DOCKER_TAG}"]
+  platforms  = ["linux/amd64"]
+  args       = build_args()
+}

build.sh

@@ -8,11 +8,6 @@ build.sh is a facade for docker compose. It runs a set of
 optional tasks in the order specified below. This is the same
 script used by CI/CD to build, test, and package FQL.
 
-If the '--latest' flag is set, then the script will use 'latest'
-as the tag for the docker images. This allows for offline
-development. Otherwise, the tag changes will every commit and
-docker attmepts a build which will fail.
-
 If the '--image build' flag is set then the script starts off by
 running 'docker build' for the 'fql-build' docker image. The tag
 is determined by the git tag/hash and the FDB version. This image
@@ -30,8 +25,7 @@ generated under the /docs directory.
 
 If the '--image fql' flag is set then the script runs 'docker
 build' for the 'fql' docker image. The tag is determined by the
-git tag/hash and the version of the FDB library specified in the
-'.env' file.
+git tag/hash and the FDB version.
 
 If the '--run' flag is provided then all the args after this flag
 are passed to an instance of the 'fql' docker image. Normally
@@ -51,9 +45,8 @@ separating them with commas.
   ./build.sh --image build,fql
 
 When building Docker images, the dependencies of the Dockerfile
-are specified in the '.env' file. When this file is changed,
-you'll need to rebuild the docker images for the changes to take
-effect.
+are specified in 'bake.hcl'. When this file is changed, you'll
+need to rebuild the docker images for the changes to take effect.
 END
 }
 
@@ -81,40 +74,6 @@ function join_array {
 }
 
 
-# code_version returns the latest tag for the current
-# Git commit. If there are no tags associated with
-# the commit then the short hash is returned.
-
-function code_version {
-  local tag=""
-  if tag="$(git describe --tags)"; then
-    echo "$tag"
-    return 0
-  fi
-
-  git rev-parse --short HEAD
-}
-
-
-# fdb_version returns the version of the FDB
-# library specified by the env var FDB_VER.
-# If FDB_VER is not defined then the .env
-# file is read to obtain the version.
-
-function fdb_version {
-  if [[ -n "$FDB_VER" ]]; then
-    echo "$FDB_VER"
-    return 0
-  fi
-
-  local regex='FDB_VER=([^'$'\n'']*)'
-  if ! [[ "$(cat .env)" =~ $regex ]]; then
-    fail "Couldn't find FDB version in .env file."
-  fi
-  echo "${BASH_REMATCH[1]}"
-}
-
-
 # Change directory to repo root.
 
 cd "${0%/*}"
@@ -145,11 +104,6 @@ while [[ $# -gt 0 ]]; do
       shift 1
       ;;
 
-    --latest)
-      LATEST="x"
-      shift 1
-      ;;
-
     --image)
       for service in $(echo "$2" | tr "," "\n"); do
         case $service in
@@ -210,23 +164,27 @@ FQL_COMMAND=${FQL_ARGS[*]}
 echo "FQL_COMMAND=${FQL_COMMAND}"
 export FQL_COMMAND
 
-DOCKER_TAG="${LATEST:-$(code_version)}_fdb.$(fdb_version)"
+DOCKER_TAG="$(./scripts/docker_tag.sh)"
 echo "DOCKER_TAG=${DOCKER_TAG}"
 export DOCKER_TAG
 
+FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FDB_VER:-6.2.30}"
+echo "FDB_DOCKER_IMAGE=${FDB_DOCKER_IMAGE}"
+export FDB_DOCKER_IMAGE
+
 
 # Run the requested commands.
 
 if [[ -n "$IMAGE_BUILD" ]]; then
-  (set -x; docker buildx bake --load build)
+  (set -x; docker buildx bake -f bake.hcl --load build)
 fi
 
 if [[ -n "$BUILD_COMMAND" ]]; then
   (set -x; docker compose run --rm build /bin/sh -c "$BUILD_COMMAND")
 fi
 
 if [[ -n "$IMAGE_FQL" ]]; then
-  (set -x; docker buildx bake --load fql)
+  (set -x; docker buildx bake -f bake.hcl --load fql)
 fi
 
 if [[ -n "$RUN_FQL" ]]; then

docker-compose.yaml → compose.yaml

@@ -5,44 +5,29 @@ services:
 
   # The build service is responsible for building,
   # linting, and testing the code.
+  #
+  # The CACHE_VOLUME env var controls where the build
+  # cache is stored. Defaults to a named volume "cache"
+  # for local dev. CI sets this to a bind mount path
+  # so the cache can be persisted between runs.
   build:
     container_name: "build"
     image: "docker.io/janderland/fql-build:${DOCKER_TAG}"
     pull_policy: "never"
     platform: "linux/amd64"
-    build:
-      context: "./docker"
-      target: "builder"
-      platforms:
-        - "linux/amd64"
-      args: &build_args
-        FQL_VER: "${DOCKER_TAG}"
-        FDB_LIB_URL: "${FDB_LIB_URL}"
-        GO_URL: "${GO_URL}"
-        GOLANGCI_LINT_VER: "${GOLANGCI_LINT_VER}"
-        SHELLCHECK_URL: "${SHELLCHECK_URL}"
-        HADOLINT_URL: "${HADOLINT_URL}"
-        JQ_URL: "${JQ_URL}"
-        PANDOC_URL: "${PANDOC_URL}"
     depends_on:
       - "fdb"
     working_dir: "/fql"
     volumes:
       - ".:/fql"
-      - "cache:/cache"
+      - "${CACHE_VOLUME:-cache}:/cache"
 
   # The fql service allows us to build and test the
   # fql Docker image.
   fql:
     container_name: "fql"
     image: "docker.io/janderland/fql:${DOCKER_TAG}"
     platform: "linux/amd64"
-    build:
-      context: "."
-      dockerfile: "./docker/Dockerfile"
-      platforms:
-        - "linux/amd64"
-      args: *build_args
     depends_on:
       - "fdb"