diff --git a/.env b/.env deleted file mode 100644 index 2a98d381..00000000 --- a/.env +++ /dev/null @@ -1,14 +0,0 @@ -# This is only defined to hide undefined -# variable warnings while running docker -# compose commands without build.sh. -DOCKER_TAG= - -FDB_VER=6.2.30 -FDB_LIB_URL=https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb -FDB_DOCKER_IMAGE=foundationdb/foundationdb:6.2.30 -GO_URL=https://go.dev/dl/go1.19.1.linux-amd64.tar.gz -GOLANGCI_LINT_VER=v1.49.0 -SHELLCHECK_URL=https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz -HADOLINT_URL=https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64 -JQ_URL=https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -PANDOC_URL=https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 7992bcbf..633a2e9e 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -12,10 +12,10 @@ jobs: include: - fdb_ver: 6.2.30 fdb_lib_url: https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb - fdb_docker_image: foundationdb/foundationdb:6.2.30 + - fdb_ver: 7.1.61 fdb_lib_url: https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients_7.1.61-1_amd64.deb - fdb_docker_image: foundationdb/foundationdb:7.1.61 + runs-on: ubuntu-latest steps: - name: Checkout @@ -24,27 +24,57 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Build images with cache + - name: Compute Docker tag + id: docker + run: echo "tag=$(./scripts/docker_tag.sh)" >> $GITHUB_OUTPUT + env: + FDB_VER: ${{ matrix.fdb_ver }} + + - name: Build image uses: docker/bake-action@v5 env: - DOCKER_TAG: ${{ github.sha }}_fdb.${{ matrix.fdb_ver }} + DOCKER_TAG: ${{ steps.docker.outputs.tag }} + FDB_VER: ${{ matrix.fdb_ver }} FDB_LIB_URL: ${{ matrix.fdb_lib_url }} - GOLANGCI_LINT_VER: ${{ secrets.GOLANGCI_LINT_VER || 'v1.49.0' }} - SHELLCHECK_URL: ${{ secrets.SHELLCHECK_URL || 'https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz' }} - HADOLINT_URL: ${{ secrets.HADOLINT_URL || 'https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64' }} - JQ_URL: ${{ secrets.JQ_URL || 'https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64' }} - PANDOC_URL: ${{ secrets.PANDOC_URL || 'https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb' }} - GO_URL: ${{ secrets.GO_URL || 'https://go.dev/dl/go1.19.1.linux-amd64.tar.gz' }} with: - files: docker-bake.json + files: bake.hcl targets: build + load: true set: | *.cache-from=type=gha,scope=build-${{ matrix.fdb_ver }} *.cache-to=type=gha,mode=max,scope=build-${{ matrix.fdb_ver }} + - name: Restore cache + uses: actions/cache@v4 + with: + path: ~/.cache/fql + key: go-${{ matrix.fdb_ver }}-${{ hashFiles('go.sum') }} + restore-keys: | + go-${{ matrix.fdb_ver }}- + + - name: Cache FDB image + id: cache-fdb + uses: actions/cache@v4 + with: + path: ~/.cache/fql/fdb-image.tar + key: fdb-image-${{ matrix.fdb_ver }} + + - name: Load FDB image from cache + if: steps.cache-fdb.outputs.cache-hit == 'true' + run: docker load -i ~/.cache/fql/fdb-image.tar + + - name: Pull and save FDB image + if: steps.cache-fdb.outputs.cache-hit != 'true' + run: | + mkdir -p ~/.cache/fql + docker pull foundationdb/foundationdb:${{ matrix.fdb_ver }} + docker save foundationdb/foundationdb:${{ matrix.fdb_ver }} -o ~/.cache/fql/fdb-image.tar + - name: Verify run: ./build.sh --generate --verify env: FDB_VER: ${{ matrix.fdb_ver }} - FDB_LIB_URL: ${{ matrix.fdb_lib_url }} - FDB_DOCKER_IMAGE: ${{ matrix.fdb_docker_image }} + CACHE_VOLUME: ~/.cache/fql + + - name: Fix cache permissions + run: sudo chown -R $(id -u):$(id -g) ~/.cache/fql diff --git a/.gitignore b/.gitignore index 360b324c..bb80bc36 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ -.idea /fql *.cluster diff --git a/CLAUDE.md b/CLAUDE.md index 7f527fdf..df9f8892 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,6 +80,7 @@ Query syntax uses directories (`/path/to/dir`), tuples `("elem1", 2, 0xFF)`, and The build system uses Docker Compose for consistent builds: - `build.sh` script provides unified interface -- `docker-compose.yaml` defines build and FDB containers +- `bake.hcl` defines Docker build configuration +- `compose.yaml` defines runtime services (build, fql, fdb containers) - Use `--latest` flag for offline development - FDB container automatically managed for tests \ No newline at end of file diff --git a/bake.hcl b/bake.hcl new file mode 100644 index 00000000..acc2a5ef --- /dev/null +++ b/bake.hcl @@ -0,0 +1,76 @@ +# Docker Bake configuration for FQL. +# +# Variables can be overridden via environment variables. +# The CI workflow passes these as env vars from the matrix. + +variable "DOCKER_TAG" { + default = "latest" +} + +variable "FDB_VER" { + default = "6.2.30" +} + +variable "FDB_LIB_URL" { + default = "https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb" +} + +variable "GO_URL" { + default = "https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" +} + +variable "GOLANGCI_LINT_VER" { + default = "v1.49.0" +} + +variable "SHELLCHECK_URL" { + default = "https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz" +} + +variable "HADOLINT_URL" { + default = "https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64" +} + +variable "JQ_URL" { + default = "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" +} + +variable "PANDOC_URL" { + default = "https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" +} + +# Shared build arguments used by both targets. +function "build_args" { + params = [] + result = { + FQL_VER = DOCKER_TAG + FDB_LIB_URL = FDB_LIB_URL + GO_URL = GO_URL + GOLANGCI_LINT_VER = GOLANGCI_LINT_VER + SHELLCHECK_URL = SHELLCHECK_URL + HADOLINT_URL = HADOLINT_URL + JQ_URL = JQ_URL + PANDOC_URL = PANDOC_URL + } +} + +group "default" { + targets = ["build", "fql"] +} + +target "build" { + context = "./docker" + dockerfile = "Dockerfile" + target = "builder" + tags = ["docker.io/janderland/fql-build:${DOCKER_TAG}"] + platforms = ["linux/amd64"] + args = build_args() +} + +target "fql" { + context = "." + dockerfile = "./docker/Dockerfile" + tags = ["docker.io/janderland/fql:${DOCKER_TAG}"] + platforms = ["linux/amd64"] + args = build_args() +} diff --git a/build.sh b/build.sh index b3a112c4..a2d6d7fe 100755 --- a/build.sh +++ b/build.sh @@ -8,11 +8,6 @@ build.sh is a facade for docker compose. It runs a set of optional tasks in the order specified below. This is the same script used by CI/CD to build, test, and package FQL. -If the '--latest' flag is set, then the script will use 'latest' -as the tag for the docker images. This allows for offline -development. Otherwise, the tag changes will every commit and -docker attmepts a build which will fail. - If the '--image build' flag is set then the script starts off by running 'docker build' for the 'fql-build' docker image. The tag is determined by the git tag/hash and the FDB version. This image @@ -30,8 +25,7 @@ generated under the /docs directory. If the '--image fql' flag is set then the script runs 'docker build' for the 'fql' docker image. The tag is determined by the -git tag/hash and the version of the FDB library specified in the -'.env' file. +git tag/hash and the FDB version. If the '--run' flag is provided then all the args after this flag are passed to an instance of the 'fql' docker image. Normally @@ -51,9 +45,8 @@ separating them with commas. ./build.sh --image build,fql When building Docker images, the dependencies of the Dockerfile -are specified in the '.env' file. When this file is changed, -you'll need to rebuild the docker images for the changes to take -effect. +are specified in 'bake.hcl'. When this file is changed, you'll +need to rebuild the docker images for the changes to take effect. END } @@ -81,40 +74,6 @@ function join_array { } -# code_version returns the latest tag for the current -# Git commit. If there are no tags associated with -# the commit then the short hash is returned. - -function code_version { - local tag="" - if tag="$(git describe --tags)"; then - echo "$tag" - return 0 - fi - - git rev-parse --short HEAD -} - - -# fdb_version returns the version of the FDB -# library specified by the env var FDB_VER. -# If FDB_VER is not defined then the .env -# file is read to obtain the version. - -function fdb_version { - if [[ -n "$FDB_VER" ]]; then - echo "$FDB_VER" - return 0 - fi - - local regex='FDB_VER=([^'$'\n'']*)' - if ! [[ "$(cat .env)" =~ $regex ]]; then - fail "Couldn't find FDB version in .env file." - fi - echo "${BASH_REMATCH[1]}" -} - - # Change directory to repo root. cd "${0%/*}" @@ -145,11 +104,6 @@ while [[ $# -gt 0 ]]; do shift 1 ;; - --latest) - LATEST="x" - shift 1 - ;; - --image) for service in $(echo "$2" | tr "," "\n"); do case $service in @@ -210,15 +164,19 @@ FQL_COMMAND=${FQL_ARGS[*]} echo "FQL_COMMAND=${FQL_COMMAND}" export FQL_COMMAND -DOCKER_TAG="${LATEST:-$(code_version)}_fdb.$(fdb_version)" +DOCKER_TAG="$(./scripts/docker_tag.sh)" echo "DOCKER_TAG=${DOCKER_TAG}" export DOCKER_TAG +FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FDB_VER:-6.2.30}" +echo "FDB_DOCKER_IMAGE=${FDB_DOCKER_IMAGE}" +export FDB_DOCKER_IMAGE + # Run the requested commands. if [[ -n "$IMAGE_BUILD" ]]; then - (set -x; docker buildx bake --load build) + (set -x; docker buildx bake -f bake.hcl --load build) fi if [[ -n "$BUILD_COMMAND" ]]; then @@ -226,7 +184,7 @@ if [[ -n "$BUILD_COMMAND" ]]; then fi if [[ -n "$IMAGE_FQL" ]]; then - (set -x; docker buildx bake --load fql) + (set -x; docker buildx bake -f bake.hcl --load fql) fi if [[ -n "$RUN_FQL" ]]; then diff --git a/docker-compose.yaml b/compose.yaml similarity index 57% rename from docker-compose.yaml rename to compose.yaml index 970655e9..3d58aa7a 100644 --- a/docker-compose.yaml +++ b/compose.yaml @@ -5,31 +5,22 @@ services: # The build service is responsible for building, # linting, and testing the code. + # + # The CACHE_VOLUME env var controls where the build + # cache is stored. Defaults to a named volume "cache" + # for local dev. CI sets this to a bind mount path + # so the cache can be persisted between runs. build: container_name: "build" image: "docker.io/janderland/fql-build:${DOCKER_TAG}" pull_policy: "never" platform: "linux/amd64" - build: - context: "./docker" - target: "builder" - platforms: - - "linux/amd64" - args: &build_args - FQL_VER: "${DOCKER_TAG}" - FDB_LIB_URL: "${FDB_LIB_URL}" - GO_URL: "${GO_URL}" - GOLANGCI_LINT_VER: "${GOLANGCI_LINT_VER}" - SHELLCHECK_URL: "${SHELLCHECK_URL}" - HADOLINT_URL: "${HADOLINT_URL}" - JQ_URL: "${JQ_URL}" - PANDOC_URL: "${PANDOC_URL}" depends_on: - "fdb" working_dir: "/fql" volumes: - ".:/fql" - - "cache:/cache" + - "${CACHE_VOLUME:-cache}:/cache" # The fql service allows us to build and test the # fql Docker image. @@ -37,12 +28,6 @@ services: container_name: "fql" image: "docker.io/janderland/fql:${DOCKER_TAG}" platform: "linux/amd64" - build: - context: "." - dockerfile: "./docker/Dockerfile" - platforms: - - "linux/amd64" - args: *build_args depends_on: - "fdb" diff --git a/docker-bake.json b/docker-bake.json deleted file mode 100644 index ea75db00..00000000 --- a/docker-bake.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "variable": { - "DOCKER_TAG": { - "default": "latest" - }, - "FDB_LIB_URL": { - "default": "https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb" - }, - "GO_URL": { - "default": "https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" - }, - "GOLANGCI_LINT_VER": { - "default": "v1.49.0" - }, - "SHELLCHECK_URL": { - "default": "https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz" - }, - "HADOLINT_URL": { - "default": "https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64" - }, - "JQ_URL": { - "default": "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" - }, - "PANDOC_URL": { - "default": "https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" - }, - "FQL_VER": { - "default": "${DOCKER_TAG}" - } - }, - "group": { - "default": { - "targets": [ - "build", - "fql" - ] - } - }, - "target": { - "build": { - "context": "./docker", - "dockerfile": "Dockerfile", - "target": "builder", - "tags": [ - "docker.io/janderland/fql-build:${DOCKER_TAG}" - ], - "platforms": [ - "linux/amd64" - ], - "args": { - "FQL_VER": "${FQL_VER}", - "FDB_LIB_URL": "${FDB_LIB_URL}", - "GO_URL": "${GO_URL}", - "GOLANGCI_LINT_VER": "${GOLANGCI_LINT_VER}", - "SHELLCHECK_URL": "${SHELLCHECK_URL}", - "HADOLINT_URL": "${HADOLINT_URL}", - "JQ_URL": "${JQ_URL}", - "PANDOC_URL": "${PANDOC_URL}" - }, - "cache-from": [ - "type=gha,scope=build" - ], - "cache-to": [ - "type=gha,mode=max,scope=build" - ] - }, - "fql": { - "context": ".", - "dockerfile": "./docker/Dockerfile", - "tags": [ - "docker.io/janderland/fql:${DOCKER_TAG}" - ], - "platforms": [ - "linux/amd64" - ], - "args": { - "FQL_VER": "${FQL_VER}", - "FDB_LIB_URL": "${FDB_LIB_URL}", - "GO_URL": "${GO_URL}", - "GOLANGCI_LINT_VER": "${GOLANGCI_LINT_VER}", - "SHELLCHECK_URL": "${SHELLCHECK_URL}", - "HADOLINT_URL": "${HADOLINT_URL}", - "JQ_URL": "${JQ_URL}", - "PANDOC_URL": "${PANDOC_URL}" - }, - "cache-from": [ - "type=gha,scope=fql" - ], - "cache-to": [ - "type=gha,mode=max,scope=fql" - ] - } - } -} diff --git a/scripts/docker_tag.sh b/scripts/docker_tag.sh new file mode 100755 index 00000000..2f96dc78 --- /dev/null +++ b/scripts/docker_tag.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -eo pipefail + +# docker_tag.sh outputs the Docker image tag based on +# the git version and FDB version. This script is used +# by build.sh and CI workflows to ensure consistent tags. +# +# Environment variables: +# LATEST - If set, uses "latest" instead of git version +# FDB_VER - FDB version (defaults to 6.2.30) + +# code_version returns the latest tag for the current +# Git commit. If there are no tags associated with +# the commit then the short hash is returned. + +function code_version { + local tag="" + if tag="$(git describe --tags 2>/dev/null)"; then + echo "$tag" + return 0 + fi + git rev-parse --short HEAD +} + +# fdb_version returns the version of the FDB library. +# Uses FDB_VER env var if set, otherwise defaults to +# the version specified in bake.hcl. + +function fdb_version { + echo "${FDB_VER:-6.2.30}" +} + +echo "${LATEST:-$(code_version)}_fdb.$(fdb_version)" diff --git a/scripts/verify_generation.sh b/scripts/verify_generation.sh index e5583ee1..1f44a7c1 100755 --- a/scripts/verify_generation.sh +++ b/scripts/verify_generation.sh @@ -7,6 +7,7 @@ cd "${0%/*}/.." STATUS="$(git status --short)" if [[ -n "$STATUS" ]]; then echo "ERR! Generated code cannot be verified while there are uncommitted changes." + echo "$STATUS" exit 1 fi @@ -16,7 +17,7 @@ fi STATUS="$(git status --short)" if [[ -n "$STATUS" ]]; then echo "ERR! The following generated files are outdated:" - cut -d' ' -f3 <<< "$STATUS" + echo "$STATUS" echo echo "If this is CI/CD, you'll need to execute" echo "'go generate ./...' on your local machine"