From 771bb8b9eebee8ea5e0f1bcd72038cc8baff1aae Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 27 Dec 2025 23:23:16 +0000 Subject: [PATCH 1/8] Integrate fenv into build system This commit updates the build system to use fenv (FoundationDB development environment) for consistent Docker-based builds and testing. Changes: - Add fenv as a git submodule - Create docker/Dockerfile.fenv extending fenv base image with project-specific build tools (Go, golangci-lint, shellcheck, hadolint, jq, pandoc) - Update build.sh to use fenv.sh for build/verify/generate tasks - Update CLAUDE.md documentation to reflect fenv integration Benefits: - Standardized FoundationDB development environment - Automatic FDB cluster management for integration tests - Consistent behavior between local and CI environments - Persistent cache directory for faster builds --- .gitmodules | 3 ++ CLAUDE.md | 15 ++++---- build.sh | 82 ++++++++++++++++++++++-------------------- docker/Dockerfile.fenv | 54 ++++++++++++++++++++++++++++ fenv | 1 + 5 files changed, 110 insertions(+), 45 deletions(-) create mode 100644 .gitmodules create mode 100644 docker/Dockerfile.fenv create mode 160000 fenv diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..a2e8c4cc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "fenv"] + path = fenv + url = https://github.com/janderland/fenv.git diff --git a/CLAUDE.md b/CLAUDE.md index df9f8892..51cdf034 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ FQL is a query language and alternative client API for FoundationDB written in G - **Run FQL interactively**: `./build.sh --run [args]` ### Docker-based Development -The build system uses Docker Compose for consistent builds and testing with FoundationDB integration. Always use `./build.sh --verify` instead of running Go commands directly to ensure proper environment setup. +The build system uses fenv (https://github.com/janderland/fenv), a Docker-based FoundationDB development environment, for consistent builds and testing with FoundationDB integration. Always use `./build.sh --verify` instead of running Go commands directly to ensure proper environment setup. ## Architecture @@ -78,9 +78,10 @@ Query syntax uses directories (`/path/to/dir`), tuples `("elem1", 2, 0xFF)`, and ## Docker Environment -The build system uses Docker Compose for consistent builds: -- `build.sh` script provides unified interface -- `bake.hcl` defines Docker build configuration -- `compose.yaml` defines runtime services (build, fql, fdb containers) -- Use `--latest` flag for offline development -- FDB container automatically managed for tests \ No newline at end of file +The build system uses fenv for consistent builds: +- `build.sh` script provides unified interface wrapping `fenv/fenv.sh` +- `docker/Dockerfile.fenv` extends fenv base image with project-specific build tools +- `bake.hcl` defines Docker build configuration for final fql image +- `compose.yaml` defines runtime service for running fql image +- fenv automatically manages FDB container for integration testing +- Use `--latest` flag for offline development \ No newline at end of file diff --git a/build.sh b/build.sh index a2d6d7fe..bdfd41ea 100755 --- a/build.sh +++ b/build.sh @@ -4,12 +4,12 @@ set -eo pipefail function print_help { cat << END -build.sh is a facade for docker compose. It runs a set of -optional tasks in the order specified below. This is the same -script used by CI/CD to build, test, and package FQL. +build.sh is a facade for fenv (FoundationDB development environment). +It runs a set of optional tasks in the order specified below. This is +the same script used by CI/CD to build, test, and package FQL. If the '--image build' flag is set then the script starts off by -running 'docker build' for the 'fql-build' docker image. The tag +building the extended fenv image with all build tools. The tag is determined by the git tag/hash and the FDB version. This image is used to run the 'generate' and 'verify' tasks below. @@ -18,7 +18,7 @@ code generated by 'go generate ./...' is up to date. If the '--verify' flag is set then the script builds, lints, and tests the codebase. This task interacts with an FDB docker -container which is automatically started. +container which is automatically started by fenv. If the --docs' flag is set then the documentation HTML will be generated under the /docs directory. @@ -44,9 +44,8 @@ separating them with commas. ./build.sh --image build,fql -When building Docker images, the dependencies of the Dockerfile -are specified in 'bake.hcl'. When this file is changed, you'll -need to rebuild the docker images for the changes to take effect. +This script now uses fenv to provide a consistent build environment +with FoundationDB integration. END } @@ -139,54 +138,61 @@ while [[ $# -gt 0 ]]; do done -# Build variables required by the docker compose command. - -BUILD_TASKS=() - -if [[ -n "$VERIFY_GENERATION" ]]; then - BUILD_TASKS+=('./scripts/verify_generation.sh') -fi - -if [[ -n "$VERIFY_CODEBASE" ]]; then - BUILD_TASKS+=('./scripts/setup_database.sh') - BUILD_TASKS+=("./scripts/verify_codebase.sh") -fi - -if [[ -n "$GENERATE_DOCS" ]]; then - BUILD_TASKS+=('./scripts/generate_docs.sh') -fi - -BUILD_COMMAND="$(join_array ' && ' "${BUILD_TASKS[@]}")" -echo "BUILD_COMMAND=${BUILD_COMMAND}" -export BUILD_COMMAND - -FQL_COMMAND=${FQL_ARGS[*]} -echo "FQL_COMMAND=${FQL_COMMAND}" -export FQL_COMMAND +# Build variables for fenv and docker commands. DOCKER_TAG="$(./scripts/docker_tag.sh)" echo "DOCKER_TAG=${DOCKER_TAG}" export DOCKER_TAG -FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FDB_VER:-6.2.30}" -echo "FDB_DOCKER_IMAGE=${FDB_DOCKER_IMAGE}" -export FDB_DOCKER_IMAGE +# Set fenv environment variables +export FENV_FDB_VER="${FDB_VER:-6.2.30}" +export FENV_PROJECT_NAME="fql" + +# fenv.sh script location +FENV_SCRIPT="./fenv/fenv.sh" + +# Common fenv flags +FENV_FLAGS=() +FENV_FLAGS+=(--docker ./docker/Dockerfile.fenv) # Run the requested commands. +# Build the extended fenv image if requested if [[ -n "$IMAGE_BUILD" ]]; then - (set -x; docker buildx bake -f bake.hcl --load build) + echo "Building extended fenv image..." + (set -x; "$FENV_SCRIPT" "${FENV_FLAGS[@]}" --build) +fi + +# Execute build tasks (generate, verify, docs) using fenv +if [[ -n "$VERIFY_GENERATION" ]]; then + echo "Verifying code generation..." + (set -x; "$FENV_SCRIPT" "${FENV_FLAGS[@]}" --exec ./scripts/verify_generation.sh) fi -if [[ -n "$BUILD_COMMAND" ]]; then - (set -x; docker compose run --rm build /bin/sh -c "$BUILD_COMMAND") +if [[ -n "$VERIFY_CODEBASE" ]]; then + echo "Setting up database and verifying codebase..." + (set -x; "$FENV_SCRIPT" "${FENV_FLAGS[@]}" --exec ./scripts/setup_database.sh --exec ./scripts/verify_codebase.sh) +fi + +if [[ -n "$GENERATE_DOCS" ]]; then + echo "Generating documentation..." + (set -x; "$FENV_SCRIPT" "${FENV_FLAGS[@]}" --exec ./scripts/generate_docs.sh) fi +# Build the final fql image if requested if [[ -n "$IMAGE_FQL" ]]; then + echo "Building fql runtime image..." + FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FENV_FDB_VER}" + export FDB_DOCKER_IMAGE (set -x; docker buildx bake -f bake.hcl --load fql) fi +# Run fql interactively if requested if [[ -n "$RUN_FQL" ]]; then + echo "Running fql with args: ${FQL_ARGS[*]}" + FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FENV_FDB_VER}" + FQL_COMMAND="${FQL_ARGS[*]}" + export FDB_DOCKER_IMAGE FQL_COMMAND (set -x; docker compose run --rm fql 'docker:docker@{fdb}:4500' "${FQL_ARGS[@]}") fi diff --git a/docker/Dockerfile.fenv b/docker/Dockerfile.fenv new file mode 100644 index 00000000..69eaeab8 --- /dev/null +++ b/docker/Dockerfile.fenv @@ -0,0 +1,54 @@ +# Extended fenv image with FQL build tools +ARG FENV_DOCKER_TAG +FROM fenv:${FENV_DOCKER_TAG} + +# Install additional system packages needed for build +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + build-essential=12.9 \ + ca-certificates=20* \ + git=1:2.39.* \ + curl=7.88.* && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Install Go +ARG GO_URL="https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" +RUN curl -fsSL ${GO_URL} | tar -C /usr/local -xz +ENV PATH="/root/go/bin:/usr/local/go/bin:${PATH}" +ENV GOCACHE="/cache/gocache" +ENV GOMODCACHE="/cache/gomod" + +# Install golangci-lint +ARG GOLANGCI_LINT_URL="https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh" +ARG GOLANGCI_LINT_VER="v1.49.0" +RUN curl -fsSL ${GOLANGCI_LINT_URL} | sh -s -- -b "$(go env GOPATH)/bin" ${GOLANGCI_LINT_VER} +ENV GOLANGCI_LINT_CACHE="/cache/golangci-lint" + +# Install shellcheck +ARG SHELLCHECK_URL="https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz" +RUN curl -fsSL ${SHELLCHECK_URL} -o /tmp/shellcheck.tar.xz && \ + tar -xf /tmp/shellcheck.tar.xz -C /tmp && \ + mv /tmp/shellcheck-*/shellcheck /usr/local/bin && \ + rm -rf /tmp/shellcheck* + +# Install hadolint +ARG HADOLINT_URL="https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64" +RUN curl -fsSL ${HADOLINT_URL} -o /usr/local/bin/hadolint && \ + chmod +x /usr/local/bin/hadolint + +# Install jq +ARG JQ_URL="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" +RUN curl -fsSL ${JQ_URL} -o /usr/local/bin/jq && \ + chmod +x /usr/local/bin/jq + +# Install pandoc +ARG PANDOC_URL="https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" +RUN curl -fsSL ${PANDOC_URL} -o /tmp/pandoc.deb && \ + dpkg -i /tmp/pandoc.deb && \ + rm /tmp/pandoc.deb + +# Configure git to allow any user to run git commands on /src +# This allows the user which runs CI to be different from the +# user which built the Docker image. +RUN git config --global --add safe.directory /src diff --git a/fenv b/fenv new file mode 160000 index 00000000..387b628b --- /dev/null +++ b/fenv @@ -0,0 +1 @@ +Subproject commit 387b628b6ddef5e8104a36b9906588e474e5c170 From 0d013640e23a2a6bb7a6d753b958b72323040cf4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 27 Dec 2025 23:54:27 +0000 Subject: [PATCH 2/8] Consolidate Dockerfiles into single multi-stage build This commit eliminates duplication by merging docker/Dockerfile and docker/Dockerfile.fenv into a single multi-stage Dockerfile. Changes: - Refactor docker/Dockerfile with three stages: 1. fenv-builder: Extends fenv base image with FQL build tools 2. gobuild: Compiles the FQL binary 3. final: Minimal runtime image with FDB client and binary - Remove docker/Dockerfile.fenv (now obsolete) - Update build.sh to use consolidated Dockerfile with --target flag - Update bake.hcl to target fenv-builder stage and add FENV_DOCKER_TAG - Update CLAUDE.md to reflect consolidated build architecture Benefits: - Single source of truth for all Docker builds - No duplication of build tool installation - Same Dockerfile used by both fenv (development) and final image (runtime) - Easier to maintain and update dependencies --- CLAUDE.md | 7 +++- bake.hcl | 7 +++- build.sh | 3 +- docker/Dockerfile | 95 +++++++++++++++++++++++------------------- docker/Dockerfile.fenv | 54 ------------------------ 5 files changed, 65 insertions(+), 101 deletions(-) delete mode 100644 docker/Dockerfile.fenv diff --git a/CLAUDE.md b/CLAUDE.md index 51cdf034..84befbcd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,8 +80,11 @@ Query syntax uses directories (`/path/to/dir`), tuples `("elem1", 2, 0xFF)`, and The build system uses fenv for consistent builds: - `build.sh` script provides unified interface wrapping `fenv/fenv.sh` -- `docker/Dockerfile.fenv` extends fenv base image with project-specific build tools -- `bake.hcl` defines Docker build configuration for final fql image +- `docker/Dockerfile` is a multi-stage build: + - `fenv-builder` stage extends fenv base image with project-specific build tools + - `gobuild` stage compiles the FQL binary + - Final stage creates minimal runtime image +- `bake.hcl` defines Docker build configuration - `compose.yaml` defines runtime service for running fql image - fenv automatically manages FDB container for integration testing - Use `--latest` flag for offline development \ No newline at end of file diff --git a/bake.hcl b/bake.hcl index acc2a5ef..e24648d6 100644 --- a/bake.hcl +++ b/bake.hcl @@ -15,6 +15,10 @@ variable "FDB_LIB_URL" { default = "https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb" } +variable "FENV_DOCKER_TAG" { + default = "6.2.30" +} + variable "GO_URL" { default = "https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" } @@ -45,6 +49,7 @@ function "build_args" { result = { FQL_VER = DOCKER_TAG FDB_LIB_URL = FDB_LIB_URL + FENV_DOCKER_TAG = FENV_DOCKER_TAG GO_URL = GO_URL GOLANGCI_LINT_VER = GOLANGCI_LINT_VER SHELLCHECK_URL = SHELLCHECK_URL @@ -61,7 +66,7 @@ group "default" { target "build" { context = "./docker" dockerfile = "Dockerfile" - target = "builder" + target = "fenv-builder" tags = ["docker.io/janderland/fql-build:${DOCKER_TAG}"] platforms = ["linux/amd64"] args = build_args() diff --git a/build.sh b/build.sh index bdfd41ea..b3d320ad 100755 --- a/build.sh +++ b/build.sh @@ -153,7 +153,8 @@ FENV_SCRIPT="./fenv/fenv.sh" # Common fenv flags FENV_FLAGS=() -FENV_FLAGS+=(--docker ./docker/Dockerfile.fenv) +FENV_FLAGS+=(--docker ./docker/Dockerfile) +FENV_FLAGS+=(--target fenv-builder) # Run the requested commands. diff --git a/docker/Dockerfile b/docker/Dockerfile index 7bae9287..80273c82 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,65 +1,62 @@ -# 'builder' includes all build & testing dependencies. -FROM debian:12 AS builder +# 'fenv-builder' extends the fenv base image with FQL build & testing dependencies. +# This stage is used by fenv for local development and CI/CD verification tasks. +ARG FENV_DOCKER_TAG +FROM fenv:${FENV_DOCKER_TAG} AS fenv-builder -RUN apt-get update &&\ +# Install additional system packages needed for build +RUN apt-get update && \ apt-get install --no-install-recommends -y \ build-essential=12.9 \ ca-certificates=20* \ git=1:2.39.* \ - curl=7.88.* &&\ - apt-get clean &&\ + curl=7.88.* && \ + apt-get clean && \ rm -rf /var/lib/apt/lists/* -# The FDB installer is not deleted so it can be used to -# install the library in the final stage at the end of -# this file. -ARG FDB_LIB_URL -RUN curl -Lo /fdb.deb $FDB_LIB_URL &&\ - dpkg -i /fdb.deb - -ARG GO_URL -RUN curl -Lo /go.tar.gz $GO_URL &&\ - tar -C /usr/local -xzf /go.tar.gz &&\ - rm /go.tar.gz +# Install Go +ARG GO_URL="https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" +RUN curl -fsSL ${GO_URL} | tar -C /usr/local -xz ENV PATH="/root/go/bin:/usr/local/go/bin:${PATH}" ENV GOCACHE="/cache/gocache" ENV GOMODCACHE="/cache/gomod" +# Install golangci-lint ARG GOLANGCI_LINT_URL="https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh" -ARG GOLANGCI_LINT_VER -RUN curl -Lo /golint.sh $GOLANGCI_LINT_URL &&\ - sh /golint.sh -b "$(go env GOPATH)/bin" $GOLANGCI_LINT_VER &&\ - rm /golint.sh +ARG GOLANGCI_LINT_VER="v1.49.0" +RUN curl -fsSL ${GOLANGCI_LINT_URL} | sh -s -- -b "$(go env GOPATH)/bin" ${GOLANGCI_LINT_VER} ENV GOLANGCI_LINT_CACHE="/cache/golangci-lint" -ARG SHELLCHECK_URL -RUN curl -Lo /shellcheck.tar.xz $SHELLCHECK_URL &&\ - tar -xf /shellcheck.tar.xz &&\ - mv /shellcheck-*/shellcheck /usr/local/bin &&\ - rm -r /shellcheck.tar.xz /shellcheck-* +# Install shellcheck +ARG SHELLCHECK_URL="https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz" +RUN curl -fsSL ${SHELLCHECK_URL} -o /tmp/shellcheck.tar.xz && \ + tar -xf /tmp/shellcheck.tar.xz -C /tmp && \ + mv /tmp/shellcheck-*/shellcheck /usr/local/bin && \ + rm -rf /tmp/shellcheck* -ARG HADOLINT_URL -RUN curl -Lo /usr/local/bin/hadolint $HADOLINT_URL &&\ +# Install hadolint +ARG HADOLINT_URL="https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64" +RUN curl -fsSL ${HADOLINT_URL} -o /usr/local/bin/hadolint && \ chmod +x /usr/local/bin/hadolint -ARG JQ_URL -RUN curl -Lo /usr/local/bin/jq $JQ_URL &&\ +# Install jq +ARG JQ_URL="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" +RUN curl -fsSL ${JQ_URL} -o /usr/local/bin/jq && \ chmod +x /usr/local/bin/jq -ARG PANDOC_URL -RUN curl -Lo /pandoc.deb $PANDOC_URL &&\ - dpkg -i /pandoc.deb &&\ - rm /pandoc.deb +# Install pandoc +ARG PANDOC_URL="https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" +RUN curl -fsSL ${PANDOC_URL} -o /tmp/pandoc.deb && \ + dpkg -i /tmp/pandoc.deb && \ + rm /tmp/pandoc.deb -# Configure git so it allows any user to run git commands -# on the /fql directory. This allows the user which runs -# CI to be different from the user which built the Docker -# image. -RUN git config --global --add safe.directory /fql +# Configure git to allow any user to run git commands on /src +# This allows the user which runs CI to be different from the +# user which built the Docker image. +RUN git config --global --add safe.directory /src -# 'gobuild' executes 'go build'. -FROM builder AS gobuild +# 'gobuild' stage compiles the FQL binary. +FROM fenv-builder AS gobuild COPY . /src WORKDIR /src @@ -68,11 +65,23 @@ ARG FQL_VER RUN go build -o /fql -ldflags="-X 'github.com/janderland/fql/internal/app.Version=${FQL_VER}'" -# The final stage builds the 'fql' image. +# 'fdb-installer' stage downloads the FDB client .deb for the final stage. +FROM debian:12 AS fdb-installer + +RUN apt-get update && \ + apt-get install --no-install-recommends -y curl=7.88.* && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ARG FDB_LIB_URL +RUN curl -fsSL ${FDB_LIB_URL} -o /fdb.deb + + +# The final stage builds the minimal 'fql' runtime image. FROM debian:12 -COPY --from=gobuild /fdb.deb /fdb.deb -RUN dpkg -i ./fdb.deb &&\ +COPY --from=fdb-installer /fdb.deb /fdb.deb +RUN dpkg -i ./fdb.deb && \ rm /fdb.deb ENV TERM="xterm-256color" diff --git a/docker/Dockerfile.fenv b/docker/Dockerfile.fenv deleted file mode 100644 index 69eaeab8..00000000 --- a/docker/Dockerfile.fenv +++ /dev/null @@ -1,54 +0,0 @@ -# Extended fenv image with FQL build tools -ARG FENV_DOCKER_TAG -FROM fenv:${FENV_DOCKER_TAG} - -# Install additional system packages needed for build -RUN apt-get update && \ - apt-get install --no-install-recommends -y \ - build-essential=12.9 \ - ca-certificates=20* \ - git=1:2.39.* \ - curl=7.88.* && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Install Go -ARG GO_URL="https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" -RUN curl -fsSL ${GO_URL} | tar -C /usr/local -xz -ENV PATH="/root/go/bin:/usr/local/go/bin:${PATH}" -ENV GOCACHE="/cache/gocache" -ENV GOMODCACHE="/cache/gomod" - -# Install golangci-lint -ARG GOLANGCI_LINT_URL="https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh" -ARG GOLANGCI_LINT_VER="v1.49.0" -RUN curl -fsSL ${GOLANGCI_LINT_URL} | sh -s -- -b "$(go env GOPATH)/bin" ${GOLANGCI_LINT_VER} -ENV GOLANGCI_LINT_CACHE="/cache/golangci-lint" - -# Install shellcheck -ARG SHELLCHECK_URL="https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz" -RUN curl -fsSL ${SHELLCHECK_URL} -o /tmp/shellcheck.tar.xz && \ - tar -xf /tmp/shellcheck.tar.xz -C /tmp && \ - mv /tmp/shellcheck-*/shellcheck /usr/local/bin && \ - rm -rf /tmp/shellcheck* - -# Install hadolint -ARG HADOLINT_URL="https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64" -RUN curl -fsSL ${HADOLINT_URL} -o /usr/local/bin/hadolint && \ - chmod +x /usr/local/bin/hadolint - -# Install jq -ARG JQ_URL="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" -RUN curl -fsSL ${JQ_URL} -o /usr/local/bin/jq && \ - chmod +x /usr/local/bin/jq - -# Install pandoc -ARG PANDOC_URL="https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" -RUN curl -fsSL ${PANDOC_URL} -o /tmp/pandoc.deb && \ - dpkg -i /tmp/pandoc.deb && \ - rm /tmp/pandoc.deb - -# Configure git to allow any user to run git commands on /src -# This allows the user which runs CI to be different from the -# user which built the Docker image. -RUN git config --global --add safe.directory /src From 63c5a8ff6548748ed203f8a650ace5ccccde503e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 00:21:37 +0000 Subject: [PATCH 3/8] Address PR feedback - simplify build system This commit addresses all review comments from PR #242: 1. Rename 'fenv-builder' to 'builder' for consistency 2. Remove git config (fenv base image already handles this) 3. Remove jq installation (fenv handles database initialization) 4. Consolidate fdb-installer stage into final stage (simpler) 5. Remove bake.hcl (defaults already in Dockerfile) 6. Combine FENV_SCRIPT and FENV_FLAGS into FENV_CMD variable 7. Remove setup_database.sh (fenv handles this automatically) Changes: - docker/Dockerfile: Rename stage, remove jq, remove git config, consolidate FDB download into final stage - build.sh: Combine fenv variables into FENV_CMD, remove database setup call, replace bake with direct docker build - bake.hcl: Removed (no longer needed) - scripts/setup_database.sh: Removed (fenv handles this) - scripts/docker_tag.sh: Update comment (no longer references bake.hcl) - .github/workflows/verify.yml: Replace bake action with build.sh - CLAUDE.md: Update documentation to reflect changes Benefits: - Simpler build system with less duplication - Relies on fenv for database management - Direct docker build instead of bake abstraction - Fewer files to maintain --- .github/workflows/verify.yml | 11 +---- CLAUDE.md | 3 +- bake.hcl | 81 ------------------------------------ build.sh | 34 +++++++-------- docker/Dockerfile | 30 ++++--------- scripts/docker_tag.sh | 3 +- scripts/setup_database.sh | 45 -------------------- 7 files changed, 26 insertions(+), 181 deletions(-) delete mode 100644 bake.hcl delete mode 100755 scripts/setup_database.sh diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 633a2e9e..20cb7e27 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -31,18 +31,9 @@ jobs: FDB_VER: ${{ matrix.fdb_ver }} - name: Build image - uses: docker/bake-action@v5 + run: ./build.sh --image build env: - DOCKER_TAG: ${{ steps.docker.outputs.tag }} FDB_VER: ${{ matrix.fdb_ver }} - FDB_LIB_URL: ${{ matrix.fdb_lib_url }} - with: - files: bake.hcl - targets: build - load: true - set: | - *.cache-from=type=gha,scope=build-${{ matrix.fdb_ver }} - *.cache-to=type=gha,mode=max,scope=build-${{ matrix.fdb_ver }} - name: Restore cache uses: actions/cache@v4 diff --git a/CLAUDE.md b/CLAUDE.md index 84befbcd..f6ecb269 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,10 +81,9 @@ Query syntax uses directories (`/path/to/dir`), tuples `("elem1", 2, 0xFF)`, and The build system uses fenv for consistent builds: - `build.sh` script provides unified interface wrapping `fenv/fenv.sh` - `docker/Dockerfile` is a multi-stage build: - - `fenv-builder` stage extends fenv base image with project-specific build tools + - `builder` stage extends fenv base image with project-specific build tools - `gobuild` stage compiles the FQL binary - Final stage creates minimal runtime image -- `bake.hcl` defines Docker build configuration - `compose.yaml` defines runtime service for running fql image - fenv automatically manages FDB container for integration testing - Use `--latest` flag for offline development \ No newline at end of file diff --git a/bake.hcl b/bake.hcl deleted file mode 100644 index e24648d6..00000000 --- a/bake.hcl +++ /dev/null @@ -1,81 +0,0 @@ -# Docker Bake configuration for FQL. -# -# Variables can be overridden via environment variables. -# The CI workflow passes these as env vars from the matrix. - -variable "DOCKER_TAG" { - default = "latest" -} - -variable "FDB_VER" { - default = "6.2.30" -} - -variable "FDB_LIB_URL" { - default = "https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb" -} - -variable "FENV_DOCKER_TAG" { - default = "6.2.30" -} - -variable "GO_URL" { - default = "https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" -} - -variable "GOLANGCI_LINT_VER" { - default = "v1.49.0" -} - -variable "SHELLCHECK_URL" { - default = "https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz" -} - -variable "HADOLINT_URL" { - default = "https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64" -} - -variable "JQ_URL" { - default = "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" -} - -variable "PANDOC_URL" { - default = "https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" -} - -# Shared build arguments used by both targets. -function "build_args" { - params = [] - result = { - FQL_VER = DOCKER_TAG - FDB_LIB_URL = FDB_LIB_URL - FENV_DOCKER_TAG = FENV_DOCKER_TAG - GO_URL = GO_URL - GOLANGCI_LINT_VER = GOLANGCI_LINT_VER - SHELLCHECK_URL = SHELLCHECK_URL - HADOLINT_URL = HADOLINT_URL - JQ_URL = JQ_URL - PANDOC_URL = PANDOC_URL - } -} - -group "default" { - targets = ["build", "fql"] -} - -target "build" { - context = "./docker" - dockerfile = "Dockerfile" - target = "fenv-builder" - tags = ["docker.io/janderland/fql-build:${DOCKER_TAG}"] - platforms = ["linux/amd64"] - args = build_args() -} - -target "fql" { - context = "." - dockerfile = "./docker/Dockerfile" - tags = ["docker.io/janderland/fql:${DOCKER_TAG}"] - platforms = ["linux/amd64"] - args = build_args() -} diff --git a/build.sh b/build.sh index b3d320ad..645bcbe6 100755 --- a/build.sh +++ b/build.sh @@ -148,13 +148,8 @@ export DOCKER_TAG export FENV_FDB_VER="${FDB_VER:-6.2.30}" export FENV_PROJECT_NAME="fql" -# fenv.sh script location -FENV_SCRIPT="./fenv/fenv.sh" - -# Common fenv flags -FENV_FLAGS=() -FENV_FLAGS+=(--docker ./docker/Dockerfile) -FENV_FLAGS+=(--target fenv-builder) +# fenv command with common flags +FENV_CMD=(./fenv/fenv.sh --docker ./docker/Dockerfile --target builder) # Run the requested commands. @@ -162,38 +157,41 @@ FENV_FLAGS+=(--target fenv-builder) # Build the extended fenv image if requested if [[ -n "$IMAGE_BUILD" ]]; then echo "Building extended fenv image..." - (set -x; "$FENV_SCRIPT" "${FENV_FLAGS[@]}" --build) + (set -x; "${FENV_CMD[@]}" --build) fi # Execute build tasks (generate, verify, docs) using fenv if [[ -n "$VERIFY_GENERATION" ]]; then echo "Verifying code generation..." - (set -x; "$FENV_SCRIPT" "${FENV_FLAGS[@]}" --exec ./scripts/verify_generation.sh) + (set -x; "${FENV_CMD[@]}" --exec ./scripts/verify_generation.sh) fi if [[ -n "$VERIFY_CODEBASE" ]]; then - echo "Setting up database and verifying codebase..." - (set -x; "$FENV_SCRIPT" "${FENV_FLAGS[@]}" --exec ./scripts/setup_database.sh --exec ./scripts/verify_codebase.sh) + echo "Verifying codebase..." + (set -x; "${FENV_CMD[@]}" --exec ./scripts/verify_codebase.sh) fi if [[ -n "$GENERATE_DOCS" ]]; then echo "Generating documentation..." - (set -x; "$FENV_SCRIPT" "${FENV_FLAGS[@]}" --exec ./scripts/generate_docs.sh) + (set -x; "${FENV_CMD[@]}" --exec ./scripts/generate_docs.sh) fi # Build the final fql image if requested if [[ -n "$IMAGE_FQL" ]]; then echo "Building fql runtime image..." - FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FENV_FDB_VER}" - export FDB_DOCKER_IMAGE - (set -x; docker buildx bake -f bake.hcl --load fql) + FDB_LIB_URL="https://github.com/apple/foundationdb/releases/download/${FENV_FDB_VER}/foundationdb-clients_${FENV_FDB_VER}-1_amd64.deb" + (set -x; docker build \ + -f ./docker/Dockerfile \ + -t "docker.io/janderland/fql:${DOCKER_TAG}" \ + --build-arg "FQL_VER=${DOCKER_TAG}" \ + --build-arg "FDB_LIB_URL=${FDB_LIB_URL}" \ + --build-arg "FENV_DOCKER_TAG=${FENV_FDB_VER}" \ + .) fi # Run fql interactively if requested if [[ -n "$RUN_FQL" ]]; then echo "Running fql with args: ${FQL_ARGS[*]}" - FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FENV_FDB_VER}" - FQL_COMMAND="${FQL_ARGS[*]}" - export FDB_DOCKER_IMAGE FQL_COMMAND + export FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FENV_FDB_VER}" (set -x; docker compose run --rm fql 'docker:docker@{fdb}:4500' "${FQL_ARGS[@]}") fi diff --git a/docker/Dockerfile b/docker/Dockerfile index 80273c82..5948db59 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ -# 'fenv-builder' extends the fenv base image with FQL build & testing dependencies. +# 'builder' extends the fenv base image with FQL build & testing dependencies. # This stage is used by fenv for local development and CI/CD verification tasks. ARG FENV_DOCKER_TAG -FROM fenv:${FENV_DOCKER_TAG} AS fenv-builder +FROM fenv:${FENV_DOCKER_TAG} AS builder # Install additional system packages needed for build RUN apt-get update && \ @@ -38,25 +38,15 @@ ARG HADOLINT_URL="https://github.com/hadolint/hadolint/releases/download/v2.7.0/ RUN curl -fsSL ${HADOLINT_URL} -o /usr/local/bin/hadolint && \ chmod +x /usr/local/bin/hadolint -# Install jq -ARG JQ_URL="https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" -RUN curl -fsSL ${JQ_URL} -o /usr/local/bin/jq && \ - chmod +x /usr/local/bin/jq - # Install pandoc ARG PANDOC_URL="https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" RUN curl -fsSL ${PANDOC_URL} -o /tmp/pandoc.deb && \ dpkg -i /tmp/pandoc.deb && \ rm /tmp/pandoc.deb -# Configure git to allow any user to run git commands on /src -# This allows the user which runs CI to be different from the -# user which built the Docker image. -RUN git config --global --add safe.directory /src - # 'gobuild' stage compiles the FQL binary. -FROM fenv-builder AS gobuild +FROM builder AS gobuild COPY . /src WORKDIR /src @@ -65,8 +55,8 @@ ARG FQL_VER RUN go build -o /fql -ldflags="-X 'github.com/janderland/fql/internal/app.Version=${FQL_VER}'" -# 'fdb-installer' stage downloads the FDB client .deb for the final stage. -FROM debian:12 AS fdb-installer +# The final stage builds the minimal 'fql' runtime image. +FROM debian:12 RUN apt-get update && \ apt-get install --no-install-recommends -y curl=7.88.* && \ @@ -74,14 +64,8 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* ARG FDB_LIB_URL -RUN curl -fsSL ${FDB_LIB_URL} -o /fdb.deb - - -# The final stage builds the minimal 'fql' runtime image. -FROM debian:12 - -COPY --from=fdb-installer /fdb.deb /fdb.deb -RUN dpkg -i ./fdb.deb && \ +RUN curl -fsSL ${FDB_LIB_URL} -o /fdb.deb && \ + dpkg -i /fdb.deb && \ rm /fdb.deb ENV TERM="xterm-256color" diff --git a/scripts/docker_tag.sh b/scripts/docker_tag.sh index 2f96dc78..c0277409 100755 --- a/scripts/docker_tag.sh +++ b/scripts/docker_tag.sh @@ -23,8 +23,7 @@ function code_version { } # fdb_version returns the version of the FDB library. -# Uses FDB_VER env var if set, otherwise defaults to -# the version specified in bake.hcl. +# Uses FDB_VER env var if set, otherwise defaults to 6.2.30. function fdb_version { echo "${FDB_VER:-6.2.30}" diff --git a/scripts/setup_database.sh b/scripts/setup_database.sh deleted file mode 100755 index 18dc6b11..00000000 --- a/scripts/setup_database.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -ue - -# The first argument is the hostname of the FDB container. -FDB_HOSTNAME=${1:-fdb} -echo "FDB_HOSTNAME=$FDB_HOSTNAME" - -# The second argument is the description & ID of the FDB cluster. -# https://apple.github.io/foundationdb/administration.html#cluster-file-format -FDB_DESCRIPTION_ID=${2:-docker:docker} -echo "FDB_DESCRIPTION_ID=$FDB_DESCRIPTION_ID" - -# Obtain the IP for FDB from the given hostname. -FDB_IP=$(getent hosts "$FDB_HOSTNAME" | awk '{print $1}') -echo "FDB_IP=$FDB_IP" - -# This variable is recognised by fdbcli and ensures that a fdb.cluster file in -# the current directory won't interfere with this script. -export FDB_CLUSTER_FILE="/etc/foundationdb/fdb.cluster" - -# Create the FDB cluster file. -echo "${FDB_DESCRIPTION_ID}@${FDB_IP}:4500" > $FDB_CLUSTER_FILE -echo "FDB_CLUSTER_FILE: $(cat $FDB_CLUSTER_FILE)" - -# Search for the "unreadable_configuration" message in the cluster's status. This message -# would let us know that the database hasn't been initialized. We need to disable the '-e' -# flag so that bash won't exit if jq returns a non-zero code. -JQ_CODE=0 -jq -e '.cluster.messages[] | select(.name | contains("unreadable_configuration"))' <(set -x; fdbcli --exec 'status json') || JQ_CODE=$? - -# jq should only return codes between 0 & 4 inclusive. Our particular query never -# returns 'null' or 'false', so we shouldn't see code 1. Codes 2 & 3 occur on -# system & compile errors respectively, so the only valid codes are 0 & 4. If the -# code is not 0 or 4 then something unexpected happened so return early. -# https://stedolan.github.io/jq/manual/#Invokingjq -if [[ $JQ_CODE -lt 0 || ( $JQ_CODE -gt 0 && $JQ_CODE -lt 4 ) || $JQ_CODE -gt 4 ]]; then - echo "ERR! Unexpected jq exit code $JQ_CODE" - exit "$JQ_CODE" -fi - -# If this is a new instance of FDB, configure the database. -# https://apple.github.io/foundationdb/administration.html#re-creating-a-database -if [[ $JQ_CODE -eq 0 ]]; then - (set -x; fdbcli --exec "configure new single memory") -fi From ad7355667b1ef17301c8a18cb9c93efcf1b92043 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 00:35:18 +0000 Subject: [PATCH 4/8] Address additional PR feedback This commit addresses the second round of PR feedback: 1. Remove redundant installations from Dockerfile - fenv base image already provides: build-essential, ca-certificates, git, curl, shellcheck, hadolint - Only install what's unique to FQL: Go, golangci-lint, pandoc 2. Use FENV_FDB_VER everywhere instead of FDB_VER - Replace FDB_VER with FENV_FDB_VER in build.sh - Update GitHub workflow to use FENV_FDB_VER - Generate FDB_LIB_URL dynamically in Dockerfile from FENV_FDB_VER 3. Remove scripts/docker_tag.sh - Use fenv's docker_tag.sh instead - Update build.sh to call ./fenv/docker_tag.sh 4. Simplify build.sh help message - Remove explanatory text about fenv Changes: - docker/Dockerfile: Remove redundant apt-get, shellcheck, hadolint; generate FDB URL from FENV_FDB_VER instead of taking FDB_LIB_URL param - build.sh: Use FENV_FDB_VER, call fenv/docker_tag.sh, remove fenv explanation from help, remove FDB_LIB_URL build arg - scripts/docker_tag.sh: Removed (use fenv's version) - .github/workflows/verify.yml: Replace FDB_VER with FENV_FDB_VER Benefits: - Simpler Dockerfile with less duplication - Consistent use of FENV_FDB_VER throughout - Leverage fenv's versioning infrastructure --- .github/workflows/verify.yml | 10 ++-------- build.sh | 16 ++++++---------- docker/Dockerfile | 26 ++------------------------ scripts/docker_tag.sh | 32 -------------------------------- 4 files changed, 10 insertions(+), 74 deletions(-) delete mode 100755 scripts/docker_tag.sh diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 20cb7e27..047caf3d 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -24,16 +24,10 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Compute Docker tag - id: docker - run: echo "tag=$(./scripts/docker_tag.sh)" >> $GITHUB_OUTPUT - env: - FDB_VER: ${{ matrix.fdb_ver }} - - name: Build image run: ./build.sh --image build env: - FDB_VER: ${{ matrix.fdb_ver }} + FENV_FDB_VER: ${{ matrix.fdb_ver }} - name: Restore cache uses: actions/cache@v4 @@ -64,7 +58,7 @@ jobs: - name: Verify run: ./build.sh --generate --verify env: - FDB_VER: ${{ matrix.fdb_ver }} + FENV_FDB_VER: ${{ matrix.fdb_ver }} CACHE_VOLUME: ~/.cache/fql - name: Fix cache permissions diff --git a/build.sh b/build.sh index 645bcbe6..36fd66a9 100755 --- a/build.sh +++ b/build.sh @@ -43,9 +43,6 @@ Multiple image names can be specified on the '--image' flag by separating them with commas. ./build.sh --image build,fql - -This script now uses fenv to provide a consistent build environment -with FoundationDB integration. END } @@ -140,14 +137,14 @@ done # Build variables for fenv and docker commands. -DOCKER_TAG="$(./scripts/docker_tag.sh)" -echo "DOCKER_TAG=${DOCKER_TAG}" -export DOCKER_TAG - # Set fenv environment variables -export FENV_FDB_VER="${FDB_VER:-6.2.30}" +export FENV_FDB_VER="${FENV_FDB_VER:-6.2.30}" export FENV_PROJECT_NAME="fql" +DOCKER_TAG="$(./fenv/docker_tag.sh)" +echo "DOCKER_TAG=${DOCKER_TAG}" +export DOCKER_TAG + # fenv command with common flags FENV_CMD=(./fenv/fenv.sh --docker ./docker/Dockerfile --target builder) @@ -179,12 +176,11 @@ fi # Build the final fql image if requested if [[ -n "$IMAGE_FQL" ]]; then echo "Building fql runtime image..." - FDB_LIB_URL="https://github.com/apple/foundationdb/releases/download/${FENV_FDB_VER}/foundationdb-clients_${FENV_FDB_VER}-1_amd64.deb" (set -x; docker build \ -f ./docker/Dockerfile \ -t "docker.io/janderland/fql:${DOCKER_TAG}" \ --build-arg "FQL_VER=${DOCKER_TAG}" \ - --build-arg "FDB_LIB_URL=${FDB_LIB_URL}" \ + --build-arg "FENV_FDB_VER=${FENV_FDB_VER}" \ --build-arg "FENV_DOCKER_TAG=${FENV_FDB_VER}" \ .) fi diff --git a/docker/Dockerfile b/docker/Dockerfile index 5948db59..9674bb01 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -3,16 +3,6 @@ ARG FENV_DOCKER_TAG FROM fenv:${FENV_DOCKER_TAG} AS builder -# Install additional system packages needed for build -RUN apt-get update && \ - apt-get install --no-install-recommends -y \ - build-essential=12.9 \ - ca-certificates=20* \ - git=1:2.39.* \ - curl=7.88.* && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - # Install Go ARG GO_URL="https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" RUN curl -fsSL ${GO_URL} | tar -C /usr/local -xz @@ -26,18 +16,6 @@ ARG GOLANGCI_LINT_VER="v1.49.0" RUN curl -fsSL ${GOLANGCI_LINT_URL} | sh -s -- -b "$(go env GOPATH)/bin" ${GOLANGCI_LINT_VER} ENV GOLANGCI_LINT_CACHE="/cache/golangci-lint" -# Install shellcheck -ARG SHELLCHECK_URL="https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz" -RUN curl -fsSL ${SHELLCHECK_URL} -o /tmp/shellcheck.tar.xz && \ - tar -xf /tmp/shellcheck.tar.xz -C /tmp && \ - mv /tmp/shellcheck-*/shellcheck /usr/local/bin && \ - rm -rf /tmp/shellcheck* - -# Install hadolint -ARG HADOLINT_URL="https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64" -RUN curl -fsSL ${HADOLINT_URL} -o /usr/local/bin/hadolint && \ - chmod +x /usr/local/bin/hadolint - # Install pandoc ARG PANDOC_URL="https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" RUN curl -fsSL ${PANDOC_URL} -o /tmp/pandoc.deb && \ @@ -63,8 +41,8 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -ARG FDB_LIB_URL -RUN curl -fsSL ${FDB_LIB_URL} -o /fdb.deb && \ +ARG FENV_FDB_VER +RUN curl -fsSL "https://github.com/apple/foundationdb/releases/download/${FENV_FDB_VER}/foundationdb-clients_${FENV_FDB_VER}-1_amd64.deb" -o /fdb.deb && \ dpkg -i /fdb.deb && \ rm /fdb.deb diff --git a/scripts/docker_tag.sh b/scripts/docker_tag.sh deleted file mode 100755 index c0277409..00000000 --- a/scripts/docker_tag.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# docker_tag.sh outputs the Docker image tag based on -# the git version and FDB version. This script is used -# by build.sh and CI workflows to ensure consistent tags. -# -# Environment variables: -# LATEST - If set, uses "latest" instead of git version -# FDB_VER - FDB version (defaults to 6.2.30) - -# code_version returns the latest tag for the current -# Git commit. If there are no tags associated with -# the commit then the short hash is returned. - -function code_version { - local tag="" - if tag="$(git describe --tags 2>/dev/null)"; then - echo "$tag" - return 0 - fi - git rev-parse --short HEAD -} - -# fdb_version returns the version of the FDB library. -# Uses FDB_VER env var if set, otherwise defaults to 6.2.30. - -function fdb_version { - echo "${FDB_VER:-6.2.30}" -} - -echo "${LATEST:-$(code_version)}_fdb.$(fdb_version)" From c34cae1564d49e2f64f4a93152fcd928e304a28d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 00:36:47 +0000 Subject: [PATCH 5/8] Fix workflow to checkout submodules The workflow was failing because it didn't checkout the fenv submodule, which is now required since build.sh calls: - ./fenv/docker_tag.sh for versioning - ./fenv/fenv.sh for build tasks Changes: - Add 'submodules: true' to checkout action - Remove unused fdb_lib_url from matrix (no longer needed) This fixes the workflow build errors. --- .github/workflows/verify.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 047caf3d..4666a3d8 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -9,17 +9,13 @@ jobs: strategy: matrix: fdb_ver: [6.2.30, 7.1.61] - include: - - fdb_ver: 6.2.30 - fdb_lib_url: https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb - - - fdb_ver: 7.1.61 - fdb_lib_url: https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients_7.1.61-1_amd64.deb runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 8e48530ed72c12099da9e1a6f061db9912f96b39 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 01:24:30 +0000 Subject: [PATCH 6/8] Fix workflow build failure - use Dockerfile.builder The workflow was failing with "Invalid flag '--target'" because fenv.sh doesn't support the --target flag. Root cause: - build.sh was calling: fenv.sh --docker ./docker/Dockerfile --target builder - fenv.sh only supports: --build, --exec, --down, --docker FILE - No --target flag exists in fenv.sh Solution: - Create separate docker/Dockerfile.builder with just the builder stage - Update build.sh to use --docker ./docker/Dockerfile.builder - Remove the unsupported --target flag This allows fenv.sh to build the correct image (builder with all build tools) without needing target selection. Changes: - docker/Dockerfile.builder: New file with builder stage only - build.sh: Use Dockerfile.builder, remove --target flag - CLAUDE.md: Update documentation to reflect new structure This fixes the workflow build errors. --- CLAUDE.md | 8 ++++---- build.sh | 2 +- docker/Dockerfile.builder | 23 +++++++++++++++++++++++ 3 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 docker/Dockerfile.builder diff --git a/CLAUDE.md b/CLAUDE.md index f6ecb269..ddf474e9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,10 +80,10 @@ Query syntax uses directories (`/path/to/dir`), tuples `("elem1", 2, 0xFF)`, and The build system uses fenv for consistent builds: - `build.sh` script provides unified interface wrapping `fenv/fenv.sh` -- `docker/Dockerfile` is a multi-stage build: - - `builder` stage extends fenv base image with project-specific build tools +- `docker/Dockerfile.builder` extends fenv base image with project-specific build tools (Go, golangci-lint, pandoc) +- `docker/Dockerfile` is a multi-stage build for the final fql runtime image: + - `builder` stage extends fenv base image (same as Dockerfile.builder) - `gobuild` stage compiles the FQL binary - Final stage creates minimal runtime image - `compose.yaml` defines runtime service for running fql image -- fenv automatically manages FDB container for integration testing -- Use `--latest` flag for offline development \ No newline at end of file +- fenv automatically manages FDB container for integration testing \ No newline at end of file diff --git a/build.sh b/build.sh index 36fd66a9..39a1e81a 100755 --- a/build.sh +++ b/build.sh @@ -146,7 +146,7 @@ echo "DOCKER_TAG=${DOCKER_TAG}" export DOCKER_TAG # fenv command with common flags -FENV_CMD=(./fenv/fenv.sh --docker ./docker/Dockerfile --target builder) +FENV_CMD=(./fenv/fenv.sh --docker ./docker/Dockerfile.builder) # Run the requested commands. diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder new file mode 100644 index 00000000..06cf910c --- /dev/null +++ b/docker/Dockerfile.builder @@ -0,0 +1,23 @@ +# Builder image extends the fenv base image with FQL build & testing dependencies. +# This image is used by fenv for local development and CI/CD verification tasks. +ARG FENV_DOCKER_TAG +FROM fenv:${FENV_DOCKER_TAG} + +# Install Go +ARG GO_URL="https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" +RUN curl -fsSL ${GO_URL} | tar -C /usr/local -xz +ENV PATH="/root/go/bin:/usr/local/go/bin:${PATH}" +ENV GOCACHE="/cache/gocache" +ENV GOMODCACHE="/cache/gomod" + +# Install golangci-lint +ARG GOLANGCI_LINT_URL="https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh" +ARG GOLANGCI_LINT_VER="v1.49.0" +RUN curl -fsSL ${GOLANGCI_LINT_URL} | sh -s -- -b "$(go env GOPATH)/bin" ${GOLANGCI_LINT_VER} +ENV GOLANGCI_LINT_CACHE="/cache/golangci-lint" + +# Install pandoc +ARG PANDOC_URL="https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" +RUN curl -fsSL ${PANDOC_URL} -o /tmp/pandoc.deb && \ + dpkg -i /tmp/pandoc.deb && \ + rm /tmp/pandoc.deb From 2f6e9ca64c25426b5c63149891c2f9960d90c14f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 02:16:56 +0000 Subject: [PATCH 7/8] Simplify Dockerfile - remove duplicate builder stage Fixed docker/Dockerfile to eliminate duplication and use the pre-built fenv-ext-fql builder image instead of rebuilding its own builder stage. Problem: - docker/Dockerfile.builder defines the builder stage (for fenv dev/test) - docker/Dockerfile also had its own builder stage (duplicate definition) - When building final fql image, docker/Dockerfile tried to use FROM fenv:${FENV_DOCKER_TAG} but the tag was incorrect Solution: - Remove the duplicate builder stage from docker/Dockerfile - Use the pre-built fenv-ext-fql image built by --image build - Pass FENV_EXT_DOCKER_TAG (not FENV_DOCKER_TAG) to docker build Changes: - docker/Dockerfile: Remove builder stage, use fenv-ext-fql directly - build.sh: Pass FENV_EXT_DOCKER_TAG=${DOCKER_TAG} to docker build - CLAUDE.md: Update documentation to clarify the image relationship Benefits: - No duplication between Dockerfile.builder and Dockerfile - Simpler build process - builder is built once via fenv - Correct image tags are used throughout --- CLAUDE.md | 6 +++--- build.sh | 2 +- docker/Dockerfile | 31 ++++--------------------------- 3 files changed, 8 insertions(+), 31 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ddf474e9..e0e0d7d6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -81,9 +81,9 @@ Query syntax uses directories (`/path/to/dir`), tuples `("elem1", 2, 0xFF)`, and The build system uses fenv for consistent builds: - `build.sh` script provides unified interface wrapping `fenv/fenv.sh` - `docker/Dockerfile.builder` extends fenv base image with project-specific build tools (Go, golangci-lint, pandoc) + - Built via `./build.sh --image build` which creates `fenv-ext-fql` image - `docker/Dockerfile` is a multi-stage build for the final fql runtime image: - - `builder` stage extends fenv base image (same as Dockerfile.builder) - - `gobuild` stage compiles the FQL binary - - Final stage creates minimal runtime image + - `gobuild` stage uses the pre-built `fenv-ext-fql` builder image to compile the FQL binary + - Final stage creates minimal runtime image with just the binary and FDB client - `compose.yaml` defines runtime service for running fql image - fenv automatically manages FDB container for integration testing \ No newline at end of file diff --git a/build.sh b/build.sh index 39a1e81a..8a99a60b 100755 --- a/build.sh +++ b/build.sh @@ -181,7 +181,7 @@ if [[ -n "$IMAGE_FQL" ]]; then -t "docker.io/janderland/fql:${DOCKER_TAG}" \ --build-arg "FQL_VER=${DOCKER_TAG}" \ --build-arg "FENV_FDB_VER=${FENV_FDB_VER}" \ - --build-arg "FENV_DOCKER_TAG=${FENV_FDB_VER}" \ + --build-arg "FENV_EXT_DOCKER_TAG=${DOCKER_TAG}" \ .) fi diff --git a/docker/Dockerfile b/docker/Dockerfile index 9674bb01..230f6f24 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,30 +1,7 @@ -# 'builder' extends the fenv base image with FQL build & testing dependencies. -# This stage is used by fenv for local development and CI/CD verification tasks. -ARG FENV_DOCKER_TAG -FROM fenv:${FENV_DOCKER_TAG} AS builder - -# Install Go -ARG GO_URL="https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" -RUN curl -fsSL ${GO_URL} | tar -C /usr/local -xz -ENV PATH="/root/go/bin:/usr/local/go/bin:${PATH}" -ENV GOCACHE="/cache/gocache" -ENV GOMODCACHE="/cache/gomod" - -# Install golangci-lint -ARG GOLANGCI_LINT_URL="https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh" -ARG GOLANGCI_LINT_VER="v1.49.0" -RUN curl -fsSL ${GOLANGCI_LINT_URL} | sh -s -- -b "$(go env GOPATH)/bin" ${GOLANGCI_LINT_VER} -ENV GOLANGCI_LINT_CACHE="/cache/golangci-lint" - -# Install pandoc -ARG PANDOC_URL="https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" -RUN curl -fsSL ${PANDOC_URL} -o /tmp/pandoc.deb && \ - dpkg -i /tmp/pandoc.deb && \ - rm /tmp/pandoc.deb - - -# 'gobuild' stage compiles the FQL binary. -FROM builder AS gobuild +# 'gobuild' stage compiles the FQL binary using the pre-built builder image from fenv. +# The builder image (fenv-ext-fql) is built separately via ./build.sh --image build +ARG FENV_EXT_DOCKER_TAG +FROM fenv-ext-fql:${FENV_EXT_DOCKER_TAG} AS gobuild COPY . /src WORKDIR /src From a99fa6a2cb262b906cfe87c154d1948e6bb71e6f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 28 Dec 2025 02:26:31 +0000 Subject: [PATCH 8/8] Remove redundant cache restore step from workflow As noted in PR feedback, fenv handles cache management internally via the CACHE_VOLUME environment variable. The manual "Restore cache" step for ~/.cache/fql is redundant because: 1. fenv.sh mounts CACHE_VOLUME (~/cache/fql) to /cache in the container 2. This directory persists on the GitHub Actions runner between steps 3. Go build artifacts and golangci-lint cache are stored there 4. No explicit cache restore/save action is needed The FDB image caching is kept as it serves a different purpose: caching the Docker image itself to avoid re-pulling from Docker Hub. Changes: - Remove "Restore cache" step from workflow - Keep CACHE_VOLUME env var for verify step - Keep FDB image caching steps (different optimization) --- .github/workflows/verify.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 4666a3d8..74ae654d 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -25,14 +25,6 @@ jobs: env: FENV_FDB_VER: ${{ matrix.fdb_ver }} - - name: Restore cache - uses: actions/cache@v4 - with: - path: ~/.cache/fql - key: go-${{ matrix.fdb_ver }}-${{ hashFiles('go.sum') }} - restore-keys: | - go-${{ matrix.fdb_ver }}- - - name: Cache FDB image id: cache-fdb uses: actions/cache@v4