diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 633a2e9e..74ae654d 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -9,48 +9,21 @@ jobs: strategy: matrix: fdb_ver: [6.2.30, 7.1.61] - include: - - fdb_ver: 6.2.30 - fdb_lib_url: https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb - - - fdb_ver: 7.1.61 - fdb_lib_url: https://github.com/apple/foundationdb/releases/download/7.1.61/foundationdb-clients_7.1.61-1_amd64.deb runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Compute Docker tag - id: docker - run: echo "tag=$(./scripts/docker_tag.sh)" >> $GITHUB_OUTPUT - env: - FDB_VER: ${{ matrix.fdb_ver }} - - name: Build image - uses: docker/bake-action@v5 + run: ./build.sh --image build env: - DOCKER_TAG: ${{ steps.docker.outputs.tag }} - FDB_VER: ${{ matrix.fdb_ver }} - FDB_LIB_URL: ${{ matrix.fdb_lib_url }} - with: - files: bake.hcl - targets: build - load: true - set: | - *.cache-from=type=gha,scope=build-${{ matrix.fdb_ver }} - *.cache-to=type=gha,mode=max,scope=build-${{ matrix.fdb_ver }} - - - name: Restore cache - uses: actions/cache@v4 - with: - path: ~/.cache/fql - key: go-${{ matrix.fdb_ver }}-${{ hashFiles('go.sum') }} - restore-keys: | - go-${{ matrix.fdb_ver }}- + FENV_FDB_VER: ${{ matrix.fdb_ver }} - name: Cache FDB image id: cache-fdb @@ -73,7 +46,7 @@ jobs: - name: Verify run: ./build.sh --generate --verify env: - FDB_VER: ${{ matrix.fdb_ver }} + FENV_FDB_VER: ${{ matrix.fdb_ver }} CACHE_VOLUME: ~/.cache/fql - name: Fix cache permissions diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..a2e8c4cc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "fenv"] + path = fenv + url = https://github.com/janderland/fenv.git diff --git a/CLAUDE.md b/CLAUDE.md index df9f8892..e0e0d7d6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ FQL is a query language and alternative client API for FoundationDB written in G - **Run FQL interactively**: `./build.sh --run [args]` ### Docker-based Development -The build system uses Docker Compose for consistent builds and testing with FoundationDB integration. Always use `./build.sh --verify` instead of running Go commands directly to ensure proper environment setup. +The build system uses fenv (https://github.com/janderland/fenv), a Docker-based FoundationDB development environment, for consistent builds and testing with FoundationDB integration. Always use `./build.sh --verify` instead of running Go commands directly to ensure proper environment setup. ## Architecture @@ -78,9 +78,12 @@ Query syntax uses directories (`/path/to/dir`), tuples `("elem1", 2, 0xFF)`, and ## Docker Environment -The build system uses Docker Compose for consistent builds: -- `build.sh` script provides unified interface -- `bake.hcl` defines Docker build configuration -- `compose.yaml` defines runtime services (build, fql, fdb containers) -- Use `--latest` flag for offline development -- FDB container automatically managed for tests \ No newline at end of file +The build system uses fenv for consistent builds: +- `build.sh` script provides unified interface wrapping `fenv/fenv.sh` +- `docker/Dockerfile.builder` extends fenv base image with project-specific build tools (Go, golangci-lint, pandoc) + - Built via `./build.sh --image build` which creates `fenv-ext-fql` image +- `docker/Dockerfile` is a multi-stage build for the final fql runtime image: + - `gobuild` stage uses the pre-built `fenv-ext-fql` builder image to compile the FQL binary + - Final stage creates minimal runtime image with just the binary and FDB client +- `compose.yaml` defines runtime service for running fql image +- fenv automatically manages FDB container for integration testing \ No newline at end of file diff --git a/bake.hcl b/bake.hcl deleted file mode 100644 index acc2a5ef..00000000 --- a/bake.hcl +++ /dev/null @@ -1,76 +0,0 @@ -# Docker Bake configuration for FQL. -# -# Variables can be overridden via environment variables. -# The CI workflow passes these as env vars from the matrix. - -variable "DOCKER_TAG" { - default = "latest" -} - -variable "FDB_VER" { - default = "6.2.30" -} - -variable "FDB_LIB_URL" { - default = "https://github.com/apple/foundationdb/releases/download/6.2.30/foundationdb-clients_6.2.30-1_amd64.deb" -} - -variable "GO_URL" { - default = "https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" -} - -variable "GOLANGCI_LINT_VER" { - default = "v1.49.0" -} - -variable "SHELLCHECK_URL" { - default = "https://github.com/koalaman/shellcheck/releases/download/v0.10.0/shellcheck-v0.10.0.linux.x86_64.tar.xz" -} - -variable "HADOLINT_URL" { - default = "https://github.com/hadolint/hadolint/releases/download/v2.7.0/hadolint-Linux-x86_64" -} - -variable "JQ_URL" { - default = "https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64" -} - -variable "PANDOC_URL" { - default = "https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" -} - -# Shared build arguments used by both targets. -function "build_args" { - params = [] - result = { - FQL_VER = DOCKER_TAG - FDB_LIB_URL = FDB_LIB_URL - GO_URL = GO_URL - GOLANGCI_LINT_VER = GOLANGCI_LINT_VER - SHELLCHECK_URL = SHELLCHECK_URL - HADOLINT_URL = HADOLINT_URL - JQ_URL = JQ_URL - PANDOC_URL = PANDOC_URL - } -} - -group "default" { - targets = ["build", "fql"] -} - -target "build" { - context = "./docker" - dockerfile = "Dockerfile" - target = "builder" - tags = ["docker.io/janderland/fql-build:${DOCKER_TAG}"] - platforms = ["linux/amd64"] - args = build_args() -} - -target "fql" { - context = "." - dockerfile = "./docker/Dockerfile" - tags = ["docker.io/janderland/fql:${DOCKER_TAG}"] - platforms = ["linux/amd64"] - args = build_args() -} diff --git a/build.sh b/build.sh index a2d6d7fe..8a99a60b 100755 --- a/build.sh +++ b/build.sh @@ -4,12 +4,12 @@ set -eo pipefail function print_help { cat << END -build.sh is a facade for docker compose. It runs a set of -optional tasks in the order specified below. This is the same -script used by CI/CD to build, test, and package FQL. +build.sh is a facade for fenv (FoundationDB development environment). +It runs a set of optional tasks in the order specified below. This is +the same script used by CI/CD to build, test, and package FQL. If the '--image build' flag is set then the script starts off by -running 'docker build' for the 'fql-build' docker image. The tag +building the extended fenv image with all build tools. The tag is determined by the git tag/hash and the FDB version. This image is used to run the 'generate' and 'verify' tasks below. @@ -18,7 +18,7 @@ code generated by 'go generate ./...' is up to date. If the '--verify' flag is set then the script builds, lints, and tests the codebase. This task interacts with an FDB docker -container which is automatically started. +container which is automatically started by fenv. If the --docs' flag is set then the documentation HTML will be generated under the /docs directory. @@ -43,10 +43,6 @@ Multiple image names can be specified on the '--image' flag by separating them with commas. ./build.sh --image build,fql - -When building Docker images, the dependencies of the Dockerfile -are specified in 'bake.hcl'. When this file is changed, you'll -need to rebuild the docker images for the changes to take effect. END } @@ -139,54 +135,59 @@ while [[ $# -gt 0 ]]; do done -# Build variables required by the docker compose command. - -BUILD_TASKS=() - -if [[ -n "$VERIFY_GENERATION" ]]; then - BUILD_TASKS+=('./scripts/verify_generation.sh') -fi - -if [[ -n "$VERIFY_CODEBASE" ]]; then - BUILD_TASKS+=('./scripts/setup_database.sh') - BUILD_TASKS+=("./scripts/verify_codebase.sh") -fi +# Build variables for fenv and docker commands. -if [[ -n "$GENERATE_DOCS" ]]; then - BUILD_TASKS+=('./scripts/generate_docs.sh') -fi +# Set fenv environment variables +export FENV_FDB_VER="${FENV_FDB_VER:-6.2.30}" +export FENV_PROJECT_NAME="fql" -BUILD_COMMAND="$(join_array ' && ' "${BUILD_TASKS[@]}")" -echo "BUILD_COMMAND=${BUILD_COMMAND}" -export BUILD_COMMAND - -FQL_COMMAND=${FQL_ARGS[*]} -echo "FQL_COMMAND=${FQL_COMMAND}" -export FQL_COMMAND - -DOCKER_TAG="$(./scripts/docker_tag.sh)" +DOCKER_TAG="$(./fenv/docker_tag.sh)" echo "DOCKER_TAG=${DOCKER_TAG}" export DOCKER_TAG -FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FDB_VER:-6.2.30}" -echo "FDB_DOCKER_IMAGE=${FDB_DOCKER_IMAGE}" -export FDB_DOCKER_IMAGE +# fenv command with common flags +FENV_CMD=(./fenv/fenv.sh --docker ./docker/Dockerfile.builder) # Run the requested commands. +# Build the extended fenv image if requested if [[ -n "$IMAGE_BUILD" ]]; then - (set -x; docker buildx bake -f bake.hcl --load build) + echo "Building extended fenv image..." + (set -x; "${FENV_CMD[@]}" --build) +fi + +# Execute build tasks (generate, verify, docs) using fenv +if [[ -n "$VERIFY_GENERATION" ]]; then + echo "Verifying code generation..." + (set -x; "${FENV_CMD[@]}" --exec ./scripts/verify_generation.sh) fi -if [[ -n "$BUILD_COMMAND" ]]; then - (set -x; docker compose run --rm build /bin/sh -c "$BUILD_COMMAND") +if [[ -n "$VERIFY_CODEBASE" ]]; then + echo "Verifying codebase..." + (set -x; "${FENV_CMD[@]}" --exec ./scripts/verify_codebase.sh) +fi + +if [[ -n "$GENERATE_DOCS" ]]; then + echo "Generating documentation..." + (set -x; "${FENV_CMD[@]}" --exec ./scripts/generate_docs.sh) fi +# Build the final fql image if requested if [[ -n "$IMAGE_FQL" ]]; then - (set -x; docker buildx bake -f bake.hcl --load fql) + echo "Building fql runtime image..." + (set -x; docker build \ + -f ./docker/Dockerfile \ + -t "docker.io/janderland/fql:${DOCKER_TAG}" \ + --build-arg "FQL_VER=${DOCKER_TAG}" \ + --build-arg "FENV_FDB_VER=${FENV_FDB_VER}" \ + --build-arg "FENV_EXT_DOCKER_TAG=${DOCKER_TAG}" \ + .) fi +# Run fql interactively if requested if [[ -n "$RUN_FQL" ]]; then + echo "Running fql with args: ${FQL_ARGS[*]}" + export FDB_DOCKER_IMAGE="foundationdb/foundationdb:${FENV_FDB_VER}" (set -x; docker compose run --rm fql 'docker:docker@{fdb}:4500' "${FQL_ARGS[@]}") fi diff --git a/docker/Dockerfile b/docker/Dockerfile index 7bae9287..230f6f24 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,65 +1,7 @@ -# 'builder' includes all build & testing dependencies. -FROM debian:12 AS builder - -RUN apt-get update &&\ - apt-get install --no-install-recommends -y \ - build-essential=12.9 \ - ca-certificates=20* \ - git=1:2.39.* \ - curl=7.88.* &&\ - apt-get clean &&\ - rm -rf /var/lib/apt/lists/* - -# The FDB installer is not deleted so it can be used to -# install the library in the final stage at the end of -# this file. -ARG FDB_LIB_URL -RUN curl -Lo /fdb.deb $FDB_LIB_URL &&\ - dpkg -i /fdb.deb - -ARG GO_URL -RUN curl -Lo /go.tar.gz $GO_URL &&\ - tar -C /usr/local -xzf /go.tar.gz &&\ - rm /go.tar.gz -ENV PATH="/root/go/bin:/usr/local/go/bin:${PATH}" -ENV GOCACHE="/cache/gocache" -ENV GOMODCACHE="/cache/gomod" - -ARG GOLANGCI_LINT_URL="https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh" -ARG GOLANGCI_LINT_VER -RUN curl -Lo /golint.sh $GOLANGCI_LINT_URL &&\ - sh /golint.sh -b "$(go env GOPATH)/bin" $GOLANGCI_LINT_VER &&\ - rm /golint.sh -ENV GOLANGCI_LINT_CACHE="/cache/golangci-lint" - -ARG SHELLCHECK_URL -RUN curl -Lo /shellcheck.tar.xz $SHELLCHECK_URL &&\ - tar -xf /shellcheck.tar.xz &&\ - mv /shellcheck-*/shellcheck /usr/local/bin &&\ - rm -r /shellcheck.tar.xz /shellcheck-* - -ARG HADOLINT_URL -RUN curl -Lo /usr/local/bin/hadolint $HADOLINT_URL &&\ - chmod +x /usr/local/bin/hadolint - -ARG JQ_URL -RUN curl -Lo /usr/local/bin/jq $JQ_URL &&\ - chmod +x /usr/local/bin/jq - -ARG PANDOC_URL -RUN curl -Lo /pandoc.deb $PANDOC_URL &&\ - dpkg -i /pandoc.deb &&\ - rm /pandoc.deb - -# Configure git so it allows any user to run git commands -# on the /fql directory. This allows the user which runs -# CI to be different from the user which built the Docker -# image. -RUN git config --global --add safe.directory /fql - - -# 'gobuild' executes 'go build'. -FROM builder AS gobuild +# 'gobuild' stage compiles the FQL binary using the pre-built builder image from fenv. +# The builder image (fenv-ext-fql) is built separately via ./build.sh --image build +ARG FENV_EXT_DOCKER_TAG +FROM fenv-ext-fql:${FENV_EXT_DOCKER_TAG} AS gobuild COPY . /src WORKDIR /src @@ -68,11 +10,17 @@ ARG FQL_VER RUN go build -o /fql -ldflags="-X 'github.com/janderland/fql/internal/app.Version=${FQL_VER}'" -# The final stage builds the 'fql' image. +# The final stage builds the minimal 'fql' runtime image. FROM debian:12 -COPY --from=gobuild /fdb.deb /fdb.deb -RUN dpkg -i ./fdb.deb &&\ +RUN apt-get update && \ + apt-get install --no-install-recommends -y curl=7.88.* && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ARG FENV_FDB_VER +RUN curl -fsSL "https://github.com/apple/foundationdb/releases/download/${FENV_FDB_VER}/foundationdb-clients_${FENV_FDB_VER}-1_amd64.deb" -o /fdb.deb && \ + dpkg -i /fdb.deb && \ rm /fdb.deb ENV TERM="xterm-256color" diff --git a/docker/Dockerfile.builder b/docker/Dockerfile.builder new file mode 100644 index 00000000..06cf910c --- /dev/null +++ b/docker/Dockerfile.builder @@ -0,0 +1,23 @@ +# Builder image extends the fenv base image with FQL build & testing dependencies. +# This image is used by fenv for local development and CI/CD verification tasks. +ARG FENV_DOCKER_TAG +FROM fenv:${FENV_DOCKER_TAG} + +# Install Go +ARG GO_URL="https://go.dev/dl/go1.19.1.linux-amd64.tar.gz" +RUN curl -fsSL ${GO_URL} | tar -C /usr/local -xz +ENV PATH="/root/go/bin:/usr/local/go/bin:${PATH}" +ENV GOCACHE="/cache/gocache" +ENV GOMODCACHE="/cache/gomod" + +# Install golangci-lint +ARG GOLANGCI_LINT_URL="https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh" +ARG GOLANGCI_LINT_VER="v1.49.0" +RUN curl -fsSL ${GOLANGCI_LINT_URL} | sh -s -- -b "$(go env GOPATH)/bin" ${GOLANGCI_LINT_VER} +ENV GOLANGCI_LINT_CACHE="/cache/golangci-lint" + +# Install pandoc +ARG PANDOC_URL="https://github.com/jgm/pandoc/releases/download/3.3/pandoc-3.3-1-amd64.deb" +RUN curl -fsSL ${PANDOC_URL} -o /tmp/pandoc.deb && \ + dpkg -i /tmp/pandoc.deb && \ + rm /tmp/pandoc.deb diff --git a/fenv b/fenv new file mode 160000 index 00000000..387b628b --- /dev/null +++ b/fenv @@ -0,0 +1 @@ +Subproject commit 387b628b6ddef5e8104a36b9906588e474e5c170 diff --git a/scripts/docker_tag.sh b/scripts/docker_tag.sh deleted file mode 100755 index 2f96dc78..00000000 --- a/scripts/docker_tag.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# docker_tag.sh outputs the Docker image tag based on -# the git version and FDB version. This script is used -# by build.sh and CI workflows to ensure consistent tags. -# -# Environment variables: -# LATEST - If set, uses "latest" instead of git version -# FDB_VER - FDB version (defaults to 6.2.30) - -# code_version returns the latest tag for the current -# Git commit. If there are no tags associated with -# the commit then the short hash is returned. - -function code_version { - local tag="" - if tag="$(git describe --tags 2>/dev/null)"; then - echo "$tag" - return 0 - fi - git rev-parse --short HEAD -} - -# fdb_version returns the version of the FDB library. -# Uses FDB_VER env var if set, otherwise defaults to -# the version specified in bake.hcl. - -function fdb_version { - echo "${FDB_VER:-6.2.30}" -} - -echo "${LATEST:-$(code_version)}_fdb.$(fdb_version)" diff --git a/scripts/setup_database.sh b/scripts/setup_database.sh deleted file mode 100755 index 18dc6b11..00000000 --- a/scripts/setup_database.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash -set -ue - -# The first argument is the hostname of the FDB container. -FDB_HOSTNAME=${1:-fdb} -echo "FDB_HOSTNAME=$FDB_HOSTNAME" - -# The second argument is the description & ID of the FDB cluster. -# https://apple.github.io/foundationdb/administration.html#cluster-file-format -FDB_DESCRIPTION_ID=${2:-docker:docker} -echo "FDB_DESCRIPTION_ID=$FDB_DESCRIPTION_ID" - -# Obtain the IP for FDB from the given hostname. -FDB_IP=$(getent hosts "$FDB_HOSTNAME" | awk '{print $1}') -echo "FDB_IP=$FDB_IP" - -# This variable is recognised by fdbcli and ensures that a fdb.cluster file in -# the current directory won't interfere with this script. -export FDB_CLUSTER_FILE="/etc/foundationdb/fdb.cluster" - -# Create the FDB cluster file. -echo "${FDB_DESCRIPTION_ID}@${FDB_IP}:4500" > $FDB_CLUSTER_FILE -echo "FDB_CLUSTER_FILE: $(cat $FDB_CLUSTER_FILE)" - -# Search for the "unreadable_configuration" message in the cluster's status. This message -# would let us know that the database hasn't been initialized. We need to disable the '-e' -# flag so that bash won't exit if jq returns a non-zero code. -JQ_CODE=0 -jq -e '.cluster.messages[] | select(.name | contains("unreadable_configuration"))' <(set -x; fdbcli --exec 'status json') || JQ_CODE=$? - -# jq should only return codes between 0 & 4 inclusive. Our particular query never -# returns 'null' or 'false', so we shouldn't see code 1. Codes 2 & 3 occur on -# system & compile errors respectively, so the only valid codes are 0 & 4. If the -# code is not 0 or 4 then something unexpected happened so return early. -# https://stedolan.github.io/jq/manual/#Invokingjq -if [[ $JQ_CODE -lt 0 || ( $JQ_CODE -gt 0 && $JQ_CODE -lt 4 ) || $JQ_CODE -gt 4 ]]; then - echo "ERR! Unexpected jq exit code $JQ_CODE" - exit "$JQ_CODE" -fi - -# If this is a new instance of FDB, configure the database. -# https://apple.github.io/foundationdb/administration.html#re-creating-a-database -if [[ $JQ_CODE -eq 0 ]]; then - (set -x; fdbcli --exec "configure new single memory") -fi