Build dateilager within the dockerfile using the nix environment

This switches the build process to:
 - build DL within the dockerfile for repeatable builds, instead of copying in from outside
 - use the nix environment to setup the env for the build, so it's the same go version / protoc version etc etc in and outside of docker
 - use a multistage docker build so we have slim containers with just the binary at the end

I did all this because I was fighting to get a docker image built for linux_arm64, which is what orbstack needs to run the image locally in a k8s setup! It was annoying. This also adds linux_arm64 as a build target to the makefile, and teaches the dockerfile how to build the binary for the target architecture that the dockerfile is building for.

Yeesh.

Author: airhorns

.dockerignore

@@ -0,0 +1,14 @@
+.git
+.direnv
+.vscode
+Dockerfile
+js
+test
+tmp
+bin
+release
+docs
+input
+node_modules
+development
+!development/nix

.github/actions/setup-env/action.yml

@@ -46,8 +46,4 @@ runs:
 
     - name: Add Go binaries to PATH
       run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
-      shell: bash -l {0}
-
-    - name: Install Go binaries
-      run: make install
-      shell: bash -l {0}
+      shell: bash -l {0}

Dockerfile

@@ -1,17 +1,45 @@
-FROM --platform=linux/amd64 registry.fedoraproject.org/fedora-minimal:40
+FROM nixos/nix:2.22.0 AS build-stage
+ARG TARGETARCH
 
-RUN microdnf install -y curl findutils gzip iputils less postgresql procps shadow-utils tar time which \
-    && microdnf clean all
+RUN tee -a /etc/nix/nix.conf <<EOF
+experimental-features = nix-command flakes
+filter-syscalls = false
+EOF
+
+WORKDIR /app
+
+COPY flake.nix flake.lock  ./
+COPY development ./development
+
+# Setup the nix environment in an early layer
+RUN nix develop --command "true"
+
+# Create a custom shell script that activates nix develop for any RUN command
+RUN echo '#!/usr/bin/env bash' > /bin/nix-env-shell \
+    && echo 'exec nix develop --command bash -c "$@"' >> /bin/nix-env-shell \
+    && chmod +x /bin/nix-env-shell
+
+SHELL ["/bin/nix-env-shell"]
+
+# copy the go modules and download em
+COPY go.mod go.sum ./
+RUN go mod download
+
+# copy everything else and build the project
+COPY . ./
+RUN make release/server_linux_$TARGETARCH
+
+FROM buildpack-deps:bullseye AS build-release-stage
+ARG TARGETARCH
+
+RUN apt-get update && \
+    apt-get install -y curl findutils gzip net-tools less postgresql procps tar time && \
+    rm -rf /var/cache/apt/archives /var/lib/apt/lists/*
 
 RUN GRPC_HEALTH_PROBE_VERSION=v0.4.23 \
-    && curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 \
+    && curl -Lfso /bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-${TARGETARCH} \
     && chmod +x /bin/grpc_health_probe
 
-RUN GO_MIGRATE_VERSION=v4.16.2 \
-    && curl -Lfso /tmp/migrate.tar.gz https://github.com/golang-migrate/migrate/releases/download/${GO_MIGRATE_VERSION}/migrate.linux-amd64.tar.gz \
-    && tar -xzf /tmp/migrate.tar.gz -C /bin \
-    && chmod +x /bin/migrate
-
 RUN useradd -ms /bin/bash main
 USER main
 WORKDIR /home/main
@@ -20,8 +48,11 @@ RUN mkdir -p /home/main/secrets
 VOLUME /home/main/secrets/tls
 VOLUME /home/main/secrets/paseto
 
-COPY release/server_linux_amd64 server
+COPY --from=build-stage /app/release/server_linux_${TARGETARCH} server
 COPY migrations migrations
 COPY entrypoint.sh entrypoint.sh
 
+# smoke test -- ensure the server command can run
+RUN ./server --help
+
 ENTRYPOINT ["./entrypoint.sh"]

Makefile

@@ -21,24 +21,14 @@ PROTO_FILES := $(shell find internal/pb/ -type f -name '*.proto')
 MIGRATE_DIR := ./migrations
 SERVICE := $(PROJECT).server
 
-.PHONY: install migrate migrate-create clean build lint release
+.PHONY: migrate migrate-create clean build lint release
 .PHONY: test test-one test-fuzz test-js lint-js build-js
 .PHONY: reset-db setup-local server server-profile install-js
 .PHONY: client-update client-large-update client-get client-rebuild client-rebuild-with-cache
 .PHONY: client-getcache client-gc-contents client-gc-project client-gc-random-projects
 .PHONY: health upload-container-image run-container gen-docs
 .PHONY: load-test-new load-test-get load-test-update
 
-install:
-	go install google.golang.org/protobuf/cmd/[email protected]
-	go install google.golang.org/grpc/cmd/[email protected]
-	go install github.com/grpc-ecosystem/[email protected]
-	go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/[email protected]
-	go install github.com/bojand/ghz/cmd/[email protected]
-	go install github.com/gadget-inc/fsdiff/cmd/[email protected]
-	go install github.com/stamblerre/gocode@latest
-	go install golang.org/x/tools/cmd/goimports@latest
-
 migrate:
 	migrate -database $(DB_URI)?sslmode=disable -path $(MIGRATE_DIR) up
 
@@ -78,9 +68,14 @@ build: internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go bin/server bin/client deve
 lint:
 	golangci-lint run
 
+
+
 release/%_linux_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $<
 
+release/%_linux_arm64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build $(BUILD_FLAGS) -o $@ $<
+
 release/%_macos_amd64: cmd/%/main.go $(PKG_GO_FILES) $(INTERNAL_GO_FILES) go.sum
 	CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build $(BUILD_FLAGS) -o $@ $<
 
@@ -193,22 +188,23 @@ health:
 	grpc-health-probe -addr $(GRPC_SERVER)
 	grpc-health-probe -addr $(GRPC_SERVER) -service $(SERVICE)
 
-upload-container-image: release
+upload-container-image:
 ifndef version
 	$(error version variable must be set)
 else
-	docker build -t gcr.io/gadget-core-production/dateilager:$(version) -t gcr.io/gadget-core-production/dateilager:latest .
+	docker build --platform linux/amd64 -t gcr.io/gadget-core-production/dateilager:$(version) -t gcr.io/gadget-core-production/dateilager:latest .
 	docker push gcr.io/gadget-core-production/dateilager:$(version)
 	docker push gcr.io/gadget-core-production/dateilager:latest
 endif
 
-upload-prerelease-container-image: release
-	docker build -t gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT) .
-	docker push gcr.io/gadget-core-production/dateilager:$(GIT_COMMIT)
+upload-prerelease-container-image:
+	docker build --platform linux/arm64,linux/amd64 --push -t us-central1-docker.pkg.dev/gadget-core-production/core-production/dateilager:pre-$(GIT_COMMIT) .
+
+build-local-container:
+	docker build --load -t dl-local:dev .
 
-run-container: release
-	docker build -t dl-local:latest .
-	docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto dl-local:latest $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl
+run-container: release build-local-container
+	docker run --rm -it -p 127.0.0.1:$(GRPC_PORT):$(GRPC_PORT)/tcp -v ./development:/home/main/secrets/tls -v ./development:/home/main/secrets/paseto dl-local:dev $(GRPC_PORT) "postgres://$(DB_USER):$(DB_PASS)@host.docker.internal:5432" dl
 
 gen-docs:
 	go run cmd/gen-docs/main.go

Readme.md

@@ -2,11 +2,11 @@
 
 Dateilager is a content-addressed, networked filesystem for keeping large, similar directories among a wide variety of hosts. It has a few key properties:
 
- - file contents and versions are stored in by a central server, with readers and writers as clients
- - files and trees of files are stored only once in a content-addressed store by the server for efficiency, but are still served securely to only clients with access
- - readers and writers keep a copy of the filesystem synced locally for very fast local access (and trade off consistency or atomicity)
- - incremental updates by readers are O(changes) and optimized to be very fast
- - one Dateilager server supports storing many different independent filesystems with secured access to each (multi-tenancy)
+- file contents and versions are stored in by a central server, with readers and writers as clients
+- files and trees of files are stored only once in a content-addressed store by the server for efficiency, but are still served securely to only clients with access
+- readers and writers keep a copy of the filesystem synced locally for very fast local access (and trade off consistency or atomicity)
+- incremental updates by readers are O(changes) and optimized to be very fast
+- one Dateilager server supports storing many different independent filesystems with secured access to each (multi-tenancy)
 
 Dateilager is used in production to power https://gadget.dev to sync the filesystems for all the apps on the Gadget platform. Dateilager shines at syncing the (often very large) `node_modules` folder shared between the many node.js applications, as well as the comparatively small `.js` files comprising the actual business logic of one particular app.
 
@@ -37,10 +37,6 @@ Ensure that you have a working [Go development environment](https://golang.org/d
 
 You will also require `npm`.
 
-```bash
-$ make install
-```
-
 ### Setup VSCode (Optional)
 
 We recommend using VSCode for development, and there's an example settings file at `.vscode/settings.example.json` to get started with:

flake.nix

@@ -11,6 +11,7 @@
       "x86_64-linux"
       "x86_64-darwin"
       "aarch64-darwin"
+      "aarch64-linux"
     ]
       (system: nixpkgs.lib.fix (flake:
         let
@@ -36,7 +37,7 @@
           callPackage = pkgs.newScope (flake.packages // { inherit lib callPackage; });
         in
         {
-          packages = {
+          packages =  {
             ## DateiLager development scripts
 
             clean = callPackage ./development/nix/scripts/clean.nix { };
@@ -63,6 +64,27 @@
           };
 
           defaultPackage = flake.packages.dateilager;
+
+          devShell = pkgs.mkShell {
+            buildInputs = with pkgs; [
+              flake.packages.go
+              flake.packages.nodejs
+              flake.packages.postgresql
+              flake.packages.dev
+              flake.packages.clean
+              git
+              protobuf
+              protoc-gen-go
+              protoc-gen-go-grpc
+              go-migrate
+              mkcert
+            ];
+
+            shellHook = ''
+              # prepend the built binaries to the $PATH
+              export PATH="./bin":$PATH
+          '';
+          };
         }
       )));
 }