Nixify development environment

.development/nix/scripts/clean.nix

@@ -0,0 +1,23 @@
+{ lib
+, writeShellScriptBin
+, git
+}:
+
+let
+  keepFiles = [
+    # Local-specific files
+    ".envrc.local"
+    ".vscode"
+    "TODO"
+
+    # Cache files
+    "node_modules" # Has no functional effect on `dev` service since new/updated modules will be fetched when needed
+  ];
+in
+writeShellScriptBin "clean" ''
+  ${git}/bin/git clean -xdf ${lib.concatStringsSep " " (builtins.map (file: "-e ${file}") keepFiles)}
+'' // {
+  meta = {
+    description = "Reset DateiLager back to a clean state as if it was cloned for the first time";
+  };
+}

.development/nix/scripts/dev.nix

@@ -0,0 +1,60 @@
+{ lib
+, ansifilter
+, coreutils
+, moreutils
+, services
+, writeShellScriptBin
+}:
+
+let
+  packages = lib.unique ([
+    ansifilter
+    coreutils
+    moreutils
+  ] ++ builtins.concatMap (service: service.packages or []) services);
+
+  compileService = isLast: service:
+    let
+      colorize = text:
+        "$'\\e[1;${service.ansiColor or ""}m${text}\\e[0m'";
+
+      consoleFilter = lib.optionalString (service ? consoleFilter)
+        "| grep -v ${service.consoleFilter}";
+    in
+      builtins.concatStringsSep "\n" ([]
+        ++ lib.optional (service ? env) service.env
+        ++ lib.optional (service ? setup) ''
+          (${service.setup}) 2>&1 | \
+            ts ${colorize "${service.name} (setup)>"}
+        ''
+        ++ lib.optional (service ? run) ''
+          (${service.run}) 2>&1 | pee \
+            "ts ${colorize "${service.name}>"}${consoleFilter}" \
+            'ansifilter > tmp/log/${service.name}.log'${lib.optionalString (!isLast) " &"}
+        ''
+        ++ lib.optional (!(service ? run) && isLast) ''
+          sleep infinity
+        '');
+
+  compileServices = services:
+    builtins.concatStringsSep "\n"
+      (lib.imap0
+        (i: service:
+          compileService (i == builtins.length services - 1) service)
+        services);
+in
+writeShellScriptBin "dev" ''
+  set -e
+  export PATH=${lib.makeBinPath packages}:$PATH
+
+  # Stop all services when Ctrl-C is pressed or error occurs
+  trap 'kill $(jobs -p) 2>/dev/null; wait $(jobs -p)' INT TERM ERR EXIT
+
+  mkdir -p tmp/log
+
+  ${compileServices services}
+'' // {
+  meta = with lib; {
+    description = "Runs all the services necessary for DateiLager development";
+  };
+}

.development/nix/services/default.nix

@@ -0,0 +1,8 @@
+{ lib
+, callPackage
+}:
+
+[
+  (callPackage ./postgres.nix { })
+  (callPackage ./setup-db.nix { })
+]

.development/nix/services/postgres.nix

@@ -0,0 +1,51 @@
+{ coreutils
+, postgresql
+}:
+
+{
+  name = "postgres";
+  ansiColor = "34";
+
+  packages = [
+    coreutils # sleep
+    postgresql
+  ];
+
+  env = ''
+    export PGDATA="$PWD/tmp/postgres"
+    export PGUSER=postgres
+    export PGPASSWORD=password
+    export PGURI="postgres://$PGUSER:[email protected]"
+
+    wait_for_postgres() {
+      until psql "$PGURI/postgres" -c '\q' 2> /dev/null; do
+        sleep 0.2
+      done
+    }
+
+    database_exists() {
+      if [ "$(psql "$PGURI/postgres" -qtAc "SELECT 1 FROM pg_database WHERE datname = '$1'")" == "1" ]; then
+        exit 0
+      else
+        exit 1
+      fi
+    }
+
+    create_database() {
+      psql "$PGURI/postgres" -c "CREATE DATABASE $dbname;"
+    }
+  '';
+
+  setup = ''
+    if [ ! -d "$PGDATA" ]; then
+      echo "== Creating postgres database cluster =="
+      initdb --username="$PGUSER" --pwfile=<(echo "$PGPASSWORD")
+    fi
+  '';
+
+  # Don't try to create a unix socket
+  # We only use TCP sockets and some systems require root permissions for unix sockets
+  run = ''
+    postgres -c unix_socket_directories= -c timezone=UTC -c fsync=off -c synchronous_commit=off -c full_page_writes=off
+  '';
+}

.development/nix/services/setup-db.nix

@@ -0,0 +1,17 @@
+{}:
+
+{
+  name = "setup-db";
+  ansiColor = "36";
+
+  setup = ''
+    wait_for_postgres
+
+    for dbname in dl dl_tests; do
+      if ! $(database_exists $dbname); then
+        echo "== Creating '$dbname' database =="
+        create_database $dbname
+      fi
+    done
+  '';
+}

.envrc

@@ -0,0 +1,3 @@
+use flake
+
+source_env_if_exists .envrc.local

.github/actions/setup-env/action.yml

@@ -0,0 +1,51 @@
+name: 'Set up environment'
+description: ''
+inputs: {}
+outputs: {}
+runs:
+  using: composite
+  steps:
+    - uses: cachix/install-nix-action@v14
+      with:
+        install_url: https://nixos-nix-install-tests.cachix.org/serve/vij683ly7sl95nnhb67bdjjfabclr85m/install
+        install_options: '--tarball-url-prefix https://nixos-nix-install-tests.cachix.org/serve'
+        extra_nix_config: |
+          experimental-features = nix-command flakes
+
+    - run: nix flake check
+      shell: bash
+
+    - name: Set cache paths
+      id: cache-paths
+      run: |
+        echo "::set-output name=go_mod::$(go env GOMODCACHE)"
+        echo "::set-output name=go_build::$(go env GOCACHE)"
+        echo "::set-output name=npm::$(npm config get cache)"
+      shell: nix develop -c bash -eo pipefail -l {0}
+
+    - name: Cache go mods
+      uses: actions/cache@v2
+      with:
+        path: |
+          ${{ steps.cache-paths.outputs.go_mod }}
+          ${{ steps.cache-paths.outputs.go_build }}
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-go-
+
+    - name: Cache npm packages
+      uses: actions/cache@v2
+      with:
+        path: ${{ steps.cache-paths.outputs.npm }}
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+
+    - run: go mod download
+      shell: nix develop -c bash -eo pipefail -l {0}
+
+    - run: echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+      shell: nix develop -c bash -eo pipefail -l {0}
+
+    - run: make install
+      shell: nix develop -c bash -eo pipefail -l {0}

.github/workflows/release.yml

@@ -6,40 +6,15 @@ on:
       - v*
 
 jobs:
+  test:
+    uses: ./.github/workflows/test.yml
   build:
+    needs: test
     runs-on: ubuntu-20.04
-
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/setup-go@v2
-      with:
-        go-version: 1.16
-    - uses: actions/setup-node@v2
-      with:
-        node-version: 16
-    - uses: arduino/setup-protoc@v1
-
-    - name: Cache go mods
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.cache/go-build
-          ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
-    - name: Cache npm packages
-      uses: actions/cache@v2
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-
-    - run: go mod download
-    - run: make install
-    - run: make internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go js/src/fs.client.ts
-
+    - uses: ./.github/actions/setup-env
+    - run: nix develop -c make internal/pb/fs.pb.go internal/pb/fs_grpc.pb.go js/src/fs.client.ts
     - uses: goreleaser/goreleaser-action@v2
       with:
         version: latest

.github/workflows/test.yml

@@ -2,62 +2,14 @@ name: Test
 
 on:
   push:
+  workflow_call:
 
 jobs:
-  build:
-    # Multi OS builds will require nix for Postgres
+  test:
     runs-on: ubuntu-20.04
-
-    services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: "password"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
     steps:
     - uses: actions/checkout@v2
-    - uses: actions/setup-go@v2
-      with:
-        go-version: 1.16
-    - uses: actions/setup-node@v2
-      with:
-        node-version: 16
-    - uses: arduino/setup-protoc@v1
-
-    - name: Cache go mods
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.cache/go-build
-          ~/go/pkg/mod
-        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-        restore-keys: |
-          ${{ runner.os }}-go-
-    - name: Cache npm packages
-      uses: actions/cache@v2
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-    - run: go mod download
-
-    - name: Install mkcert
-      run: |
-        sudo apt-get update && sudo apt-get install libnss3-tools
-        curl -o /usr/local/bin/mkcert -L 'https://github.com/FiloSottile/mkcert/releases/download/v1.4.3/mkcert-v1.4.3-linux-amd64'
-        chmod +x /usr/local/bin/mkcert
-        mkcert -install
-    - run: make install
-
-    - run: make build
-    - run: make DB_URI=$DB_URI test
-      env:
-        DB_URI: postgres://postgres:password@localhost:5432/postgres
+    - uses: ./.github/actions/setup-env
+    - run: nix develop -c dev &
+    - run: nix develop -c make build
+    - run: nix develop -c make test

.gitignore

@@ -11,3 +11,9 @@ release/
 *.pb.go
 fs*.ts
 **/node_modules/
+tmp/*
+!tmp/.gitkeep
+.direnv
+.idea/
+.DS_STORE
+.envrc.local