refactor backend project layout

.gitignore

@@ -1,7 +1,7 @@
 __pycache__
 .cache
 .DS_Store
-configs/.*.env
+config/.*
 .history
 .idea
 .ipynb_checkpoints

.gitlab-ci.yml

@@ -23,47 +23,17 @@ build:web:
     - docker push ${IMAGE}:${CI_COMMIT_SHORT_SHA}
     - docker push ${IMAGE}:latest
 
-.build:go:
+build:backend:
+  variables:
+    IMAGE: gcr.io/beneath/backend
   script: 
     - docker pull ${IMAGE}:builder || true
-    - docker build -f ./build/${CMD_NAME}/Dockerfile --target builder -t ${IMAGE}:builder --cache-from ${IMAGE}:builder .
-    - docker build -f ./build/${CMD_NAME}/Dockerfile -t ${IMAGE}:latest -t ${IMAGE}:${CI_COMMIT_SHORT_SHA} --cache-from ${IMAGE}:builder .
+    - docker build -f ./build/backend/Dockerfile --target builder -t ${IMAGE}:builder --cache-from ${IMAGE}:builder .
+    - docker build -f ./build/backend/Dockerfile -t ${IMAGE}:latest -t ${IMAGE}:${CI_COMMIT_SHORT_SHA} --cache-from ${IMAGE}:builder .
     - docker push ${IMAGE}:${CI_COMMIT_SHORT_SHA}
     - docker push ${IMAGE}:latest
     - docker push ${IMAGE}:builder
 
-build:control:
-  extends:
-  - .build
-  - .build:go
-  variables:
-    CMD_NAME: control
-    IMAGE: gcr.io/beneath/${CMD_NAME}
-
-build:gateway:
-  extends:
-  - .build
-  - .build:go
-  variables:
-    CMD_NAME: gateway
-    IMAGE: gcr.io/beneath/${CMD_NAME}
-
-build:pipeline:
-  extends:
-  - .build
-  - .build:go
-  variables:
-    CMD_NAME: pipeline
-    IMAGE: gcr.io/beneath/${CMD_NAME}
-
-build:taskqueue:
-  extends:
-  - .build
-  - .build:go
-  variables:
-    CMD_NAME: taskqueue
-    IMAGE: gcr.io/beneath/${CMD_NAME}
-
 .deploy:
   stage: deploy
   image: dtzar/helm-kubectl:3.2.1
@@ -93,7 +63,7 @@ deploy:control:
   extends:
   - .deploy
   - .deploy:prod
-  needs: ["build:control"]
+  needs: ["build:backend"]
   variables:
     CMD_NAME: control
     REPLICAS: ${REPLICAS_CONTROL}
@@ -103,7 +73,7 @@ deploy:gateway:
   extends:
   - .deploy
   - .deploy:prod
-  needs: ["build:gateway"]
+  needs: ["build:backend"]
   variables:
     CMD_NAME: gateway
     REPLICAS: ${REPLICAS_GATEWAY}
@@ -113,7 +83,7 @@ deploy:pipeline:
   extends:
   - .deploy
   - .deploy:prod
-  needs: ["build:pipeline"]
+  needs: ["build:backend"]
   variables:
     CMD_NAME: pipeline
     REPLICAS: ${REPLICAS_PIPELINE}
@@ -123,7 +93,7 @@ deploy:taskqueue:
   extends:
   - .deploy
   - .deploy:prod
-  needs: ["build:taskqueue"]
+  needs: ["build:backend"]
   variables:
     CMD_NAME: taskqueue
     REPLICAS: ${REPLICAS_TASKQUEUE}

build/README.md

@@ -1,3 +1,3 @@
 # `build/`
 
-This directory contains build manifests (currently only Dockerfiles) for each executable in `cmd/` that is deployed in production.
+This directory contains build manifests (Dockerfiles) for the web frontend in `web/` and backend executable in `cmd/beneath/`.

build/pipeline/Dockerfile → build/backend/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 COPY go.mod go.sum ./
 RUN go mod download
 COPY . .
-RUN CGO_ENABLED=0 GOOS=linux go build -a -o main cmd/pipeline/main.go
+RUN CGO_ENABLED=0 GOOS=linux go build -a -o main cmd/beneath/main.go
 
 # Create run image
 FROM alpine:latest

bus/README.md

@@ -0,0 +1,18 @@
+# `bus/`
+
+The bus facilitates sync (in-process) and async (background worker) event dispatch and handling. Services (in `services/`) send and subscribe to events messages, which are defined in `models/`.
+
+The bus was inspired by the [communication model in Grafana](https://github.com/grafana/grafana/blob/master/contribute/architecture/communication.md).
+
+## Sync and async handlers
+
+- *Sync event handlers* are called sequentially in-process when an event is published, and causes the event publisher to fail on error. If an event is sent in a DB transaction, subscribed handlers will also run inside that transaction!
+- *Async event handlers* are called in a background worker. They retry on error. Event messages are serializes with msgpack and passed through the MQ (`infrastructure/mq/`).
+
+Use sync event handlers for light operations or operations that require the publisher to fail on error. Use async event handlers for everything else.
+
+## Background worker
+
+The background worker calls `bus.Run` to start processing async events dispatched from other processes. It's started with the `control-worker` startable (see `cmd/beneath/`).
+
+**It's critical that every service that handles async events is initialized in the worker process** before bus.Run is called! (See `cmd/beneath/dependencies/services.go`.)