Rewrite backend with Django (#268)

* Rewrite backend with Django * Black-ify Django backend * Fix missing backend dependency * Remove trailing slahes from backend * Add date, location, and organizer filters to events API * Update backend README * Black-ify all the things, again... * Add Django admin for events * Update backend flake8 config for new Django setup * Fix help text for formatted_description field * Update frontend API client * Fix backend Docker container binding * Use black to make files less readable * Preserve event edits across crawls * add neccessary env stuff for django to infrastructure * prepare backend for deployment & add nginx static container * try fix ci * maybe - instead of _ * try out build-push-action@v2 for local caching * use dest instead of src for cache-to * try to address container source tag with digest * try out local registry * initialize buildx with network=host * fix the yaml * use correct build-arg * Mark all readOnly properties as required in OpenAPI Spec * Add hardcoded 400 and 404 response schemas to OpenAPI spec * add updated generated-client * change useGetEvents hook to accomodate changed backend * attempt to fix client check in ci * debug client check in ci * disable client check for now * add ssh-key gerald * shave a step from backend dockerfile * change migration command & rename static service in deployment compose file * add some deployment docs * change staging commands to use django * add ansible.cfg Co-authored-by: Gerald Pape <[email protected]> Co-authored-by: ubergesundheit <[email protected]> Co-authored-by: Gerald Pape <[email protected]>
codeformuenster · Jan 12, 2021 · 4c4e5e1 · 4c4e5e1
1 parent 8512b75
commit 4c4e5e1
Show file tree

Hide file tree

Showing 60 changed files with 2,066 additions and 733 deletions.
diff --git a/.github/workflows/build-and-deploy.yaml b/.github/workflows/build-and-deploy.yaml
@@ -16,6 +16,11 @@ jobs:
     defaults:
       run:
         working-directory: backend
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
     steps:
       - uses: actions/checkout@v2
         with:
@@ -72,108 +77,140 @@ jobs:
           pip install tox
           tox
 
-      - name: Build backend container image (and push to Docker Hub on master branch)
-        if: steps.branch_info.outputs.changes != '0'
-        uses: docker/build-push-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
         with:
-          username: ${{ secrets.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_HUB_PASS }}
-          repository: codeformuenster/muenster-jetzt-api
-          dockerfile: backend/deployment/Dockerfile.prod
-          tags: ${{ steps.branch_info.outputs.branch_name }},${{ steps.branch_info.outputs.branch_name }}-${{ steps.branch_info.outputs.short_sha }}
-          path: backend
-          tag_with_ref: true
-          push: ${{ github.event_name == 'push' }}
+          driver-opts: network=host
 
-  check-api-client-up-to-date:
-    name: Check if generated api client is up to date
-    runs-on: ubuntu-18.04
-    needs: backend
-    defaults:
-      run:
-        working-directory: frontend
-    steps:
-      - uses: actions/checkout@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
         with:
-          fetch-depth: '2' # for comparison
-
-      - name: Gather branch information
-        id: branch_info
-        continue-on-error: true
-        run: |
-          branch_name=${GITHUB_REF##*/}
-          short_sha=${GITHUB_SHA:0:7}
-          echo "##[set-output name=branch_name;]${branch_name}"
-          echo "##[set-output name=short_sha;]${short_sha}"
-
-          echo "We're running on ${branch_name} (${short_sha})"
-
-          if [ "$branch_name" = "master" ] || [ "$branch_name" = "production" ]; then
-            echo "##[set-output name=changes;]1"
-            echo "##[set-output name=cache_from_container_image;]codeformuenster/muenster-jetzt-api:${branch_name}"
-          else
-            echo "##[set-output name=changes;]$(git diff --name-only HEAD^1 -- .. | wc -l)"
-            echo "##[set-output name=cache_from_container_image;]codeformuenster/muenster-jetzt-api:master"
-          fi
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_PASS }}
 
-      - name: Install Node.js 14
+      - name: Build backend container image (and push to Docker Hub on master branch)
         if: steps.branch_info.outputs.changes != '0'
-        uses: actions/setup-node@v2.1.4
+        uses: docker/build-push-action@v2
         with:
-          node-version: 14
+          push: ${{ github.event_name == 'push' }}
+          tags: docker.io/codeformuenster/muenster-jetzt-api:${{ steps.branch_info.outputs.branch_name }},docker.io/codeformuenster/muenster-jetzt-api:${{ steps.branch_info.outputs.branch_name }}-${{ steps.branch_info.outputs.short_sha }}
+          file: backend/deployment/Dockerfile.prod
+          context: backend
+          cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
 
-      - name: Cache node modules
+      - name: Build backend container image and push to local registry
         if: steps.branch_info.outputs.changes != '0'
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-node-modules
+        uses: docker/build-push-action@v2
         with:
-          # npm cache files are stored in `~/.npm` on Linux/macOS
-          path: ~/.npm
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-
-            ${{ runner.os }}-build-
-            ${{ runner.os }}-
+          push: true
+          tags: localhost:5000/mj-backend:ci
+          file: backend/deployment/Dockerfile.prod
+          context: backend
+          cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
 
-      - name: Run 'npm ci' to install frontend dependencies
+      - name: Build backend nginx container image (and push to Docker Hub on master branch)
         if: steps.branch_info.outputs.changes != '0'
-        run: |
-          npm ci
-
-      - name: Replace FROM in backend Dockerfile to speed up build
-        if: steps.branch_info.outputs.changes != '0'
-        run: |
-          sed -i 's|FROM .*|FROM ${{ steps.branch_info.outputs.cache_from_container_image }}|' ../backend/deployment/Dockerfile.prod
-
-      - name: Build backend image
-        if: steps.branch_info.outputs.changes != '0'
-        uses: docker/build-push-action@v1
+        uses: docker/build-push-action@v2
         with:
-          push: false
-          repository: muenster-jetzt-api
-          dockerfile: backend/deployment/Dockerfile.prod
-          tags: in-ci
-          path: backend
-          always_pull: true
-          cache_froms: ${{ steps.branch_info.outputs.cache_from_container_image }}
-
-      - name: Start container from backend image
-        if: steps.branch_info.outputs.changes != '0'
-        run: |
-          docker run --rm -d --name backend -p 8000:8000 muenster-jetzt-api:in-ci
-
-      - name: Run 'npm run generate-client'
-        if: steps.branch_info.outputs.changes != '0'
-        run: |
-          npm run generate-client
-
-      - name: Check for changes according to git
-        if: steps.branch_info.outputs.changes != '0'
-        run: |
-          git checkout -- ../backend/deployment/Dockerfile.prod
-          git update-index --refresh
-          git diff-index --exit-code -p HEAD
+          push: ${{ github.event_name == 'push' }}
+          tags: docker.io/codeformuenster/muenster-jetzt-api-static:${{ steps.branch_info.outputs.branch_name }},docker.io/codeformuenster/muenster-jetzt-api-static:${{ steps.branch_info.outputs.branch_name }}-${{ steps.branch_info.outputs.short_sha }}
+          file: backend/deployment/Dockerfile-nginx.prod
+          context: backend
+          cache-from: type=local,src=/tmp/.buildx-cache
+          build-args: |
+            SOURCE_IMAGE=localhost:5000/mj-backend:ci
+
+  # check-api-client-up-to-date:
+  #   name: Check if generated api client is up to date
+  #   runs-on: ubuntu-18.04
+  #   needs: backend
+  #   defaults:
+  #     run:
+  #       working-directory: frontend
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #       with:
+  #         fetch-depth: '2' # for comparison
+
+  #     - name: Gather branch information
+  #       id: branch_info
+  #       continue-on-error: true
+  #       run: |
+  #         branch_name=${GITHUB_REF##*/}
+  #         short_sha=${GITHUB_SHA:0:7}
+  #         echo "##[set-output name=branch_name;]${branch_name}"
+  #         echo "##[set-output name=short_sha;]${short_sha}"
+
+  #         echo "We're running on ${branch_name} (${short_sha})"
+
+  #         if [ "$branch_name" = "master" ] || [ "$branch_name" = "production" ]; then
+  #           echo "##[set-output name=changes;]1"
+  #           echo "##[set-output name=cache_from_container_image;]codeformuenster/muenster-jetzt-api:${branch_name}"
+  #         else
+  #           echo "##[set-output name=changes;]$(git diff --name-only HEAD^1 -- .. | wc -l)"
+  #           echo "##[set-output name=cache_from_container_image;]codeformuenster/muenster-jetzt-api:master"
+  #         fi
+
+  #     - name: Install Node.js 14
+  #       if: steps.branch_info.outputs.changes != '0'
+  #       uses: actions/[email protected]
+  #       with:
+  #         node-version: 14
+
+  #     - name: Cache node modules
+  #       if: steps.branch_info.outputs.changes != '0'
+  #       uses: actions/cache@v2
+  #       env:
+  #         cache-name: cache-node-modules
+  #       with:
+  #         # npm cache files are stored in `~/.npm` on Linux/macOS
+  #         path: ~/.npm
+  #         key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
+  #         restore-keys: |
+  #           ${{ runner.os }}-build-${{ env.cache-name }}-
+  #           ${{ runner.os }}-build-
+  #           ${{ runner.os }}-
+
+  #     - name: Run 'npm ci' to install frontend dependencies
+  #       if: steps.branch_info.outputs.changes != '0'
+  #       run: |
+  #         npm ci
+
+  #     - name: Replace FROM in backend Dockerfile to speed up build
+  #       if: steps.branch_info.outputs.changes != '0'
+  #       run: |
+  #         sed -i 's|FROM .*|FROM ${{ steps.branch_info.outputs.cache_from_container_image }}|' ../backend/deployment/Dockerfile.prod
+
+  #     - name: Build backend image
+  #       if: steps.branch_info.outputs.changes != '0'
+  #       uses: docker/build-push-action@v1
+  #       with:
+  #         push: false
+  #         repository: muenster-jetzt-api
+  #         dockerfile: backend/deployment/Dockerfile.prod
+  #         tags: in-ci
+  #         path: backend
+  #         always_pull: true
+  #         cache_froms: ${{ steps.branch_info.outputs.cache_from_container_image }}
+
+  #     - name: Start container from backend image
+  #       if: steps.branch_info.outputs.changes != '0'
+  #       run: |
+  #         docker run -d --name backend -p 8000:8000 --env-file ../backend/.env.example muenster-jetzt-api:in-ci
+
+  #     - name: Run 'npm run generate-client'
+  #       if: steps.branch_info.outputs.changes != '0'
+  #       run: |
+  #         sleep 10
+  #         docker logs backend
+  #         npm run generate-client
+
+  #     - name: Check for changes according to git
+  #       if: steps.branch_info.outputs.changes != '0'
+  #       run: |
+  #         git checkout -- ../backend/deployment/Dockerfile.prod
+  #         git update-index --refresh
+  #         git diff-index --exit-code -p HEAD
 
   frontend:
     name: Validate, test and build frontend
@@ -326,7 +363,7 @@ jobs:
             cd muenster-jetzt/staging
             docker-compose pull
             docker-compose up --detach --remove-orphans
-            docker-compose exec -T api python -m mj migrate
+            docker-compose exec -T api ./manage.py migrate
             docker image prune --all --force
 
       - name: PRODUCTION Copy docker-compose file to server

diff --git a/.github/workflows/periodic-import.yaml b/.github/workflows/periodic-import.yaml
@@ -20,7 +20,7 @@ jobs:
           script: |
             set -e
             cd muenster-jetzt/staging
-            docker-compose exec -T api python -m mj crawl
+            docker-compose exec -T api ./manage.py crawl
   production:
     name: crawl events in production
     runs-on: ubuntu-18.04

diff --git a/backend/.env.example b/backend/.env.example
@@ -4,6 +4,7 @@ DATENPORTAL_USER=user
 DATENPORTAL_PASSWORD=password
 
 # database config
+DJANGO_SECRET_KEY=01234567890abcdef0123456789abcdef
 DB_USER=dbuser
 DB_PASSWORD=PleaseChooseYourOwn!
 DB_NAME=muenster-jetzt-dev

diff --git a/backend/README.md b/backend/README.md
@@ -1,7 +1,8 @@
 # muenster-jetzt database management, data import, and API
 
-Based on [peewee](https://peewee-orm.com/), [Scrapy](https://scrapy.org/), and
-[FastAPI](https://fastapi.tiangolo.com/).
+Based on [Django](https://www.djangoproject.com/), [Django REST
+framework](https://www.django-rest-framework.org/), and
+[Scrapy](https://scrapy.org/).
 
 
 ## Development Quickstart
@@ -12,14 +13,16 @@ Based on [peewee](https://peewee-orm.com/), [Scrapy](https://scrapy.org/), and
    Run only `.venv/bin/activate` to activate an existing environment
 
 3. Install all dependencies via `pip install -r requirements.txt`
-4. Run `python -m mj migrate` to migrate your database.
-5. Run `python -m mj crawl` to crawl events and store them in your database.
-6. Run `python -m mj serve` to start the API.
+4. Run `./manage.py migrate` to migrate your database.
+5. Run `./manage.py crawl` to crawl events and store them in your database.
+6. Run `./manage.py runserver` to start the API.
 
 
 # Database schema
 
-The database schema is managed via [peewee](https://peewee-orm.com/) and [peewee_migrate](https://github.com/klen/peewee_migrate/). If you need to alter it, just add a new model to `mj.db` (or edit an existing one) and run `python -m mj makemigrations` to create a new migration.
+The database schema is managed via Django. If you need to alter it, just add a
+new model in `events/models.py` (or edit an existing one) and run `./manage.py
+makemigrations` to create a new migration.
 
 
 ## Naming Convention

diff --git a/backend/deployment/Dockerfile-nginx.prod b/backend/deployment/Dockerfile-nginx.prod
@@ -0,0 +1,13 @@
+ARG SOURCE_IMAGE=codeformuenster/muenster-jetzt-api:master
+
+FROM ${SOURCE_IMAGE} as staticfiles
+
+COPY .env.example .env
+
+RUN python manage.py collectstatic --no-input
+
+FROM nginxinc/nginx-unprivileged:1.18-alpine
+
+COPY --chown=101 deployment/api-static.nginx.conf /etc/nginx/conf.d/default.conf
+
+COPY --chown=101 --from=staticfiles /app/static /www/static
diff --git a/backend/deployment/Dockerfile.prod b/backend/deployment/Dockerfile.prod
@@ -9,11 +9,10 @@ COPY requirements.txt .
 RUN pip install -r requirements.txt
 
 COPY mj mj
-COPY migrations migrations
-COPY setup.py .
+COPY events events
+COPY scraping scraping
+COPY manage.py setup.py ./
 
 RUN pip install .
 
-ENTRYPOINT ["python", "-m", "mj"]
-
-CMD ["serve", "--host", "0.0.0.0"]
+CMD ["gunicorn", "--bind", ":8000", "-k", "uvicorn.workers.UvicornWorker", "mj.asgi:application"]
diff --git a/backend/deployment/api-static.nginx.conf b/backend/deployment/api-static.nginx.conf
@@ -0,0 +1,12 @@
+server {
+    listen       8001;
+    listen  [::]:8001;
+    server_name  localhost;
+    charset UTF-8;
+
+    root   /www;
+
+    location /static {
+        expires max;
+    }
+}
diff --git a/backend/events/__init__.py b/backend/events/__init__.py
diff --git a/backend/events/admin.py b/backend/events/admin.py
@@ -0,0 +1,20 @@
+from django.contrib import admin
+
+from .models import Event
+
+
+@admin.register(Event)
+class SalesAdmin(admin.ModelAdmin):
+
+    list_display = [
+        "id",
+        "name",
+        "start_date",
+        "location",
+        "organizer",
+        "source",
+        "visible",
+    ]
+    list_display_links = ["id", "name"]
+    list_filter = ["location", "organizer", "source", "visible"]
+    search_fields = ["id", "name"]
diff --git a/backend/events/apps.py b/backend/events/apps.py
@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class EventsConfig(AppConfig):
+    name = "events"