Update the backend libraries and dockerization

.env.example

@@ -1,8 +1,20 @@
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=secret
-POSTGRES_DB=civil_society_vote
-
-DATABASE_URL=postgres://postgres:secret@db/civil_society_vote
+DATABASE_USER=postgres
+DATABASE_PASSWORD=secret
+DATABASE_NAME=civil_society_vote
+DATABASE_HOST=db
+DATABASE_PORT=5432
 
 ENABLE_DEBUG_TOOLBAR=True
-LOAD_DUMMY_DATA=yes
+RUN_LOAD_DUMMY_DATA=True
+
+NO_REPLY_EMAIL=
+DEFAULT_FROM_EMAIL=
+DEFAULT_RECEIVE_EMAIL=
+
+EMAIL_BACKEND=django.core.mail.backends.console.EmailBackend
+EMAIL_HOST=
+EMAIL_PORT=25
+EMAIL_HOST_USER=
+EMAIL_HOST_PASSWORD=
+EMAIL_USE_TLS=False
+EMAIL_USE_SSL=False

.github/workflows/backend_check.yml

@@ -0,0 +1,96 @@
+name: Backend Check
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'Dockerfile*'
+      - 'docker-compose*.yml'
+  pull_request:
+    branches:
+      - 'main'
+    paths:
+      - 'backend/**.py'
+      - 'backend/requirements*.*'
+      - 'backend/pyproject.toml'
+      - 'Dockerfile*'
+      - 'docker-compose*.yml'
+      - '.github/workflows/backend_check.yml'
+      - 'data/**.csv'
+      - 'data/**.xlsx'
+
+jobs:
+  static_analysis:
+    name: Run static analysis
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out Git repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: 'pip'
+
+    - name: Cache pip
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
+    - name: Install Black & Ruff with latest pip
+      run: |
+        python -m pip install --upgrade pip
+        cat ./backend/requirements-dev.txt | grep black== | cut -d' ' -f1 | xargs pip install
+        cat ./backend/requirements-dev.txt | grep ruff== | cut -d' ' -f1 | xargs pip install
+
+    - name: Lint files using Ruff
+      run: |
+        ruff check ./backend
+
+    - name: Check formatting with Black
+      run: |
+        black --check ./backend
+
+  tests:
+    name: Run backend tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out Git repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
+        cache: 'pip'
+
+    - name: Cache pip
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements-dev.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
+    - name: Install dependencies
+      run: |
+        cd ./backend
+        python -m pip install --upgrade pip
+        pip install -r requirements-dev.txt
+
+    - name: Set up translations
+      run: |
+        sudo apt-get install gettext
+        ./backend/manage.py compilemessages
+
+    - name: Run tests
+      env:
+        DJANGO_SETTINGS_MODULE: civil_society_vote.test_settings
+      run: |
+        cd ./backend
+        pytest -Wd --cov --cov-report=xml --cov-report=term-missing --cov-fail-under=60 -n auto

.gitignore

@@ -143,7 +143,6 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-# static files generated from Django application using `collectstatic`
-media
-mediafiles
-staticfiles
+# ignore the static files destination directory
+backend/static
+backend/media

Makefile

@@ -1,49 +1,199 @@
-.PHONY: lint build-deps upgrade-deps clean clean-docker clean-py
+help:                             ## Display a help message detailing commands and their purpose
+	@echo "Commands:"
+	@grep -E '^([a-zA-Z_-]+:.*?## .*|#+ (.*))$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@echo ""
 
-help:
-	@echo "lint - apply isort and black formatting"
-	@echo "build-deps - add/remove dependencies"
-	@echo "upgrade-deps - upgrade all rependencies"
-	@echo "clean - shut down containers and clean python cache and test files"
 
-lint:
-	docker-compose run --rm --no-deps --entrypoint "bash -c" web "isort . && black --exclude venv/ ."
+## [Managing the project]
+### Stopping the containers and dropping the databases
+stop-sqlite:                      ## stops the sqlite dev project
+	$(error SQLite is not supported)
 
-migrations:
-	docker-compose run --rm --entrypoint "bash -c" web "./manage.py makemigrations hub"
+drop-sqlite:                      ## stops the sqlite dev project
+	$(error SQLite is not supported)
 
-migrate:
-	docker-compose run --rm --entrypoint "bash -c" web "./manage.py migrate"
+stop-mysql:                       ## stops the mysql dev project
+	$(error MySQL is not supported)
 
-build-deps:
-	docker-compose run --rm --no-deps --entrypoint "bash -c" web "cd .. && pip-compile -o requirements-dev.txt requirements-dev.in requirements.in && pip-compile -o requirements.txt requirements.in"
+drop-mysql:                       ## stops the mysql dev project
+	$(error MySQL is not supported)
 
-upgrade-deps:
-	docker-compose run --rm --no-deps --entrypoint "bash -c" web "cd .. && pip-compile -r -U -o requirements-dev.txt requirements-dev.in requirements.in && pip-compile -r -U -o requirements.txt requirements.in"
+stop-psql:                       ## stops the psql dev project
+	docker compose down -t 60
 
-exec-%:
-	@echo "Welcome to $*"
-	@docker-compose exec $* bash
+drop-psql:                       ## stops the psql dev project
+	docker compose down -v -t 60
 
-clean: clean-docker
+stop-prod:                        ## stops the mysql dev project
+	docker compose -f docker-compose.prod.yml down -t 60
 
-clean-docker:
-	docker-compose down -t 60
+drop-prod:                        ## stops the mysql dev project
+	docker compose -f docker-compose.prod.yml down -v -t 60
+
+### Building & starting the containers
+up-sqlite:                        ## run the project with sqlite
+	$(error SQLite is not supported)
+
+upd-sqlite:                       ## run the project with sqlite in detached mode
+	$(error SQLite is not supported)
+
+up-mysql:                         ## run the project with mysql
+	$(error MySQL is not supported)
+
+upd-mysql:                        ## run the project with mysql in detached mode
+	$(error MySQL is not supported)
+
+up-psql:                         ## run the project with psql
+	docker compose up --build
+
+upd-psql:                        ## run the project with psql in detached mode
+	docker compose up -d --build
+
+up-prod:                         ## run the project with mysql
+	docker compose -f docker-compose.prod.yml up --build
+
+upd-prod:                        ## run the project with mysql in detached mode
+	docker compose -f docker-compose.prod.yml up -d --build
+
+build-prod:
+	docker compose -f docker-compose.prod.yml build \
+		--build-arg $$(cat .env.prod | grep ENVIRONMENT) \
+		--build-arg $$(cat .env.prod | grep REACT_APP_CAPTCHA_API_KEY) \
+		--build-arg $$(cat .env.prod | grep REACT_APP_HERE_MAPS_API_KEY) \
+		--build-arg $$(cat .env.prod | grep REACT_APP_DJANGO_SITE_URL) \
+		--build-arg $$(cat .env.prod | grep REACT_APP_DJANGO_PORT) \
+		--build-arg $$(cat .env.prod | grep REACT_APP_DJANGO_API_ENDPOINT)
+
+### Using the SQLite database
+run-sqlite:
+	$(error SQLite is not supported)   ## run the project with sqlite and stop the mysql project beforehand
+rund-sqlite:
+	$(error SQLite is not supported) ## run the project with sqlite in detached mode and stop the mysql project beforehand
+redo-sqlite:
+	$(error SQLite is not supported)           ## delete the db and rerun the project with sqlite
+redod-sqlite:
+	$(error SQLite is not supported)         ## delete the db and rerun the project with sqlite in detached mode
+
+### Using the MySQL database
+run-mysql:
+	$(error SQLite is not supported)    ## run the project with mysql and stop the sqlite project beforehand
+rund-mysql:
+	$(error SQLite is not supported)  ## run the project with mysql in detached mode and stop the sqlite project beforehand
+redo-mysql:
+	$(error SQLite is not supported)              ## delete the db and rerun the project with mysql
+redod-mysql:
+	$(error SQLite is not supported)            ## delete the db and rerun the project with mysql in detached mode
+
+### Using the PostgreSQL database
+run-psql: up-psql      ## run the project with psql and stop the mysql project beforehand
+rund-psql: upd-psql    ## run the project with psql in detached mode and stop the mysql project beforehand
+redo-psql: drop-psql up-psql                  ## delete the db and rerun the project with psql
+redod-psql: drop-psql upd-psql                ## delete the db and rerun the project with psql in detached mode
+
+### With an image built for production
+run-prod: up-prod                 ## run the project with production settings
+rund-prod: upd-prod               ## run the project with production settings in detached mode
+redo-prod: drop-prod up-prod      ## delete the db and rerun the project with mysql
+redod-prod: drop-prod upd-prod    ## delete the db and rerun the project with mysql in detached mode
+
+### Other run options
+run: run-psql                   ## set the default run command to sqlite
+redo: redo-psql                 ## set the default redo command to sqlite
+rund: rund-psql                 ## set the default run command to sqlite
+redod: redod-psql               ## set the default redo command to sqlite
+
+stop: stop-psql stop-prod ## stop all running projects
+
+drop: drop-psql drop-prod ## drop all databases
+
+
+## [Monitoring the containers]
+logs-dev:                         ## show the logs of the containers
+	docker logs -f votong_dev
+
+logs: logs-dev                    ## set the default logs command to dev
+
+logs-prod:                        ## show the logs of the containers
+	docker logs -f votong_prod
+
+
+## [Django operations]
+makemigrations:                   ## generate migrations in a clean container
+	docker exec votong_backend_dev sh -c "python3 -Wd ./backend/manage.py makemigrations $(apps)"
+
+migrate:                          ## apply migrations in a clean container
+	docker exec votong_backend_dev sh -c "python3 -Wd ./backend/manage.py migrate $(apps)"
+
+migrations: makemigrations migrate ## generate and apply migrations
+
+makemessages:                     ## generate the strings marked for translation
+	docker exec votong_backend_dev sh -c "python3 -Wd ./backend/manage.py makemessages -a"
+
+compilemessages:                  ## compile the translations
+	docker exec votong_backend_dev sh -c "python3 -Wd ./backend/manage.py compilemessages"
+
+messages: makemessages compilemessages ## generate and compile the translations
+
+collectstatic:                    ## collect the static files
+	docker exec votong_backend_dev sh -c "python3 -Wd ./backend/manage.py collectstatic --no-input"
+
+format:                           ## format the code with black & ruff
+	docker exec votong_backend_dev sh -c "black ./backend && ruff check --fix ./backend"
+
+pyshell:                          ## start a django shell
+	docker exec -it votong_backend_dev sh -c "python3 -Wd ./backend/manage.py shell"
+
+sh:                               ## start a sh shell
+	docker exec -it votong_backend_dev sh -c "sh"
+
+bash:                             ## start a bash shell
+	docker exec -it votong_backend_dev sh -c "bash"
+
+
+## [Requirements management]
+requirements-build:               ## run pip compile and add requirements from the *.in files
+	docker exec votong_backend_dev sh -c " \
+		cd ./backend && \
+		pip-compile --strip-extras --resolver=backtracking -o requirements.txt requirements.in && \
+		pip-compile --strip-extras --resolver=backtracking -o requirements-dev.txt requirements-dev.in \
+	"
+
+requirements-update:              ## run pip compile and rebuild the requirements files
+	docker exec votong_backend_dev sh -c " \
+		cd ./backend && \
+		pip-compile --strip-extras --resolver=backtracking -r -U -o requirements.txt requirements.in && \
+		pip-compile --strip-extras --resolver=backtracking -r -U -o requirements-dev.txt requirements-dev.in && \
+		chmod a+r requirements.txt && \
+		chmod a+r requirements-dev.txt \
+	"
+
+
+## [Tests]
+tests:                            ## run the tests
+	docker exec votong_backend_dev sh -c "cd ./backend && pytest -Wd $(apps)"
+
+tests-cover:                      ## run the tests with coverage
+	docker exec votong_backend_dev sh -c "cd ./backend && pytest -Wd  --cov --cov-report=xml --cov-report=term-missing --cov-fail-under=60 $(apps)"
+
+
+## [Clean-up]
+clean-docker:                     ## stop docker containers and remove orphaned images and volumes
+	docker compose down -t 60
+	docker compose -f docker-compose.prod.yml down -t 60
 	docker system prune -f
 
-clean-db: clean-docker
-	docker volume rm votong_database-data
+clean-extras:                        ## remove test, coverage, file artifacts, and compiled message files
+	find ./backend -name '*.mo' -delete
+	find ./backend -name '*.pyc' -delete
+	find ./backend -name '*.pyo' -delete
+	find ./backend -name '.coverage' -delete
+	find ./backend -name '.pytest_cache' -delete
+	find ./backend -name '.ruff_cache' -delete
+	find ./backend -name '__pycache__' -delete
+	find ./backend -name 'htmlcov' -delete
 
-clean-py:
-	find `pwd` -name '*.pyc' -delete
-	find `pwd` -name '*.pyo' -delete
-	find `pwd` -name '.coverage' -delete
-	find `pwd` -name '.pytest_cache' -delete
-	find `pwd` -name '__pycache__' -delete
-	find `pwd` -name 'htmlcov' -delete
+clean-db:                          ## remove the database files
+	rm -rf ./backend/media ./backend/static ./frontend/dist
 
-makemessages:
-	docker-compose exec web python manage.py makemessages
+clean: clean-docker clean-extras clean-db  ## remove all build, test, coverage and Python artifacts
 
-compilemessages:
-	docker-compose exec web python manage.py compilemessages