Auditlog: Add django-pghistory as audit log (optional for now) (#13169)

* working except data migration

* update performance unit test

* add management command to import scan

* remove old test

* restore __init__.py

* fix tests

* fix questionaire content type in testdata

* fix delete testcases

* fix tests

* fix query counts

* default to django-auditlog

* fix query counts

* update query counts and task counts

* add indices

* default to pghistory

* ruff

* migration

* fix backfill

* ruff

* optimize diff filter

* update counts

* default to django-auditlog

* use current user as actor for delete events

* run unit tests for both auditlog-types

* output more logs to verify auditlog-type

* Revert "use current user as actor for delete events"

This reverts commit 49986c7.

* Reapply "use current user as actor for delete events"

This reverts commit 4dc7312.

* Revert "output more logs to verify auditlog-type"

This reverts commit f5a0592.

* optimize flush audit log

* move helper

* add flushing of pghistory entries

* update counts

* small reorder

* fix tests

Author: valentijnscholten

.github/workflows/integration-tests.yml

@@ -2,12 +2,18 @@ name: Integration tests
 
 on:
   workflow_call:
+    inputs:
+        auditlog_type:
+          type: string
+          default: "django-auditlog"
 
 jobs:
   integration_tests:
     # run tests with docker compose
     name: User Interface Tests
     runs-on: ubuntu-latest
+    env:
+      AUDITLOG_TYPE: ${{ inputs.auditlog_type }}
     strategy:
       matrix:
         test-case: [

.github/workflows/rest-framework-tests.yml

@@ -6,11 +6,16 @@ on:
         platform:
           type: string
           default: "linux/amd64"
+        auditlog_type:
+          type: string
+          default: "django-auditlog"
 
 jobs:
   unit_tests:
     name: Rest Framework Unit Tests
     runs-on: ${{ inputs.platform ==  'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
+    env:
+      AUDITLOG_TYPE: ${{ inputs.auditlog_type }}
 
     strategy:
       matrix:

.github/workflows/unit-tests.yml

@@ -25,18 +25,26 @@ jobs:
     strategy:
         matrix:
             platform: ['linux/amd64', 'linux/arm64']
+            auditlog_type: ['django-auditlog', 'django-pghistory']
         fail-fast: false
     needs: build-docker-containers
     uses: ./.github/workflows/rest-framework-tests.yml
     secrets: inherit
     with:
       platform: ${{ matrix.platform}}
+      auditlog_type: ${{ matrix.auditlog_type }}
 
   # only run integration tests for linux/amd64 (default)
   test-user-interface:
     needs: build-docker-containers
     uses: ./.github/workflows/integration-tests.yml
     secrets: inherit
+    strategy:
+      matrix:
+        auditlog_type: ['django-auditlog', 'django-pghistory']
+      fail-fast: false
+    with:
+      auditlog_type: ${{ matrix.auditlog_type }}
 
   # only run k8s tests for linux/amd64 (default)
   test-k8s:

docker/entrypoint-initializer.sh

@@ -110,6 +110,8 @@ python3 manage.py makemigrations --no-input --check --dry-run --verbosity 3 || {
     cat <<-EOF
 
 ********************************************************************************
+WARNING: Missing Database Migrations Detected
+********************************************************************************
 
 You made changes to the models without creating a DB migration for them.
 
@@ -119,15 +121,25 @@ If you're not familiar with migrations in Django, please read the
 great documentation thoroughly:
 https://docs.djangoproject.com/en/5.0/topics/migrations/
 
+This is now a WARNING and the container will continue to start.
+However, you should create the necessary migrations as soon as possible using:
+docker compose exec uwsgi bash -c 'python manage.py makemigrations -v2'
+
 ********************************************************************************
 
 EOF
-    exit 1
+    echo "WARNING: Continuing startup despite missing migrations..."
 }
 
 echo "Migrating"
 python3 manage.py migrate
 
+echo "Configuring pghistory triggers based on audit settings"
+cat <<EOD | python3 manage.py shell
+from dojo.auditlog import configure_pghistory_triggers
+configure_pghistory_triggers()
+EOD
+
 echo "Admin user: ${DD_ADMIN_USER}"
 ADMIN_EXISTS=$(echo "SELECT * from auth_user;" | python manage.py dbshell | grep "${DD_ADMIN_USER}" || true)
 # Abort if the admin user already exists, instead of giving a new fake password that won't work

dojo/admin.py

@@ -1,5 +1,5 @@
-from auditlog.models import LogEntry
 from django.contrib import admin
+from django.contrib.admin.sites import NotRegistered
 from polymorphic.admin import PolymorphicChildModelAdmin, PolymorphicParentModelAdmin
 
 from dojo.models import (
@@ -14,7 +14,13 @@
     TextQuestion,
 )
 
-admin.site.unregister(LogEntry)
+# Conditionally unregister LogEntry from auditlog if it's registered
+try:
+    from auditlog.models import LogEntry
+    admin.site.unregister(LogEntry)
+except (ImportError, NotRegistered):
+    # auditlog not available or LogEntry not registered
+    pass
 
 # ==============================
 # Defect Dojo Engaegment Surveys

dojo/apps.py

@@ -5,6 +5,7 @@
 from django.db import models
 from watson import search as watson
 
+from dojo.auditlog import configure_audit_system, register_django_pghistory_models
 from dojo.checks import check_configuration_deduplication
 
 logger = logging.getLogger(__name__)
@@ -87,6 +88,13 @@ def ready(self):
         import dojo.test.signals  # noqa: PLC0415 raised: AppRegistryNotReady
         import dojo.tool_product.signals  # noqa: F401,PLC0415 raised: AppRegistryNotReady
 
+        # Configure audit system after all models are loaded
+        # This must be done in ready() to avoid "Models aren't loaded yet" errors
+        # Note: pghistory models are registered here (no database access), but trigger
+        # enabling is handled via management command to avoid database access warnings
+        register_django_pghistory_models()
+        configure_audit_system()
+
 
 def get_model_fields_with_extra(model, extra_fields=()):
     return get_model_fields(get_model_default_fields(model), extra_fields)