fix: custom encoding of the webhook payload

Author: danihodovic

.gitignore

@@ -110,3 +110,6 @@ dmypy.json
 # End of https://www.gitignore.io/api/python
 *.sqlite3
 tests/media
+.direnv
+
+tests/settings/media/*.txt

conftest.py

@@ -1,5 +1,18 @@
 import pytest
 import responses as responses_lib
+from pytest_factoryboy import register
+
+from django_webhook.test_factories import (
+    WebhookEventFactory,
+    WebhookFactory,
+    WebhookSecretFactory,
+    WebhookTopicFactory,
+)
+
+register(WebhookFactory)
+register(WebhookEventFactory)
+register(WebhookTopicFactory)
+register(WebhookSecretFactory)
 
 
 @pytest.fixture

django_webhook/apps.py

@@ -1,4 +1,5 @@
 # pylint: disable=import-outside-toplevel
+
 from django.apps import AppConfig
 
 
@@ -7,17 +8,6 @@ class WebhooksConfig(AppConfig):
     default_auto_field = "django.db.models.AutoField"
 
     def ready(self):
-        from django.conf import settings
-
-        from .settings import defaults
-
-        d = getattr(settings, "DJANGO_WEBHOOK", {})
-        for k, v in defaults.items():
-            if k not in d:
-                d[k] = v
-
-        settings.DJANGO_WEBHOOK = d
-
         # pylint: disable=unused-import
         import django_webhook.checks
         from django_webhook.models import populate_topics_from_settings

django_webhook/checks.py

@@ -1,11 +1,12 @@
 # pylint: disable=import-outside-toplevel,unused-argument
-from django.conf import settings
 from django.core.checks import Error, register
 
+from .settings import get_settings
+
 
 @register()
 def warn_about_webhooks_settings(app_configs, **kwargs):
-    webhook_settings = getattr(settings, "DJANGO_WEBHOOK")
+    webhook_settings = get_settings()
     errors = []
     if not webhook_settings:
         errors.append(

django_webhook/http.py

@@ -1,26 +1,19 @@
 import hashlib
 import hmac
-import json
 from datetime import datetime
-from json import JSONEncoder
-from typing import cast
 
-from django.conf import settings
 from django.utils import timezone
 from requests import Request
 
 from django_webhook.models import Webhook
 
 
-def prepare_request(webhook: Webhook, payload: dict):
+def prepare_request(webhook: Webhook, payload: str):
     now = timezone.now()
     timestamp = int(datetime.timestamp(now))
 
-    encoder_cls = cast(
-        type[JSONEncoder], settings.DJANGO_WEBHOOK["PAYLOAD_ENCODER_CLASS"]
-    )
     signatures = [
-        sign_payload(payload, secret, timestamp, encoder_cls)
+        sign_payload(payload, secret, timestamp)
         for secret in webhook.secrets.values_list("token", flat=True)
     ]
     headers = {
@@ -33,18 +26,13 @@ def prepare_request(webhook: Webhook, payload: dict):
         method="POST",
         url=webhook.url,
         headers=headers,
-        data=json.dumps(payload, cls=encoder_cls).encode(),
+        data=payload.encode(),
     )
     return r.prepare()
 
 
-def sign_payload(
-    payload: dict, secret: str, timestamp: int, encoder_cls: type[JSONEncoder]
-):
-    combined_payload = f"{timestamp}:{json.dumps(payload, cls=encoder_cls)}"
+def sign_payload(payload: str, secret: str, timestamp: int):
+    combined_payload = f"{timestamp}:{payload}"
     return hmac.new(
         key=secret.encode(), msg=combined_payload.encode(), digestmod=hashlib.sha256
     ).hexdigest()
-
-
-# TODO: Test that encoder is swappable

django_webhook/models.py

@@ -2,12 +2,13 @@
 import uuid
 
 from celery import states
-from django.conf import settings
 from django.core import validators
 from django.core.serializers.json import DjangoJSONEncoder
 from django.db import models
 from django.db.models.fields import DateTimeField
 
+from django_webhook.settings import get_settings
+
 from .validators import validate_topic_model
 
 topic_regex = r"\w+\.\w+\/[create|update|delete]"
@@ -111,7 +112,7 @@ def populate_topics_from_settings():
             return
         raise ex
 
-    webhook_settings = getattr(settings, "DJANGO_WEBHOOK", {})
+    webhook_settings = get_settings()
     enabled_models = webhook_settings.get("MODELS")
     if not enabled_models:
         return

django_webhook/settings.py

@@ -1,8 +1,23 @@
 from django.core.serializers.json import DjangoJSONEncoder
+from django.utils.module_loading import import_string
 
 defaults = dict(
     PAYLOAD_ENCODER_CLASS=DjangoJSONEncoder,
     STORE_EVENTS=True,
     EVENTS_RETENTION_DAYS=30,
     USE_CACHE=True,
 )
+
+
+def get_settings():
+    # pylint: disable=redefined-outer-name,import-outside-toplevel
+    from django.conf import settings
+
+    user_defined_settings = getattr(settings, "DJANGO_WEBHOOK", {})
+    webhook_settings = {**defaults, **user_defined_settings}
+
+    encoder_cls = webhook_settings["PAYLOAD_ENCODER_CLASS"]
+    if isinstance(encoder_cls, str):
+        webhook_settings["PAYLOAD_ENCODER_CLASS"] = import_string(encoder_cls)
+
+    return webhook_settings

django_webhook/signals.py

@@ -1,14 +1,15 @@
 # pylint: disable=redefined-builtin
+import json
 from datetime import timedelta
 
 from django.apps import apps
-from django.conf import settings
 from django.db import models
 from django.db.models.signals import ModelSignal, post_delete, post_save
 from django.forms import model_to_dict
 
 from django_webhook.models import Webhook
 
+from .settings import get_settings
 from .tasks import fire_webhook
 from .util import cache
 
@@ -42,15 +43,22 @@ def run(self, sender, created: bool = False, instance=None, **kwargs):
 
         topic = f"{self.model_label}/{action_type}"
         webhook_ids = _find_webhooks(topic)
+        encoder_cls = get_settings()["PAYLOAD_ENCODER_CLASS"]
 
         for id, uuid in webhook_ids:
-            payload = dict(
-                topic=topic,
+            payload_dict = dict(
                 object=model_dict(instance),
+                topic=topic,
                 object_type=self.model_label,
                 webhook_uuid=str(uuid),
             )
-            fire_webhook.delay(id, payload)
+            payload = json.dumps(payload_dict, cls=encoder_cls)
+            fire_webhook.delay(
+                id,
+                payload,
+                topic=topic,
+                object_type=self.model_label,
+            )
 
     def connect(self):
         self.signal.connect(
@@ -89,7 +97,7 @@ def model_dict(model):
 
 
 def _active_models():
-    model_names = settings.DJANGO_WEBHOOK.get("MODELS", [])
+    model_names = get_settings().get("MODELS", [])
     model_classes = []
     for name in model_names:
         parts = name.split(".")
@@ -108,7 +116,7 @@ def _find_webhooks(topic: str):
     """
     In tests and for smaller setups we don't want to cache the query.
     """
-    if settings.DJANGO_WEBHOOK["USE_CACHE"]:
+    if get_settings()["USE_CACHE"]:
         return _query_webhooks_cached(topic)
     return _query_webhooks(topic)
 

django_webhook/tasks.py

@@ -1,16 +1,17 @@
+import json
 import logging
 from datetime import timedelta
 
 from celery import current_app as app
 from celery import states
-from django.conf import settings
 from django.utils import timezone
 from requests import Session
 from requests.exceptions import RequestException
 
 from django_webhook.models import Webhook, WebhookEvent
 
 from .http import prepare_request
+from .settings import get_settings
 
 
 @app.task(
@@ -21,23 +22,30 @@
     retry_backoff_max=60 * 60,
     retry_jitter=False,
 )
-def fire_webhook(self, webhook_id: int, payload: dict):
+def fire_webhook(
+    self,
+    webhook_id: int,
+    payload: dict,
+    topic=None,
+    object_type=None,
+):
     webhook = Webhook.objects.get(id=webhook_id)
     if not webhook.active:
         logging.warning(f"Webhook: {webhook} is inactive and I will not fire it.")
         return
 
-    req = prepare_request(webhook, payload)
-    store_events = settings.DJANGO_WEBHOOK["STORE_EVENTS"]
+    req = prepare_request(webhook, payload)  # type: ignore
+    settings = get_settings()
+    store_events = settings["STORE_EVENTS"]
 
     if store_events:
         event = WebhookEvent.objects.create(
             webhook=webhook,
-            object=payload,
-            object_type=payload.get("object_type"),
+            object=json.loads(payload),  # type: ignore
+            object_type=object_type,
             status=states.PENDING,
             url=webhook.url,
-            topic=payload.get("topic"),
+            topic=topic,
         )
     try:
         Session().send(req).raise_for_status()
@@ -63,7 +71,7 @@ def clear_webhook_events():
     """
     Clears out old webhook events
     """
-    days_ago = settings.DJANGO_WEBHOOK["EVENTS_RETENTION_DAYS"]
+    days_ago = get_settings()["EVENTS_RETENTION_DAYS"]
     now = timezone.now()
     cutoff_date = now - timedelta(days=days_ago)  # type: ignore
     qs = WebhookEvent.objects.filter(created__lt=cutoff_date)

django_webhook/validators.py

@@ -1,9 +1,10 @@
-from django.conf import settings
 from django.core.exceptions import ValidationError
 
+from .settings import get_settings
+
 
 def validate_topic_model(value: str):
-    webhook_settings = getattr(settings, "DJANGO_WEBHOOK", {})
+    webhook_settings = get_settings()
     allowed_models = webhook_settings.get("MODELS", [])
     if not webhook_settings or not allowed_models:
         raise ValidationError("settings.DJANGO_WEBHOOK.MODELS is empty")