Add logzio

muckrock/core/middleware.py

@@ -1,9 +1,13 @@
-# middleware.py
 # Django
 from django.conf import settings
 from django.http import HttpResponsePermanentRedirect
 from django.utils.deprecation import MiddlewareMixin
 
+# Standard Library
+import logging
+
+logger = logging.getLogger("http_requests")
+
 
 class FlatpageRedirectMiddleware(MiddlewareMixin):
     def process_response(self, request, response):
@@ -16,3 +20,68 @@ def process_response(self, request, response):
                 if path in site_redirects:
                     return HttpResponsePermanentRedirect(site_redirects[path])
         return response
+
+
+class LogHTTPMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+
+    def __call__(self, request):
+        response = self.get_response(request)
+
+        logger.info(
+            "%s %s",
+            request.method,
+            request.path,
+            extra={
+                "request": self.format_request(request),
+                "response": self.format_response(response),
+            },
+        )
+
+        return response
+
+    def format_request(self, request):
+        """Format a request for logging"""
+        return {
+            "user": self.format_user(request.user),
+            "path": request.path,
+            "method": request.method,
+            "headers": dict(request.headers),
+            "get": dict(request.GET),
+            "post": dict(request.POST),
+            "body": request.body.decode("utf-8"),
+        }
+
+    def format_user(self, user):
+        """Format a user for logging"""
+        if user.is_authenticated:
+            return {
+                "id": user.pk,
+                "username": user.username,
+                "email": user.email,
+                "full_name": user.profile.full_name,
+                "verified_journalist": user.profile.verified_journalist,
+                "organization": self.format_organization(user.profile.organization),
+            }
+        return None
+
+    def format_organization(self, org):
+        """Format an organization for logging"""
+        return {
+            "id": org.pk,
+            "name": org.name,
+            "individual": org.individual,
+            "verified_journalist": org.verified_journalist,
+            "entitlement": org.entitlement.slug if org.entitlement_id else None,
+            "monthly_requests": org.monthly_requests,
+            "number_requests": org.number_requests,
+        }
+
+    def format_response(self, response):
+        """Format a response for logging"""
+        return {
+            "status_code": response.status_code,
+            "headers": dict(response.headers),
+            "body": response.content.decode("utf-8")[:2000],
+        }

muckrock/settings/base.py

@@ -221,6 +221,7 @@ def boolcheck(setting):
     "simple_history.middleware.HistoryRequestMiddleware",
     "daily_active_users.middleware.DailyActiveUserMiddleware",
     "muckrock.core.middleware.FlatpageRedirectMiddleware",
+    "muckrock.core.middleware.LogHTTPMiddleware",
 )
 
 FLATPAGES_REDIRECTS = {
@@ -502,6 +503,14 @@ def show_toolbar(request):
             "level": "WARNING",
             "class": "sentry_sdk.integrations.logging.EventHandler",
         },
+        "logzio": {
+            "class": "logzio.handler.LogzioHandler",
+            "level": "INFO",
+            "token": os.environ.get("LOGZIO_TOKEN", ""),
+            "logzio_type": "muckrock",
+            "logs_drain_timeout": 5,
+            "url": "https://listener.logz.io:8071",
+        },
     },
     "loggers": {
         "django": {"handlers": ["null"], "propagate": True, "level": "INFO"},
@@ -521,6 +530,11 @@ def show_toolbar(request):
             "propagate": False,
         },
         "dogslow": {"level": "WARNING", "handlers": ["dogslow"]},
+        "http_requests": {
+            "handlers": ["logzio"],
+            "level": "INFO",
+            "propagate": False,
+        },
     },
 }
 

pip/requirements.in

@@ -53,6 +53,7 @@ img2pdf # convert images to PDFs
 ipython # better python shell
 jsonfield # Used for storing JSON data, required by django-geojson
 lob # sending mail via lob.com
+logzio-python-handler # For logging API requests
 markdown # Used for rendering Markdown, obviously!
 memoize # cachable properties
 micawber # Used for embeds in assignments

pip/requirements.txt

@@ -345,6 +345,8 @@ listcrunch==1.0.1
     # via python-documentcloud
 lob==4.0.1
     # via -r pip/requirements.in
+logzio-python-handler==4.1.9
+    # via -r pip/requirements.in
 lxml==5.3.0
     # via
     #   govqa
@@ -439,6 +441,7 @@ protobuf==6.33.5
     # via
     #   google-api-core
     #   googleapis-common-protos
+    #   logzio-python-handler
     #   proto-plus
 psutil==5.7.2
     # via scout-apm
@@ -547,6 +550,7 @@ requests[security]==2.32.5
     #   google-cloud-storage
     #   huggingface-hub
     #   lob
+    #   logzio-python-handler
     #   phaxio
     #   plaid-python
     #   premailer