ref: Performance improvements when serializing events (#368)

A customer was experiencing performance issues when serializing frames with large local vars. Instead of recursing through very large data that we're going to go trim later anyway, do everything in one go.

Author: untitaker

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = E203, E266, E501, W503, E402, E731
+ignore = E203, E266, E501, W503, E402, E731, C901
 max-line-length = 80
 max-complexity = 18
 select = B,C,E,F,W,T4,B9

pytest.ini

@@ -1,3 +1,4 @@
 [pytest]
 DJANGO_SETTINGS_MODULE = tests.integrations.django.myapp.settings
 addopts = --boxed --tb=short
+markers = tests_internal_exceptions

sentry_sdk/client.py

@@ -3,17 +3,15 @@
 import random
 from datetime import datetime
 
-from sentry_sdk._compat import string_types, text_type
+from sentry_sdk._compat import string_types, text_type, iteritems
 from sentry_sdk.utils import (
-    strip_event_mut,
-    flatten_metadata,
-    convert_types,
     handle_in_app,
     get_type_name,
     capture_internal_exceptions,
     current_stacktrace,
     logger,
 )
+from sentry_sdk.serializer import Serializer
 from sentry_sdk.transport import make_transport
 from sentry_sdk.consts import DEFAULT_OPTIONS, SDK_INFO
 from sentry_sdk.integrations import setup_integrations
@@ -43,7 +41,7 @@ def get_options(*args, **kwargs):
     if dsn is not None and options.get("dsn") is None:
         options["dsn"] = dsn  # type: ignore
 
-    for key, value in options.items():
+    for key, value in iteritems(options):
         if key not in rv:
             raise TypeError("Unknown option %r" % (key,))
         rv[key] = value  # type: ignore
@@ -146,9 +144,7 @@ def _prepare_event(
         # Postprocess the event here so that annotated types do
         # generally not surface in before_send
         if event is not None:
-            event = convert_types(event)
-            strip_event_mut(event)
-            event = flatten_metadata(event)
+            event = Serializer().serialize_event(event)
 
         before_send = self.options["before_send"]
         if before_send is not None:

sentry_sdk/integrations/_wsgi_common.py

@@ -2,7 +2,7 @@
 
 from sentry_sdk.hub import Hub, _should_send_default_pii
 from sentry_sdk.utils import AnnotatedValue
-from sentry_sdk._compat import text_type
+from sentry_sdk._compat import text_type, iteritems
 
 if False:
     from typing import Any
@@ -75,8 +75,8 @@ def parsed_body(self):
         form = self.form()
         files = self.files()
         if form or files:
-            data = dict(form.items())
-            for k, v in files.items():
+            data = dict(iteritems(form))
+            for k, v in iteritems(files):
                 size = self.size_of_file(v)
                 data[k] = AnnotatedValue(
                     "", {"len": size, "rem": [["!raw", "x", 0, size]]}
@@ -130,6 +130,6 @@ def _filter_headers(headers):
 
     return {
         k: v
-        for k, v in headers.items()
+        for k, v in iteritems(headers)
         if k.lower().replace("_", "-") not in ("set-cookie", "cookie", "authorization")
     }

sentry_sdk/integrations/django/__init__.py

@@ -30,8 +30,8 @@
 from sentry_sdk import Hub
 from sentry_sdk.hub import _should_send_default_pii
 from sentry_sdk.scope import add_global_event_processor
+from sentry_sdk.serializer import add_global_repr_processor
 from sentry_sdk.utils import (
-    add_global_repr_processor,
     capture_internal_exceptions,
     event_from_exception,
     safe_repr,

sentry_sdk/integrations/logging.py

@@ -11,6 +11,7 @@
     capture_internal_exceptions,
 )
 from sentry_sdk.integrations import Integration
+from sentry_sdk._compat import iteritems
 
 if False:
     from logging import LogRecord
@@ -135,7 +136,7 @@ def _extra_from_record(record):
     # type: (LogRecord) -> Dict[str, None]
     return {
         k: v
-        for k, v in vars(record).items()
+        for k, v in iteritems(vars(record))
         if k not in COMMON_RECORD_ATTRS and not k.startswith("_")
     }
 

sentry_sdk/integrations/pyramid.py

@@ -9,7 +9,7 @@
 
 from sentry_sdk.hub import Hub, _should_send_default_pii
 from sentry_sdk.utils import capture_internal_exceptions, event_from_exception
-from sentry_sdk._compat import reraise
+from sentry_sdk._compat import reraise, iteritems
 
 from sentry_sdk.integrations import Integration
 from sentry_sdk.integrations._wsgi_common import RequestExtractor
@@ -135,15 +135,15 @@ def form(self):
         # type: () -> Dict[str, str]
         return {
             key: value
-            for key, value in self.request.POST.items()
+            for key, value in iteritems(self.request.POST)
             if not getattr(value, "filename", None)
         }
 
     def files(self):
         # type: () -> Dict[str, cgi_FieldStorage]
         return {
             key: value
-            for key, value in self.request.POST.items()
+            for key, value in iteritems(self.request.POST)
             if getattr(value, "filename", None)
         }
 

sentry_sdk/integrations/tornado.py

@@ -15,6 +15,7 @@
     _is_json_content_type,
 )
 from sentry_sdk.integrations.logging import ignore_logger
+from sentry_sdk._compat import iteritems
 
 from tornado.web import RequestHandler, HTTPError  # type: ignore
 from tornado.gen import coroutine  # type: ignore
@@ -166,7 +167,7 @@ def content_length(self):
 
     def cookies(self):
         # type: () -> Dict
-        return {k: v.value for k, v in self.request.cookies.items()}
+        return {k: v.value for k, v in iteritems(self.request.cookies)}
 
     def raw_data(self):
         # type: () -> bytes
@@ -176,7 +177,7 @@ def form(self):
         # type: () -> Optional[Any]
         return {
             k: [v.decode("latin1", "replace") for v in vs]
-            for k, vs in self.request.body_arguments.items()
+            for k, vs in iteritems(self.request.body_arguments)
         }
 
     def is_json(self):
@@ -185,7 +186,7 @@ def is_json(self):
 
     def files(self):
         # type: () -> Dict
-        return {k: v[0] for k, v in self.request.files.items() if v}
+        return {k: v[0] for k, v in iteritems(self.request.files) if v}
 
     def size_of_file(self, file):
         return len(file.body or ())

sentry_sdk/integrations/wsgi.py

@@ -2,7 +2,7 @@
 
 from sentry_sdk.hub import Hub, _should_send_default_pii
 from sentry_sdk.utils import capture_internal_exceptions, event_from_exception
-from sentry_sdk._compat import PY2, reraise
+from sentry_sdk._compat import PY2, reraise, iteritems
 from sentry_sdk.tracing import SpanContext
 from sentry_sdk.integrations._wsgi_common import _filter_headers
 
@@ -118,7 +118,7 @@ def _get_headers(environ):
     Returns only proper HTTP headers.
 
     """
-    for key, value in environ.items():
+    for key, value in iteritems(environ):
         key = str(key)
         if key.startswith("HTTP_") and key not in (
             "HTTP_CONTENT_TYPE",

sentry_sdk/scope.py

@@ -3,7 +3,7 @@
 from functools import wraps
 from itertools import chain
 
-from sentry_sdk.utils import logger, capture_internal_exceptions, object_to_json
+from sentry_sdk.utils import logger, capture_internal_exceptions
 
 if False:
     from typing import Any
@@ -192,7 +192,7 @@ def _drop(event, cause, ty):
             event["fingerprint"] = self._fingerprint
 
         if self._extras:
-            event.setdefault("extra", {}).update(object_to_json(self._extras))
+            event.setdefault("extra", {}).update(self._extras)
 
         if self._tags:
             event.setdefault("tags", {}).update(self._tags)