Merge branch 'master' into jgreer013/issue-6596

Author: jgreer013

requirements-testing.txt

@@ -1,11 +1,9 @@
 pytest>=6.0.0
 pytest-cov
+dataclasses; python_version < "3.7"
 pytest-forked
-pytest-localserver
-pytest-timeout
 executing
 asttokens
 responses
-httpcore[http2]
 setuptools
 docker

scripts/populate_tox/config.py

@@ -108,8 +108,8 @@
                 "djangorestframework",
                 "pytest-django",
                 "Werkzeug",
+                "channels[daphne]",
             ],
-            ">=2.0": ["channels[daphne]"],
             ">=2.2,<3.1": ["six"],
             ">=3.0": ["pytest-asyncio"],
             "<3.3": [

scripts/populate_tox/package_dependencies.jsonl

@@ -80,11 +80,14 @@ deps =
     # === Common ===
     py3.8-common: hypothesis
     common: brotli
+    common: httpcore[asyncio,http2]
     common: jsonschema
     common: pysocks
     common: pytest-asyncio
-    common: httpcore[asyncio]
+    common: pytest-timeout
+    common: pytest_localserver
     common: socksio
+    common: werkzeug
     # See https://github.com/pytest-dev/pytest/issues/9621
     # and https://github.com/pytest-dev/pytest-forked/issues/67
     # for justification of the upper bound on pytest
@@ -102,11 +105,15 @@ deps =
     {py3.6,py3.7}-gevent: pytest<7.0.0
     {py3.8,py3.9,py3.10,py3.11,py3.12}-gevent: pytest
     gevent: brotli
+    gevent: httpcore[http2]
     gevent: jsonschema
     gevent: pysocks
     gevent: pytest-asyncio
+    gevent: pytest-timeout
+    gevent: pytest_localserver
     gevent: setuptools<82
     gevent: socksio
+    gevent: werkzeug
     {py3.10,py3.11}-gevent: zope.event<5.0.0
     {py3.10,py3.11}-gevent: zope.interface<8.0
 

scripts/populate_tox/releases.jsonl

@@ -90,6 +90,7 @@ def _inner(*args: "P.args", **kwargs: "P.kwargs") -> "T":
                 attributes={
                     "sentry.op": OP.DB,
                     "sentry.origin": ClickhouseDriverIntegration.origin,
+                    SPANDATA.DB_QUERY_TEXT: str(query),
                 },
             )
         else:

scripts/populate_tox/tox.jinja

@@ -155,6 +155,7 @@ def started(self, event: "CommandStartedEvent") -> None:
                 span_first_data = {
                     "db.operation.name": operation_name,
                     "db.collection.name": collection_name,
+                    SPANDATA.DB_QUERY_TEXT: query,
                     "sentry.op": OP.DB,
                     "sentry.origin": PyMongoIntegration.origin,
                     **db_data,

sentry_sdk/integrations/clickhouse_driver.py

@@ -24,8 +24,17 @@
     jsonschema = None
 
 import pytest
-from pytest_localserver.http import WSGIServer
-from werkzeug.wrappers import Request, Response
+
+try:
+    from pytest_localserver.http import WSGIServer
+except ImportError:
+    WSGIServer = None
+
+try:
+    from werkzeug.wrappers import Request, Response
+except ImportError:
+    Request = None
+    Response = None
 
 try:
     from starlette.testclient import TestClient
@@ -1624,53 +1633,61 @@ def __ne__(self, other):
 CapturedData = namedtuple("CapturedData", ["path", "event", "envelope", "compressed"])
 
 
-class CapturingServer(WSGIServer):
-    def __init__(self, host="127.0.0.1", port=0, ssl_context=None):
-        WSGIServer.__init__(self, host, port, self, ssl_context=ssl_context)
-        self.code = 204
-        self.headers = {}
-        self.captured = []
-
-    def respond_with(self, code=200, headers=None):
-        self.code = code
-        if headers:
-            self.headers = headers
-
-    def clear_captured(self):
-        del self.captured[:]
+@pytest.fixture(scope="module")
+def wsgi_capturing_server():
+    assert WSGIServer is not None
+
+    class CapturingServer(WSGIServer):
+        def __init__(self, host="127.0.0.1", port=0, ssl_context=None):
+            WSGIServer.__init__(self, host, port, self, ssl_context=ssl_context)
+            self.code = 204
+            self.headers = {}
+            self.captured = []
+
+        def respond_with(self, code=200, headers=None):
+            self.code = code
+            if headers:
+                self.headers = headers
+
+        def clear_captured(self):
+            del self.captured[:]
+
+        def __call__(self, environ, start_response):
+            """
+            This is the WSGI application.
+            """
+            assert Request is not None
+            assert Response is not None
+
+            request = Request(environ)
+            event = envelope = None
+            content_encoding = request.headers.get("content-encoding")
+            if content_encoding == "gzip":
+                rdr = gzip.GzipFile(fileobj=io.BytesIO(request.data))
+                compressed = True
+            elif content_encoding == "br":
+                rdr = io.BytesIO(brotli.decompress(request.data))
+                compressed = True
+            else:
+                rdr = io.BytesIO(request.data)
+                compressed = False
 
-    def __call__(self, environ, start_response):
-        """
-        This is the WSGI application.
-        """
-        request = Request(environ)
-        event = envelope = None
-        content_encoding = request.headers.get("content-encoding")
-        if content_encoding == "gzip":
-            rdr = gzip.GzipFile(fileobj=io.BytesIO(request.data))
-            compressed = True
-        elif content_encoding == "br":
-            assert brotli is not None
-            rdr = io.BytesIO(brotli.decompress(request.data))
-            compressed = True
-        else:
-            rdr = io.BytesIO(request.data)
-            compressed = False
-
-        if request.mimetype == "application/json":
-            event = parse_json(rdr.read())
-        else:
-            envelope = Envelope.deserialize_from(rdr)
-
-        self.captured.append(
-            CapturedData(
-                path=request.path,
-                event=event,
-                envelope=envelope,
-                compressed=compressed,
+            if request.mimetype == "application/json":
+                event = parse_json(rdr.read())
+            else:
+                envelope = Envelope.deserialize_from(rdr)
+
+            self.captured.append(
+                CapturedData(
+                    path=request.path,
+                    event=event,
+                    envelope=envelope,
+                    compressed=compressed,
+                )
             )
-        )
 
-        response = Response(status=self.code)
-        response.headers.extend(self.headers)
-        return response(environ, start_response)
+            response = Response(status=self.code)
+            response.headers.extend(self.headers)
+            return response(environ, start_response)
+
+    return CapturingServer()

sentry_sdk/integrations/pymongo.py

@@ -234,7 +234,6 @@ def test_clickhouse_client_spans(
     sentry_init,
     capture_events,
     capture_items,
-    capture_envelopes,
     span_streaming,
 ):
     sentry_init(
@@ -278,6 +277,7 @@ def test_clickhouse_client_spans(
                     "db.namespace": "",
                     "db.user": "default",
                     "sentry.op": "db",
+                    "db.query.text": "DROP TABLE IF EXISTS test",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
                     "server.port": 9000,
@@ -293,6 +293,7 @@ def test_clickhouse_client_spans(
                     "db.namespace": "",
                     "db.user": "default",
                     "sentry.op": "db",
+                    "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
                     "server.port": 9000,
@@ -308,6 +309,7 @@ def test_clickhouse_client_spans(
                     "db.namespace": "",
                     "db.user": "default",
                     "sentry.op": "db",
+                    "db.query.text": "INSERT INTO test (x) VALUES",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
                     "server.port": 9000,
@@ -323,6 +325,7 @@ def test_clickhouse_client_spans(
                     "db.namespace": "",
                     "db.user": "default",
                     "sentry.op": "db",
+                    "db.query.text": "INSERT INTO test (x) VALUES",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
                     "server.port": 9000,
@@ -338,6 +341,7 @@ def test_clickhouse_client_spans(
                     "db.namespace": "",
                     "db.user": "default",
                     "sentry.op": "db",
+                    "db.query.text": "SELECT sum(x) FROM test WHERE x > 150",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
                     "server.port": 9000,
@@ -563,6 +567,7 @@ def test_clickhouse_client_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "DROP TABLE IF EXISTS test",
                     "server.address": "localhost",
                     "server.port": 9000,
                     "thread.id": mock.ANY,
@@ -577,6 +582,7 @@ def test_clickhouse_client_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory",
                     "server.address": "localhost",
                     "server.port": 9000,
                     "thread.id": mock.ANY,
@@ -591,6 +597,7 @@ def test_clickhouse_client_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "INSERT INTO test (x) VALUES",
                     "server.address": "localhost",
                     "server.port": 9000,
                     "thread.id": mock.ANY,
@@ -605,6 +612,7 @@ def test_clickhouse_client_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "INSERT INTO test (x) VALUES",
                     "server.address": "localhost",
                     "server.port": 9000,
                 },
@@ -617,6 +625,7 @@ def test_clickhouse_client_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "SELECT sum(x) FROM test WHERE x > 150",
                     "server.address": "localhost",
                     "server.port": 9000,
                 },
@@ -1012,6 +1021,7 @@ def test_clickhouse_dbapi_spans(
                     "db.driver.name": "clickhouse-driver",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "DROP TABLE IF EXISTS test",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
@@ -1027,6 +1037,7 @@ def test_clickhouse_dbapi_spans(
                     "db.driver.name": "clickhouse-driver",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
@@ -1042,6 +1053,7 @@ def test_clickhouse_dbapi_spans(
                     "db.driver.name": "clickhouse-driver",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "INSERT INTO test (x) VALUES",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
@@ -1057,6 +1069,7 @@ def test_clickhouse_dbapi_spans(
                     "db.driver.name": "clickhouse-driver",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "INSERT INTO test (x) VALUES",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
@@ -1072,6 +1085,7 @@ def test_clickhouse_dbapi_spans(
                     "db.driver.name": "clickhouse-driver",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "SELECT sum(x) FROM test WHERE x > 150",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
@@ -1266,6 +1280,7 @@ def test_clickhouse_dbapi_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "DROP TABLE IF EXISTS test",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
@@ -1280,6 +1295,7 @@ def test_clickhouse_dbapi_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "CREATE TABLE test (x Int32) ENGINE = Memory",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
@@ -1294,6 +1310,7 @@ def test_clickhouse_dbapi_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "INSERT INTO test (x) VALUES",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
@@ -1308,6 +1325,7 @@ def test_clickhouse_dbapi_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "INSERT INTO test (x) VALUES",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",
@@ -1322,6 +1340,7 @@ def test_clickhouse_dbapi_spans_with_pii(
                     "db.system.name": "clickhouse",
                     "db.namespace": "",
                     "db.user": "default",
+                    "db.query.text": "SELECT sum(x) FROM test WHERE x > 150",
                     "sentry.op": "db",
                     "sentry.origin": "auto.db.clickhouse_driver",
                     "server.address": "localhost",

tests/conftest.py

@@ -162,6 +162,9 @@ def test_segment_span_streaming(sentry_init, capture_items, mongo_server, with_p
     assert insert_success["attributes"]["db.collection.name"] == "test_collection"
     assert insert_fail["attributes"]["db.collection.name"] == "erroneous"
 
+    for span in find, insert_success, insert_fail:
+        assert span["attributes"][SPANDATA.DB_QUERY_TEXT] == span["name"]
+
     if with_pii:
         assert "1" in find["name"]
         assert "2" in insert_success["name"]
@@ -247,6 +250,7 @@ def test_breadcrumbs_span_streaming(sentry_init, capture_items, mongo_server, wi
     assert data["db.driver.name"] == "pymongo"
     assert data["db.operation.name"] == "find"
     assert data["db.collection.name"] == "test_collection"
+    assert data["db.query.text"] == crumb["message"]
     assert data["sentry.op"] == "db"
     assert data["sentry.origin"] == "auto.db.pymongo"
     assert data[SPANDATA.SERVER_ADDRESS] == "localhost"