From bbd23f9d7be9f7355ef4c7dca833aa0d469bf8c5 Mon Sep 17 00:00:00 2001 From: Mark Story Date: Thu, 18 Jun 2026 13:58:06 -0400 Subject: [PATCH 1/7] feat(tasks) Add taskbroker-client metrics for tasks We want to standardize metrics prefixes and tags for taskworkers. Having standardized metrics makes building dashboards and alerting easier for platform teams, and makes documenting metrics for others simpler. I've put usage of the new metrics backend behind an env var so that I can incrementally enable and validate the new metrics. Refs STREAM-1174 --- pyproject.toml | 2 +- src/sentry/taskworker/runtime.py | 18 +++++++++++++++++- uv.lock | 6 +++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3caf0b14470ab4..47398d62a2939f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,7 +107,7 @@ dependencies = [ "statsd>=3.3.0", "structlog>=22.1.0", "symbolic>=13.1.1", - "taskbroker-client>=0.19.1", + "taskbroker-client>=0.19.2", "tiktoken>=0.8.0", "tokenizers>=0.22.0", "tldextract>=5.1.2", diff --git a/src/sentry/taskworker/runtime.py b/src/sentry/taskworker/runtime.py index 2ccba440dc0e46..3b15db70385c10 100644 --- a/src/sentry/taskworker/runtime.py +++ b/src/sentry/taskworker/runtime.py @@ -1,6 +1,9 @@ +import os + from django.conf import settings from django.core.cache import cache from taskbroker_client.app import TaskbrokerApp +from taskbroker_client.metrics import DatadogMetrics from sentry.taskworker.adapters import ( DjangoCacheAtMostOnceStore, @@ -10,10 +13,23 @@ make_producer, ) +metrics_class = SentryMetricsBackend() +if os.getenv("USE_TASKWORKER_METRICS", None): + # Metrics created by this interface will not + # have `sentry.` prefix, and will not have + # K8S_LABEL applied. + metrics_class = DatadogMetrics( + application="sentry", + statsd_host=os.getenv("HOST_IP", "127.0.0.1"), + statsd_port=os.getenv("SENTRY_STATSD_PORT", 8126), + sample_rate=settings.SENTRY_METRICS_SAMPLE_RATE, + enable_prefixed_metrics=True, + ) + app = TaskbrokerApp( name="sentry", producer_factory=make_producer, - metrics_class=SentryMetricsBackend(), + metrics_class=metrics_class, router_class=SentryRouter(), at_most_once_store=DjangoCacheAtMostOnceStore(cache), context_hooks=[ViewerContextHook()], diff --git a/uv.lock b/uv.lock index 8c7620a993077e..c837477b18035c 100644 --- a/uv.lock +++ b/uv.lock @@ -2427,7 +2427,7 @@ requires-dist = [ { name = "stripe", specifier = ">=6.7.0" }, { name = "structlog", specifier = ">=22.1.0" }, { name = "symbolic", specifier = ">=13.1.1" }, - { name = "taskbroker-client", specifier = ">=0.19.1" }, + { name = "taskbroker-client", specifier = ">=0.19.2" }, { name = "tiktoken", specifier = ">=0.8.0" }, { name = "tldextract", specifier = ">=5.1.2" }, { name = "tokenizers", specifier = ">=0.22.0" }, @@ -2820,7 +2820,7 @@ wheels = [ [[package]] name = "taskbroker-client" -version = "0.19.1" +version = "0.19.2" source = { registry = "https://pypi.devinfra.sentry.io/simple" } dependencies = [ { name = "confluent-kafka", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -2838,7 +2838,7 @@ dependencies = [ { name = "zstandard", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] wheels = [ - { url = "https://pypi.devinfra.sentry.io/wheels/taskbroker_client-0.19.1-py3-none-any.whl", hash = "sha256:fc20b0aa19b0dad7caf8f1182b615b624158194e766f7d84abdaa40633b2d03d" }, + { url = "https://pypi.devinfra.sentry.io/wheels/taskbroker_client-0.19.2-py3-none-any.whl", hash = "sha256:9c1408d72bce9f85b05479137145846d2528c20403d2a44ba088327e4871219e" }, ] [[package]] From 6a04c4e8d1ee2d0775b0875b7e8c69749fb40217 Mon Sep 17 00:00:00 2001 From: Mark Story Date: Thu, 18 Jun 2026 16:12:49 -0400 Subject: [PATCH 2/7] Address feedback and infer metrics config from settings --- src/sentry/taskworker/runtime.py | 46 ++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/src/sentry/taskworker/runtime.py b/src/sentry/taskworker/runtime.py index 3b15db70385c10..31c0f30f5eca79 100644 --- a/src/sentry/taskworker/runtime.py +++ b/src/sentry/taskworker/runtime.py @@ -1,3 +1,4 @@ +import logging import os from django.conf import settings @@ -13,18 +14,41 @@ make_producer, ) +logger = logging.getLogger(__name__) + + +def _extract_metrics_config() -> tuple[str | None, int | None]: + host, port = None, None + metric_options = settings.SENTRY_METRICS_OPTIONS + try: + # Use the metrics settings options to infer the host/port. + # The metrics options have different structures depending on which backend is used. + if settings.SENTRY_METRICS_BACKEND == "sentry.metrics.dualwrite.DualWriteMetricsBackend": + metric_options = settings.SENTRY_METRICS_OPTIONS["primary_backend_args"] + + # Some backends use `host` and others use `statsd_host` + host = metric_options.get("host", None) or metric_options.get("statsd_host", None) + port = metric_options.get("port", None) or metric_options.get("statsd_port", None) + except Exception as e: + logger.warning("Could not extract metrics settings", extra={"error": str(e)}) + return host, port + + metrics_class = SentryMetricsBackend() -if os.getenv("USE_TASKWORKER_METRICS", None): - # Metrics created by this interface will not - # have `sentry.` prefix, and will not have - # K8S_LABEL applied. - metrics_class = DatadogMetrics( - application="sentry", - statsd_host=os.getenv("HOST_IP", "127.0.0.1"), - statsd_port=os.getenv("SENTRY_STATSD_PORT", 8126), - sample_rate=settings.SENTRY_METRICS_SAMPLE_RATE, - enable_prefixed_metrics=True, - ) + +if os.getenv("USE_TASKWORKER_METRICS", None) == "1": + host, port = _extract_metrics_config() + if host and port: + # Metrics created by this interface will not + # have `sentry.` prefix, and will not have + # K8S_LABEL applied. + metrics_class = DatadogMetrics( + application="sentry", + statsd_host=host, + statsd_port=port, + sample_rate=settings.SENTRY_METRICS_SAMPLE_RATE, + enable_prefixed_metrics=True, + ) app = TaskbrokerApp( name="sentry", From 6fc270b51ddc96c184cdf68bee867ee40b41819d Mon Sep 17 00:00:00 2001 From: Mark Story Date: Thu, 18 Jun 2026 16:20:33 -0400 Subject: [PATCH 3/7] mypy --- src/sentry/taskworker/runtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/taskworker/runtime.py b/src/sentry/taskworker/runtime.py index 31c0f30f5eca79..9992eea931ffdc 100644 --- a/src/sentry/taskworker/runtime.py +++ b/src/sentry/taskworker/runtime.py @@ -4,7 +4,7 @@ from django.conf import settings from django.core.cache import cache from taskbroker_client.app import TaskbrokerApp -from taskbroker_client.metrics import DatadogMetrics +from taskbroker_client.metrics import DatadogMetrics, MetricsBackend from sentry.taskworker.adapters import ( DjangoCacheAtMostOnceStore, @@ -34,7 +34,7 @@ def _extract_metrics_config() -> tuple[str | None, int | None]: return host, port -metrics_class = SentryMetricsBackend() +metrics_class: MetricsBackend = SentryMetricsBackend() if os.getenv("USE_TASKWORKER_METRICS", None) == "1": host, port = _extract_metrics_config() From 2d76ea0a2e43ec221701e604edb692a284d623be Mon Sep 17 00:00:00 2001 From: Mark Story Date: Thu, 18 Jun 2026 16:27:41 -0400 Subject: [PATCH 4/7] fix ordering to address host confusion with datadog adapter --- src/sentry/taskworker/runtime.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentry/taskworker/runtime.py b/src/sentry/taskworker/runtime.py index 9992eea931ffdc..c3712145912978 100644 --- a/src/sentry/taskworker/runtime.py +++ b/src/sentry/taskworker/runtime.py @@ -27,8 +27,8 @@ def _extract_metrics_config() -> tuple[str | None, int | None]: metric_options = settings.SENTRY_METRICS_OPTIONS["primary_backend_args"] # Some backends use `host` and others use `statsd_host` - host = metric_options.get("host", None) or metric_options.get("statsd_host", None) - port = metric_options.get("port", None) or metric_options.get("statsd_port", None) + host = metric_options.get("statsd_host", None) or metric_options.get("host", None) + port = metric_options.get("statsd_port", None) or metric_options.get("port", None) except Exception as e: logger.warning("Could not extract metrics settings", extra={"error": str(e)}) return host, port From 56b556fbffe8466b6d2119fd4ec9ac022c4051bc Mon Sep 17 00:00:00 2001 From: Mark Story Date: Thu, 18 Jun 2026 16:29:04 -0400 Subject: [PATCH 5/7] Fix int cast --- src/sentry/taskworker/runtime.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sentry/taskworker/runtime.py b/src/sentry/taskworker/runtime.py index c3712145912978..8363fb63c54218 100644 --- a/src/sentry/taskworker/runtime.py +++ b/src/sentry/taskworker/runtime.py @@ -29,6 +29,7 @@ def _extract_metrics_config() -> tuple[str | None, int | None]: # Some backends use `host` and others use `statsd_host` host = metric_options.get("statsd_host", None) or metric_options.get("host", None) port = metric_options.get("statsd_port", None) or metric_options.get("port", None) + port = int(port) except Exception as e: logger.warning("Could not extract metrics settings", extra={"error": str(e)}) return host, port From 8cb98aa55c3feb96ce6dda35fa2b6ef4ac0159f7 Mon Sep 17 00:00:00 2001 From: Mark Story Date: Fri, 19 Jun 2026 11:00:17 -0400 Subject: [PATCH 6/7] mypy --- src/sentry/taskworker/runtime.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sentry/taskworker/runtime.py b/src/sentry/taskworker/runtime.py index 8363fb63c54218..c40bdb013dbc59 100644 --- a/src/sentry/taskworker/runtime.py +++ b/src/sentry/taskworker/runtime.py @@ -28,8 +28,9 @@ def _extract_metrics_config() -> tuple[str | None, int | None]: # Some backends use `host` and others use `statsd_host` host = metric_options.get("statsd_host", None) or metric_options.get("host", None) - port = metric_options.get("statsd_port", None) or metric_options.get("port", None) - port = int(port) + raw_port = metric_options.get("statsd_port", None) or metric_options.get("port", None) + if isinstance(raw_port, str): + port = int(raw_port) except Exception as e: logger.warning("Could not extract metrics settings", extra={"error": str(e)}) return host, port From 785bf9857a1a2521195b440f0167f72ac99006f4 Mon Sep 17 00:00:00 2001 From: Mark Story Date: Fri, 19 Jun 2026 11:02:58 -0400 Subject: [PATCH 7/7] forgot about int port --- src/sentry/taskworker/runtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/taskworker/runtime.py b/src/sentry/taskworker/runtime.py index c40bdb013dbc59..397a3bac160260 100644 --- a/src/sentry/taskworker/runtime.py +++ b/src/sentry/taskworker/runtime.py @@ -29,7 +29,7 @@ def _extract_metrics_config() -> tuple[str | None, int | None]: # Some backends use `host` and others use `statsd_host` host = metric_options.get("statsd_host", None) or metric_options.get("host", None) raw_port = metric_options.get("statsd_port", None) or metric_options.get("port", None) - if isinstance(raw_port, str): + if isinstance(raw_port, (str, int)): port = int(raw_port) except Exception as e: logger.warning("Could not extract metrics settings", extra={"error": str(e)})