Skip to content

Commit b658c3d

Browse files
feat(reports): Cache per-project weekly report metrics (#116739)
Resolves [ID-1589](https://linear.app/getsentry/issue/ID-1589/cache-total-project-errors-and-total-project-transactions-for-weekly). Goal: Cache per-project total errors and total transactions during weekly report generation so the frontend can display week-over-week percentage change without re-querying Snuba. **Redis Cache Layer** - Caches total errors and total transactions by Org Id, Project Id (no timestamp) - 10 day TTL --------- Co-authored-by: getsantry[bot] <66042841+getsantry[bot]@users.noreply.github.com>
1 parent 7ea92c3 commit b658c3d

3 files changed

Lines changed: 127 additions & 0 deletions

File tree

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
from __future__ import annotations
2+
3+
from typing import Any
4+
5+
from django.conf import settings
6+
from sentry_redis_tools.clients import RedisCluster, StrictRedis
7+
8+
from sentry.utils import json, metrics, redis
9+
10+
CACHE_TTL_SEC = 10 * 24 * 60 * 60 # 10 days
11+
KEY_PREFIX = "wr:proj_metrics"
12+
13+
14+
def _make_cache_key(org_id: int, project_id: int) -> str:
15+
return f"{KEY_PREFIX}:{org_id}:{project_id}"
16+
17+
18+
def _get_redis_client() -> RedisCluster[str] | StrictRedis[str]:
19+
return redis.redis_clusters.get(settings.SENTRY_WEEKLY_REPORTS_REDIS_CLUSTER)
20+
21+
22+
def cache_project_metrics(
23+
org_id: int,
24+
project_metrics: dict[int, dict[str, int]],
25+
) -> None:
26+
client = _get_redis_client()
27+
pipeline = client.pipeline()
28+
29+
for project_id, values in project_metrics.items():
30+
key = _make_cache_key(org_id, project_id)
31+
pipeline.set(key, json.dumps(values), ex=CACHE_TTL_SEC)
32+
33+
pipeline.execute()
34+
35+
36+
def read_project_metrics(
37+
org_id: int,
38+
project_ids: list[int],
39+
) -> dict[int, dict[str, Any]]:
40+
if not project_ids:
41+
return {}
42+
43+
client = _get_redis_client()
44+
pipeline = client.pipeline()
45+
46+
for project_id in project_ids:
47+
pipeline.get(_make_cache_key(org_id, project_id))
48+
49+
results = pipeline.execute()
50+
51+
result_map: dict[int, dict[str, Any]] = {}
52+
for i, project_id in enumerate(project_ids):
53+
raw = results[i]
54+
if raw is None:
55+
metrics.incr("weekly_report.cache.miss")
56+
else:
57+
result_map[project_id] = json.loads(raw)
58+
59+
return result_map

src/sentry/tasks/summaries/weekly_reports.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
OrganizationReportContextFactory,
3838
)
3939
from sentry.tasks.summaries.utils import ONE_DAY, OrganizationReportContext
40+
from sentry.tasks.summaries.weekly_report_cache import cache_project_metrics
4041
from sentry.taskworker.namespaces import reports_tasks
4142
from sentry.types.group import GroupSubStatus
4243
from sentry.users.services.user_option import user_option_service
@@ -218,6 +219,20 @@ def prepare_organization_report(
218219
lifecycle.record_halt(WeeklyReportHaltReason.EMPTY_REPORT)
219220
return
220221

222+
if not dry_run:
223+
try:
224+
project_metrics: dict[int, dict[str, int]] = {}
225+
for project_id, project_ctx in ctx.projects_context_map.items():
226+
if not project_ctx.check_if_project_is_empty():
227+
project_metrics[project_id] = {
228+
"e": project_ctx.accepted_error_count,
229+
"t": project_ctx.accepted_transaction_count,
230+
}
231+
if project_metrics:
232+
cache_project_metrics(organization_id, project_metrics)
233+
except Exception:
234+
sentry_sdk.capture_exception()
235+
221236
# Finally, deliver the reports
222237
batch = OrganizationReportBatch(ctx, batch_id, dry_run, target_user, email_override)
223238
with sentry_sdk.start_span(op="weekly_reports.deliver_reports"):
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
from sentry.tasks.summaries.weekly_report_cache import (
2+
_make_cache_key,
3+
cache_project_metrics,
4+
read_project_metrics,
5+
)
6+
from sentry.testutils.cases import TestCase
7+
8+
9+
class WeeklyReportCacheTest(TestCase):
10+
def test_make_cache_key(self) -> None:
11+
key = _make_cache_key(org_id=1, project_id=2)
12+
assert key == "wr:proj_metrics:1:2"
13+
14+
def test_write_and_read(self) -> None:
15+
org_id = self.organization.id
16+
project = self.create_project(organization=self.organization)
17+
18+
cache_project_metrics(org_id, {project.id: {"e": 500, "t": 3000}})
19+
20+
result = read_project_metrics(org_id=org_id, project_ids=[project.id])
21+
22+
assert result[project.id] == {"e": 500, "t": 3000}
23+
24+
def test_read_empty_cache(self) -> None:
25+
result = read_project_metrics(org_id=self.organization.id, project_ids=[999])
26+
27+
assert result == {}
28+
29+
def test_read_empty_project_ids(self) -> None:
30+
result = read_project_metrics(org_id=self.organization.id, project_ids=[])
31+
32+
assert result == {}
33+
34+
def test_write_empty_metrics_is_noop(self) -> None:
35+
cache_project_metrics(self.organization.id, {})
36+
37+
def test_multiple_projects(self) -> None:
38+
org_id = self.organization.id
39+
p1 = self.create_project(organization=self.organization)
40+
p2 = self.create_project(organization=self.organization)
41+
42+
cache_project_metrics(
43+
org_id,
44+
{
45+
p1.id: {"e": 100, "t": 200},
46+
p2.id: {"e": 300, "t": 400},
47+
},
48+
)
49+
50+
result = read_project_metrics(org_id=org_id, project_ids=[p1.id, p2.id])
51+
52+
assert result[p1.id] == {"e": 100, "t": 200}
53+
assert result[p2.id] == {"e": 300, "t": 400}

0 commit comments

Comments
 (0)