feat(autofix): Add analytics events for autofix phase start and completion (#112098)

Add per-phase analytics events for the autofix pipeline so we can track
when each phase (root cause, solution, code changes, impact assessment,
triage) starts and completes.

Each phase gets its own event class (`ai.autofix.root_cause.started`,
`ai.autofix.root_cause.completed`, etc.) plus a dedicated
`ai.autofix.pr_created.completed` event. All events include `group_id`,
`referrer`, `organization_id`, and `project_id`.

Started events are recorded in `trigger_autofix_explorer` when a phase
kicks
off. Completed events are recorded in
`AutofixOnCompletionHook._send_step_webhook`
when the on-completion hook fires.

Author: chromy

src/sentry/analytics/events/__init__.py

@@ -8,6 +8,7 @@
 from .api_token_deleted import *  # noqa: F401,F403
 from .auth_v2 import *  # noqa: F401,F403
 from .autofix_automation_events import *  # noqa: F401,F403
+from .autofix_events import *  # noqa: F401,F403
 from .checkin_processing_error_stored import *  # noqa: F401,F403
 from .codeowners_assignment import *  # noqa: F401,F403
 from .codeowners_created import *  # noqa: F401,F403

src/sentry/analytics/events/autofix_events.py

@@ -0,0 +1,89 @@
+from sentry import analytics
+
+
+@analytics.eventclass()
+class AiAutofixPhaseEvent(analytics.Event):
+    organization_id: int
+    project_id: int
+    group_id: int
+    referrer: str | None
+
+
+@analytics.eventclass("ai.autofix.root_cause.started")
+class AiAutofixRootCauseStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.solution.started")
+class AiAutofixSolutionStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.code_changes.started")
+class AiAutofixCodeChangesStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.root_cause.completed")
+class AiAutofixRootCauseCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.solution.completed")
+class AiAutofixSolutionCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.code_changes.completed")
+class AiAutofixCodeChangesCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.impact_assessment.started")
+class AiAutofixImpactAssessmentStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.impact_assessment.completed")
+class AiAutofixImpactAssessmentCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.triage.started")
+class AiAutofixTriageStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.triage.completed")
+class AiAutofixTriageCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.pr_created.started")
+class AiAutofixPrCreatedStartedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.pr_created.completed")
+class AiAutofixPrCreatedCompletedEvent(AiAutofixPhaseEvent):
+    pass
+
+
+@analytics.eventclass("ai.autofix.agent_handoff")
+class AiAutofixAgentHandoffEvent(AiAutofixPhaseEvent):
+    pass
+
+
+analytics.register(AiAutofixRootCauseStartedEvent)
+analytics.register(AiAutofixSolutionStartedEvent)
+analytics.register(AiAutofixCodeChangesStartedEvent)
+analytics.register(AiAutofixRootCauseCompletedEvent)
+analytics.register(AiAutofixSolutionCompletedEvent)
+analytics.register(AiAutofixCodeChangesCompletedEvent)
+analytics.register(AiAutofixImpactAssessmentStartedEvent)
+analytics.register(AiAutofixImpactAssessmentCompletedEvent)
+analytics.register(AiAutofixTriageStartedEvent)
+analytics.register(AiAutofixTriageCompletedEvent)
+analytics.register(AiAutofixPrCreatedStartedEvent)
+analytics.register(AiAutofixPrCreatedCompletedEvent)
+analytics.register(AiAutofixAgentHandoffEvent)

src/sentry/seer/autofix/autofix_agent.py

@@ -8,6 +8,22 @@
 from pydantic import BaseModel
 from rest_framework.exceptions import PermissionDenied
 
+from sentry import analytics
+from sentry.analytics.events.autofix_events import (
+    AiAutofixAgentHandoffEvent,
+    AiAutofixCodeChangesCompletedEvent,
+    AiAutofixCodeChangesStartedEvent,
+    AiAutofixImpactAssessmentCompletedEvent,
+    AiAutofixImpactAssessmentStartedEvent,
+    AiAutofixPhaseEvent,
+    AiAutofixPrCreatedStartedEvent,
+    AiAutofixRootCauseCompletedEvent,
+    AiAutofixRootCauseStartedEvent,
+    AiAutofixSolutionCompletedEvent,
+    AiAutofixSolutionStartedEvent,
+    AiAutofixTriageCompletedEvent,
+    AiAutofixTriageStartedEvent,
+)
 from sentry.constants import ENABLE_SEER_CODING_DEFAULT
 from sentry.seer.autofix.artifact_schemas import (
     ImpactAssessmentArtifact,
@@ -82,34 +98,48 @@ def __init__(
         artifact_schema: type[BaseModel] | None,
         prompt_fn: Callable[..., str],
         enable_coding: bool = False,
+        started_event: type[AiAutofixPhaseEvent] | None = None,
+        completed_event: type[AiAutofixPhaseEvent] | None = None,
     ):
         self.artifact_schema = artifact_schema
         self.prompt_fn = prompt_fn
         self.enable_coding = enable_coding
+        self.started_event = started_event
+        self.completed_event = completed_event
 
 
 # Step configurations mapping step to its artifact schema and prompt
 STEP_CONFIGS: dict[AutofixStep, StepConfig] = {
     AutofixStep.ROOT_CAUSE: StepConfig(
         artifact_schema=RootCauseArtifact,
         prompt_fn=root_cause_prompt,
+        started_event=AiAutofixRootCauseStartedEvent,
+        completed_event=AiAutofixRootCauseCompletedEvent,
     ),
     AutofixStep.SOLUTION: StepConfig(
         artifact_schema=SolutionArtifact,
         prompt_fn=solution_prompt,
+        started_event=AiAutofixSolutionStartedEvent,
+        completed_event=AiAutofixSolutionCompletedEvent,
     ),
     AutofixStep.CODE_CHANGES: StepConfig(
         artifact_schema=None,  # Code changes read from file_patches
         prompt_fn=code_changes_prompt,
         enable_coding=True,
+        started_event=AiAutofixCodeChangesStartedEvent,
+        completed_event=AiAutofixCodeChangesCompletedEvent,
     ),
     AutofixStep.IMPACT_ASSESSMENT: StepConfig(
         artifact_schema=ImpactAssessmentArtifact,
         prompt_fn=impact_assessment_prompt,
+        started_event=AiAutofixImpactAssessmentStartedEvent,
+        completed_event=AiAutofixImpactAssessmentCompletedEvent,
     ),
     AutofixStep.TRIAGE: StepConfig(
         artifact_schema=TriageArtifact,
         prompt_fn=triage_prompt,
+        started_event=AiAutofixTriageStartedEvent,
+        completed_event=AiAutofixTriageCompletedEvent,
     ),
 }
 
@@ -215,6 +245,16 @@ def trigger_autofix_explorer(
     """
 
     config = STEP_CONFIGS[step]
+
+    if config.started_event is not None:
+        analytics.record(
+            config.started_event(
+                organization_id=group.organization.id,
+                project_id=group.project_id,
+                group_id=group.id,
+                referrer=referrer.value,
+            )
+        )
     client = get_autofix_explorer_client(
         group,
         intelligence_level=intelligence_level,
@@ -500,6 +540,15 @@ def trigger_coding_agent_handoff(
         auto_create_pr=auto_create_pr,
     )
 
+    analytics.record(
+        AiAutofixAgentHandoffEvent(
+            organization_id=group.organization.id,
+            project_id=group.project_id,
+            group_id=group.id,
+            referrer=referrer.value,
+        )
+    )
+
     metrics.incr(
         "autofix.explorer.trigger",
         tags={"step": "coding_agent_handoff", "referrer": referrer.value},
@@ -532,6 +581,15 @@ def trigger_push_changes(
     if group_id != group.id:
         raise SeerPermissionError("Unknown run id for group")
 
+    analytics.record(
+        AiAutofixPrCreatedStartedEvent(
+            organization_id=group.organization.id,
+            project_id=group.project_id,
+            group_id=group.id,
+            referrer=referrer.value,
+        )
+    )
+
     client.push_changes(
         run_id,
         repo_name=repo_name,

src/sentry/seer/autofix/on_completion_hook.py

@@ -5,11 +5,13 @@
 
 from django.utils import timezone
 
-from sentry import features
+from sentry import analytics, features
+from sentry.analytics.events.autofix_events import AiAutofixPrCreatedCompletedEvent
 from sentry.models.group import Group
 from sentry.models.organization import Organization
 from sentry.models.project import Project
 from sentry.seer.autofix.autofix_agent import (
+    STEP_CONFIGS,
     AutofixStep,
     trigger_autofix_explorer,
     trigger_coding_agent_handoff,
@@ -136,6 +138,8 @@ def _send_step_webhook(
         # to find which step just completed
         webhook_action_type: SeerActionType | None = None
 
+        is_pr_created = False
+
         if current_step is not None:
             artifact = cls.find_latest_artifact_for_step(state, current_step)
             if artifact is not None:
@@ -166,6 +170,15 @@ def _send_step_webhook(
                     }
                     for pull_request in state.repo_pr_states.values()
                 ]
+                is_pr_created = True
+                analytics.record(
+                    AiAutofixPrCreatedCompletedEvent(
+                        organization_id=organization.id,
+                        project_id=group.project_id,
+                        group_id=group.id,
+                        referrer=None if current_referrer is None else current_referrer.value,
+                    )
+                )
             else:
                 webhook_action_type = SeerActionType.CODING_COMPLETED
                 diffs_by_repo = state.get_diffs_by_repo()
@@ -230,11 +243,21 @@ def _send_step_webhook(
                 },
             )
 
-        if current_step is not None:
+        if current_step is not None and not is_pr_created:
             referrer = current_referrer.value if current_referrer is not None else None
             metrics.incr(
                 "autofix.explorer.complete", tags={"step": current_step.value, "referrer": referrer}
             )
+            completed_event_cls = STEP_CONFIGS[current_step].completed_event
+            if completed_event_cls is not None:
+                analytics.record(
+                    completed_event_cls(
+                        organization_id=organization.id,
+                        project_id=group.project_id,
+                        group_id=group.id,
+                        referrer=referrer,
+                    )
+                )
 
     @classmethod
     def _maybe_trigger_supergroups_embedding(