diff --git a/src/sentry/workflow_engine/tasks/actions.py b/src/sentry/workflow_engine/tasks/actions.py index 4c67ce086b09..baa968fc8173 100644 --- a/src/sentry/workflow_engine/tasks/actions.py +++ b/src/sentry/workflow_engine/tasks/actions.py @@ -2,6 +2,7 @@ from django.db.models import Value from taskbroker_client.retry import Retry +from taskbroker_client.worker.workerchild import ProcessingDeadlineExceeded from sentry.eventstream.base import GroupState from sentry.models.activity import Activity @@ -9,7 +10,7 @@ from sentry.models.project import Project from sentry.services.eventstore.models import GroupEvent from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker import namespaces from sentry.utils import metrics from sentry.utils.exceptions import timeout_grouping_context @@ -74,14 +75,24 @@ def build_trigger_action_task_params( name="sentry.workflow_engine.tasks.trigger_action", namespace=namespaces.workflow_engine_tasks, processing_deadline_duration=30, - retry=Retry(times=3, delay=5), + retry=Retry( + times=3, + delay=5, + on=(Exception, ProcessingDeadlineExceeded), + ignore=( + Action.DoesNotExist, + Group.DoesNotExist, + Project.DoesNotExist, + ProjectNotActiveError, + Workflow.DoesNotExist, + ), + ), silo_mode=SiloMode.CELL, -) -@retry( - timeouts=True, - raise_on_no_retries=False, - ignore_and_capture=(Action.DoesNotExist, Group.DoesNotExist), - ignore=(Project.DoesNotExist, ProjectNotActiveError, Workflow.DoesNotExist), + silenced_exceptions=( + Project.DoesNotExist, + ProjectNotActiveError, + Workflow.DoesNotExist, + ), ) def trigger_action( action_id: int, diff --git a/src/sentry/workflow_engine/tasks/delayed_workflows.py b/src/sentry/workflow_engine/tasks/delayed_workflows.py index 3453e1e77bcd..61d5021a4606 100644 --- a/src/sentry/workflow_engine/tasks/delayed_workflows.py +++ b/src/sentry/workflow_engine/tasks/delayed_workflows.py @@ -3,9 +3,10 @@ from typing import Any from taskbroker_client.retry import Retry +from taskbroker_client.worker.workerchild import ProcessingDeadlineExceeded from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker.namespaces import workflow_engine_tasks from sentry.utils.exceptions import quiet_redis_noise from sentry.workflow_engine.utils import log_context @@ -17,10 +18,16 @@ name="sentry.workflow_engine.tasks.delayed_workflows", namespace=workflow_engine_tasks, processing_deadline_duration=60, - retry=Retry(times=5, delay=5), + retry=Retry( + times=5, + delay=5, + on=( + Exception, + ProcessingDeadlineExceeded, + ), + ), silo_mode=SiloMode.CELL, ) -@retry(timeouts=True) @log_context.root() def process_delayed_workflows( project_id: int, batch_key: str | None = None, *args: Any, **kwargs: Any diff --git a/src/sentry/workflow_engine/tasks/workflows.py b/src/sentry/workflow_engine/tasks/workflows.py index 31aaf5f53c76..fc8f625cdeef 100644 --- a/src/sentry/workflow_engine/tasks/workflows.py +++ b/src/sentry/workflow_engine/tasks/workflows.py @@ -6,6 +6,7 @@ from django.db import router, transaction from google.api_core.exceptions import RetryError from taskbroker_client.retry import Retry, retry_task +from taskbroker_client.worker.workerchild import ProcessingDeadlineExceeded from sentry.eventstream.base import GroupState from sentry.locks import locks @@ -15,7 +16,7 @@ from sentry.sentry_apps.tasks.service_hooks import kick_off_service_hooks from sentry.services.eventstore.models import GroupEvent from sentry.silo.base import SiloMode -from sentry.tasks.base import instrumented_task, retry +from sentry.tasks.base import instrumented_task from sentry.taskworker import namespaces from sentry.utils import metrics from sentry.utils.exceptions import quiet_redis_noise @@ -37,10 +38,9 @@ name="sentry.workflow_engine.tasks.process_workflow_activity", namespace=namespaces.workflow_engine_tasks, processing_deadline_duration=60, - retry=Retry(times=3, delay=5), + retry=Retry(times=3, delay=5, on=(Exception,)), silo_mode=SiloMode.CELL, ) -@retry def process_workflow_activity(activity_id: int, group_id: int, detector_id: int) -> None: """ Process a workflow task identified by the given activity, group, and detector. @@ -89,14 +89,25 @@ def process_workflow_activity(activity_id: int, group_id: int, detector_id: int) name="sentry.workflow_engine.tasks.process_workflows_event", namespace=namespaces.workflow_engine_tasks, processing_deadline_duration=60, - retry=Retry(times=3, delay=5), + retry=Retry( + times=3, + delay=5, + on=(Exception, ProcessingDeadlineExceeded), + ignore=( + EventNotFoundError, + Group.DoesNotExist, + Project.DoesNotExist, + ProjectNotActiveError, + ), + ), silo_mode=SiloMode.CELL, -) -@retry( - timeouts=True, - exclude=EventNotFoundError, - ignore=(Group.DoesNotExist, Project.DoesNotExist, ProjectNotActiveError), - on_silent=DataConditionGroup.DoesNotExist, + silenced_exceptions=( + EventNotFoundError, + DataConditionGroup.DoesNotExist, + Group.DoesNotExist, + Project.DoesNotExist, + ProjectNotActiveError, + ), ) def process_workflows_event( event_id: str,