diff --git a/docs/snippets/providers/mailgun-snippet-autogenerated.mdx b/docs/snippets/providers/mailgun-snippet-autogenerated.mdx index 521dc778dd..5db9da9019 100644 --- a/docs/snippets/providers/mailgun-snippet-autogenerated.mdx +++ b/docs/snippets/providers/mailgun-snippet-autogenerated.mdx @@ -7,6 +7,9 @@ This provider requires authentication. - **sender**: Sender email address to validate (required: False, sensitive: False) - **email_domain**: Custom email domain for receiving alerts (required: False, sensitive: False) - **extraction**: Extraction Rules (required: False, sensitive: False) +- **skip_dmarc_reports**: Skip DMARC reports (required: False, sensitive: False) +- **skip_spf_reports**: Skip SPF reports (required: False, sensitive: False) +- **handle_emails_without_body**: Handle emails without body content (required: False, sensitive: False) ## In workflows diff --git a/keep-ui/entities/alerts/model/useAlerts.ts b/keep-ui/entities/alerts/model/useAlerts.ts index 9fa0a2dd96..111db3195b 100644 --- a/keep-ui/entities/alerts/model/useAlerts.ts +++ b/keep-ui/entities/alerts/model/useAlerts.ts @@ -134,12 +134,30 @@ export const useAlerts = () => { } }; + // Function to reprocess error alerts with updated provider code + // If alertId is provided, reprocesses that specific alert + // If no alertId is provided, reprocesses all error alerts + const reprocessErrorAlerts = async (alertId?: string) => { + if (!api.isReady()) return { success: false, message: "API not ready" }; + + try { + const payload = alertId ? { alert_id: alertId } : {}; + const result = await api.post(`/alerts/event/error/reprocess`, payload); + await mutate(); // Refresh the data + return { success: true, ...result }; + } catch (error) { + console.error("Failed to reprocess error alert(s):", error); + return { success: false, message: "Failed to reprocess" }; + } + }; + return { data, error, isLoading, mutate, dismissErrorAlerts, + reprocessErrorAlerts, }; }; diff --git a/keep-ui/features/alerts/alert-error-event-process/ui/alert-error-event-modal.tsx b/keep-ui/features/alerts/alert-error-event-process/ui/alert-error-event-modal.tsx index 642ef2086d..19580bfb4a 100644 --- a/keep-ui/features/alerts/alert-error-event-process/ui/alert-error-event-modal.tsx +++ b/keep-ui/features/alerts/alert-error-event-process/ui/alert-error-event-modal.tsx @@ -13,6 +13,7 @@ import { Button, } from "@tremor/react"; import { DynamicImageProviderIcon } from "@/components/ui/DynamicProviderIcon"; +import { toast } from "react-toastify"; interface ErrorAlert { id: string; @@ -32,9 +33,10 @@ export const AlertErrorEventModal: React.FC = ({ onClose, }) => { const { useErrorAlerts } = useAlerts(); - const { data: errorAlerts, dismissErrorAlerts } = useErrorAlerts(); + const { data: errorAlerts, dismissErrorAlerts, reprocessErrorAlerts } = useErrorAlerts(); const [selectedAlertId, setSelectedAlertId] = useState(""); const [isDismissing, setIsDismissing] = useState(false); + const [isReprocessing, setIsReprocessing] = useState(false); // Set the first alert as selected when data loads or changes React.useEffect(() => { @@ -96,6 +98,61 @@ export const AlertErrorEventModal: React.FC = ({ } }; + const handleReprocessSelected = async () => { + if (selectedAlert) { + setIsReprocessing(true); + try { + const result = await reprocessErrorAlerts(selectedAlert.id); + if (result.success) { + toast.success( + `Reprocessed successfully! ${result.message || ""}`, + { position: "top-right" } + ); + + // Handle navigation after successful reprocessing + if (errorAlerts?.length === 1) { + setSelectedAlertId(""); + onClose(); + } else if (parseInt(selectedAlertId, 10) === errorAlerts.length - 1) { + setSelectedAlertId((parseInt(selectedAlertId, 10) - 1).toString()); + } + } else { + toast.error(`Reprocessing failed: ${result.message}`, { + position: "top-right", + }); + } + } catch (error) { + console.error("Failed to reprocess alert:", error); + toast.error("Failed to reprocess alert", { position: "top-right" }); + } finally { + setIsReprocessing(false); + } + } + }; + + const handleReprocessAll = async () => { + setIsReprocessing(true); + try { + const result = await reprocessErrorAlerts(); + if (result.success) { + toast.success( + `Reprocessed ${result.successful || 0} alert(s) successfully!`, + { position: "top-right" } + ); + onClose(); + } else { + toast.error(`Reprocessing failed: ${result.message}`, { + position: "top-right", + }); + } + } catch (error) { + console.error("Failed to reprocess alerts:", error); + toast.error("Failed to reprocess alerts", { position: "top-right" }); + } finally { + setIsReprocessing(false); + } + }; + return ( = ({
+ + @@ -166,7 +240,7 @@ export const AlertErrorEventModal: React.FC = ({ color="orange" variant="secondary" onClick={handleDismissAll} - disabled={isDismissing} + disabled={isDismissing || isReprocessing} > {isDismissing ? "Dismissing..." : "Dismiss All"} diff --git a/keep/api/core/db.py b/keep/api/core/db.py index e2efea2f2b..d8e5a8382c 100644 --- a/keep/api/core/db.py +++ b/keep/api/core/db.py @@ -5855,6 +5855,67 @@ def get_error_alerts(tenant_id: str, limit: int = 100) -> List[AlertRaw]: ) +def get_error_alerts_to_reprocess( + tenant_id: str, alert_id: str | None = None +) -> List[AlertRaw]: + """ + Get error alerts to reprocess. + + Args: + tenant_id: Tenant ID + alert_id: Optional specific alert ID to reprocess + + Returns: + List of AlertRaw objects to reprocess + """ + with Session(engine) as session: + query = session.query(AlertRaw).filter( + AlertRaw.tenant_id == tenant_id, + AlertRaw.error == True, + AlertRaw.dismissed == False, + ) + + if alert_id: + if isinstance(alert_id, str): + alert_id_uuid = uuid.UUID(alert_id) + else: + alert_id_uuid = alert_id + query = query.filter(AlertRaw.id == alert_id_uuid) + + return query.all() + + +def dismiss_error_alert_by_id(tenant_id: str, alert_id: str, dismissed_by: str | None = None) -> None: + """ + Dismiss a specific error alert after successful reprocessing. + + Args: + tenant_id: Tenant ID + alert_id: Alert ID to dismiss + dismissed_by: Optional user who dismissed the alert + """ + with Session(engine) as session: + if isinstance(alert_id, str): + alert_id_uuid = uuid.UUID(alert_id) + else: + alert_id_uuid = alert_id + + stmt = ( + update(AlertRaw) + .where( + AlertRaw.id == alert_id_uuid, + AlertRaw.tenant_id == tenant_id, + ) + .values( + dismissed=True, + dismissed_by=dismissed_by, + dismissed_at=datetime.now(tz=timezone.utc), + ) + ) + session.execute(stmt) + session.commit() + + def dismiss_error_alerts(tenant_id: str, alert_id=None, dismissed_by=None) -> None: with Session(engine) as session: stmt = ( diff --git a/keep/api/routes/alerts.py b/keep/api/routes/alerts.py index 2adaa090dc..b8f0117e4d 100644 --- a/keep/api/routes/alerts.py +++ b/keep/api/routes/alerts.py @@ -30,6 +30,7 @@ from keep.api.core.cel_to_sql.sql_providers.base import CelToSqlException from keep.api.core.config import config from keep.api.core.db import dismiss_error_alerts as dismiss_error_alerts_db +from keep.api.core.db import dismiss_error_alert_by_id from keep.api.core.db import enrich_alerts_with_incidents from keep.api.core.db import get_alert_audit as get_alert_audit_db from keep.api.core.db import ( @@ -39,6 +40,7 @@ get_enrichment, ) from keep.api.core.db import get_error_alerts as get_error_alerts_db +from keep.api.core.db import get_error_alerts_to_reprocess from keep.api.core.db import ( get_last_alerts, get_last_alerts_by_fingerprints, @@ -1459,3 +1461,121 @@ def dismiss_error_alerts( ) return {"success": True, "message": "Successfully dismissed all alerts"} + + +@router.post( + "/event/error/reprocess", + description="Reprocess error alerts with updated provider code. If alert_id is provided, reprocesses that specific alert. If no alert_id is provided, reprocesses all error alerts.", +) +def reprocess_error_alerts( + request: DismissAlertRequest = None, + authenticated_entity: AuthenticatedEntity = Depends( + IdentityManagerFactory.get_auth_verifier(["write:alert"]) + ), +) -> dict: + """ + Reprocess failed events with current provider code. + If alert_id is provided, reprocesses that specific alert. + If no alert_id is provided, reprocesses all error alerts. + """ + tenant_id = authenticated_entity.tenant_id + alert_id = request.alert_id if request else None + + logger.info( + "Reprocessing error alerts", + extra={ + "tenant_id": tenant_id, + "alert_id": alert_id, + }, + ) + + # Get error alerts to reprocess + error_alerts = get_error_alerts_to_reprocess(tenant_id, alert_id) + + if not error_alerts: + logger.info( + "No error alerts found to reprocess", + extra={"tenant_id": tenant_id, "alert_id": alert_id}, + ) + return {"success": True, "message": "No error alerts found to reprocess", "successful": 0, "failed": 0, "total": 0} + + successful = 0 + failed = 0 + failed_alerts = [] + + for error_alert in error_alerts: + try: + logger.info( + "Attempting to reprocess error alert", + extra={ + "tenant_id": tenant_id, + "alert_id": str(error_alert.id), + "provider_type": error_alert.provider_type, + }, + ) + + # Attempt to reprocess the event + process_event( + ctx={}, # No arq context for manual reprocessing + tenant_id=tenant_id, + provider_type=error_alert.provider_type, + provider_id=None, + fingerprint=None, + api_key_name=None, + trace_id=None, + event=error_alert.raw_alert, + notify_client=True, + ) + + # If successful, mark the error alert as dismissed + dismiss_error_alert_by_id( + tenant_id, + str(error_alert.id), + dismissed_by=authenticated_entity.email + ) + successful += 1 + + logger.info( + "Successfully reprocessed error alert", + extra={ + "tenant_id": tenant_id, + "alert_id": str(error_alert.id), + }, + ) + + except Exception as e: + logger.error( + f"Failed to reprocess error alert: {e}", + extra={ + "tenant_id": tenant_id, + "alert_id": str(error_alert.id), + "error": str(e), + }, + ) + failed += 1 + failed_alerts.append( + {"alert_id": str(error_alert.id), "error": str(e)} + ) + + logger.info( + "Reprocessing completed", + extra={ + "tenant_id": tenant_id, + "successful": successful, + "failed": failed, + "total": len(error_alerts), + }, + ) + + response = { + "success": successful > 0, + "message": f"Reprocessed {successful} alert(s) successfully, {failed} failed", + "successful": successful, + "failed": failed, + "total": len(error_alerts), + } + + if failed_alerts: + response["failed_alerts"] = failed_alerts + + return response diff --git a/keep/providers/mailgun_provider/mailgun_provider.py b/keep/providers/mailgun_provider/mailgun_provider.py index 99f64601ed..940e472526 100644 --- a/keep/providers/mailgun_provider/mailgun_provider.py +++ b/keep/providers/mailgun_provider/mailgun_provider.py @@ -55,6 +55,33 @@ class MailgunProviderAuthConfig: "hint": "Read more about extraction in Keep's Mailgun documentation", }, ) + skip_dmarc_reports: bool = dataclasses.field( + default=False, + metadata={ + "required": False, + "description": "Skip DMARC reports", + "hint": "Enable to automatically skip DMARC aggregate reports (not recommended)", + "type": "switch", + }, + ) + skip_spf_reports: bool = dataclasses.field( + default=False, + metadata={ + "required": False, + "description": "Skip SPF reports", + "hint": "Enable to automatically skip SPF failure reports (not recommended)", + "type": "switch", + }, + ) + handle_emails_without_body: bool = dataclasses.field( + default=True, + metadata={ + "required": False, + "description": "Handle emails without body content", + "hint": "Create alerts for emails that only have subject/attachments", + "type": "switch", + }, + ) class MailgunProvider(BaseProvider): @@ -239,107 +266,367 @@ def setup_webhook( return {"route_id": route_id, "email": email} @staticmethod - def _format_alert( - event: dict, provider_instance: "MailgunProvider" = None - ) -> AlertDto: - # We receive FormData here, convert it to simple dict. - logger.info( - "Received alert from mail", - extra={ - "from": event["from"], - "subject": event.get("subject") - }, - ) - event = dict(event) - - source = event["from"] - name = event.get("subject", source) - body_plain = event.get("Body-plain") - message = event.get("stripped-text", body_plain) + def _is_dmarc_report(event: dict) -> bool: + """ + Detect DMARC reports using multiple indicators. + + Args: + event: Email event data + + Returns: + bool: True if email is a DMARC report + """ + # Check sender patterns + sender = event.get("from", "").lower() + dmarc_senders = [ + "noreply-dmarc-support@google.com", + "dmarc-support@google.com", + "dmarc@", + "postmaster@", + "noreply@google.com" + ] + + if any(dmarc_sender in sender for dmarc_sender in dmarc_senders): + return True + + # Check subject patterns + subject = event.get("subject", "").lower() + if any(pattern in subject for pattern in ["report domain:", "dmarc", "aggregate report"]): + return True + + # Check content type for ZIP attachments (DMARC reports are typically ZIP files) + content_type = event.get("Content-Type", "").lower() + if "application/zip" in content_type: + return True + + # Check raw content if available raw_content = event.get("raw_content") + if raw_content: + if isinstance(raw_content, bytes) and b"dmarc" in raw_content.lower(): + return True + elif isinstance(raw_content, str) and "dmarc" in raw_content.lower(): + return True + + return False - if isinstance(raw_content, bytes) and b"dmarc" in raw_content.lower(): - logger.warning("DMARC alert detected, skipping") - return None - elif isinstance(raw_content, str) and "dmarc" in raw_content.lower(): - logger.warning("DMARC alert detected, skipping") - return None - - if not name or not message: - raise Exception( - "Could not create alert from email when name or message is missing." - ) + @staticmethod + def _is_spf_report(event: dict) -> bool: + """ + Detect SPF failure reports. + + Args: + event: Email event data + + Returns: + bool: True if email is an SPF report + """ + subject = event.get("subject", "").lower() + sender = event.get("from", "").lower() + + spf_patterns = ["spf fail", "spf failure", "spf report", "sender policy framework"] + return any(pattern in subject or pattern in sender for pattern in spf_patterns) - try: - timestamp = datetime.datetime.fromtimestamp( - float(event["timestamp"]) - ).isoformat() - except Exception: - timestamp = datetime.datetime.now().isoformat() - # default values - severity = "info" - status = "firing" - - # clean redundant - event.pop("signature", "") - event.pop("token", "") - - logger.info("Basic formatting done") - - alert = AlertDto( - name=name, - source=[source], - message=message, - description=message, - lastReceived=timestamp, - severity=severity, - status=status, - raw_email={**event}, + @staticmethod + def _is_bounce_notification(event: dict) -> bool: + """ + Detect bounce notifications. + + Args: + event: Email event data + + Returns: + bool: True if email is a bounce notification + """ + sender = event.get("from", "").lower() + subject = event.get("subject", "").lower() + + bounce_senders = ["mailer-daemon@", "postmaster@", "bounce@"] + bounce_patterns = ["delivery failed", "returned mail", "undelivered mail", "mail delivery"] + + return ( + any(bounce_sender in sender for bounce_sender in bounce_senders) or + any(pattern in subject for pattern in bounce_patterns) ) - # now I want to add all attributes from raw_email to the alert dto, except the ones that are already set - for key, value in event.items(): - # avoid "-" in keys cuz CEL will failed [stripped-text screw CEL] - if not hasattr(alert, key) and "-" not in key: - setattr(alert, key, value) + @staticmethod + def _classify_email_type(event: dict) -> str: + """ + Classify email type for appropriate handling. + + Args: + event: Email event data + + Returns: + str: Email type (dmarc_report, spf_report, bounce, alert) + """ + if MailgunProvider._is_dmarc_report(event): + return "dmarc_report" + elif MailgunProvider._is_spf_report(event): + return "spf_report" + elif MailgunProvider._is_bounce_notification(event): + return "bounce" + else: + return "alert" + + @staticmethod + def _describe_attachments(event: dict) -> str: + """ + Create a description of email attachments. + + Args: + event: Email event data + + Returns: + str: Description of attachments + """ + attachment_count = event.get("attachment-count", "0") + attachments = [] + + # Try to get attachment info + for i in range(1, int(attachment_count) + 1): + attachment_key = f"attachment-{i}" + if attachment_key in event: + attachment_info = str(event[attachment_key]) + # Extract filename if possible + if "filename=" in attachment_info: + filename = attachment_info.split("filename=")[1].split(",")[0].strip("'\"") + attachments.append(filename) + + if attachments: + return f"{attachment_count} attachment(s): {', '.join(attachments)}" + return f"{attachment_count} attachment(s)" + + @staticmethod + def _extract_message_content(event: dict, email_type: str) -> str: + """ + Extract message content based on email type. + + Args: + event: Email event data + email_type: Type of email (dmarc_report, spf_report, bounce, alert) + + Returns: + str: Extracted message content + """ + # Try standard body fields first + message = event.get("stripped-text") or event.get("Body-plain") + + if message: + return message + + # For DMARC reports, use subject as message + if email_type == "dmarc_report": + subject = event.get("subject", "") + if subject: + return f"DMARC Report: {subject}" + + # For emails with attachments, describe the attachment + if event.get("attachment-count") and int(event.get("attachment-count", 0)) > 0: + attachment_info = MailgunProvider._describe_attachments(event) + subject = event.get("subject", "") + if subject: + return f"{subject} ({attachment_info})" + return f"Email with {attachment_info}" + + # Fallback to subject + subject = event.get("subject", "") + if subject: + return subject + + return "No message content available" + @staticmethod + def _log_email_processing(event: dict, email_type: str, action: str): + """ + Enhanced logging for email processing. + + Args: + event: Email event data + email_type: Type of email + action: Action taken (skipped, processed, etc.) + """ logger.info( - "Alert formatted", + f"Email processing: {action}", + extra={ + "email_type": email_type, + "from": event.get("from"), + "subject": event.get("subject"), + "has_body": bool(event.get("Body-plain") or event.get("stripped-text")), + "has_attachments": bool(event.get("attachment-count")), + "content_type": event.get("Content-Type"), + "action": action + } ) - if provider_instance: + @staticmethod + def _format_alert( + event: dict, provider_instance: "MailgunProvider" = None + ) -> AlertDto: + """ + Format email event into an AlertDto. + + Args: + event: Email event data + provider_instance: Optional MailgunProvider instance for config access + + Returns: + AlertDto or None if email should be skipped + """ + try: + # We receive FormData here, convert it to simple dict. logger.info( - "Provider instance found", + "Received alert from mail", + extra={ + "from": event.get("from"), + "subject": event.get("subject") + }, ) - extraction_rules = provider_instance.authentication_config.extraction - if extraction_rules: - logger.info( - "Extraction rules found", + event = dict(event) + + # Classify email type first + email_type = MailgunProvider._classify_email_type(event) + + # Check provider instance configuration for skip settings + skip_dmarc = True # Default + skip_spf = True # Default + handle_no_body = True # Default + + if provider_instance and hasattr(provider_instance, 'authentication_config'): + skip_dmarc = getattr(provider_instance.authentication_config, 'skip_dmarc_reports', True) + skip_spf = getattr(provider_instance.authentication_config, 'skip_spf_reports', True) + handle_no_body = getattr(provider_instance.authentication_config, 'handle_emails_without_body', True) + + # Handle DMARC reports + if email_type == "dmarc_report" and skip_dmarc: + MailgunProvider._log_email_processing(event, email_type, "skipped (DMARC report)") + return None + + # Handle SPF reports + if email_type == "spf_report" and skip_spf: + MailgunProvider._log_email_processing(event, email_type, "skipped (SPF report)") + return None + + # Handle bounce notifications (optionally skip) + if email_type == "bounce": + MailgunProvider._log_email_processing(event, email_type, "processing (bounce notification)") + + # Extract basic fields + source = event.get("from", "unknown@unknown.com") + name = event.get("subject", source) + + # Extract message content with fallback logic + message = MailgunProvider._extract_message_content(event, email_type) + + # Validate required fields with flexible handling + if not name: + name = source or "Unknown Email" + logger.warning( + "Email has no subject, using source as name", + extra={"from": source, "email_type": email_type} ) - for rule in extraction_rules: - key = rule.get("key") - regex = rule.get("value") - if key in dict(event): - try: - match = re.search(regex, event[key]) - if match: - for ( - group_name, - group_value, - ) in match.groupdict().items(): - setattr(alert, group_name, group_value) - except Exception as e: - logger.exception( - f"Error extracting key {key} with regex {regex}: {e}", - extra={ - "provider_id": provider_instance.provider_id, - "tenant_id": provider_instance.context_manager.tenant_id, - }, - ) - logger.info( - "Alert extracted", - ) - return alert + + if not message: + if handle_no_body: + message = f"Email from {source} (no body content)" + logger.warning( + "Email has no body content, using fallback message", + extra={"from": source, "subject": name, "email_type": email_type} + ) + else: + MailgunProvider._log_email_processing(event, email_type, "skipped (no body content)") + return None + + # Extract timestamp + try: + timestamp = datetime.datetime.fromtimestamp( + float(event["timestamp"]) + ).isoformat() + except Exception: + timestamp = datetime.datetime.now().isoformat() + + # Default values (same as original) + severity = "info" + status = "firing" + + # Clean redundant fields + event.pop("signature", None) + event.pop("token", None) + + MailgunProvider._log_email_processing(event, email_type, "processing") + + # Create alert + alert = AlertDto( + name=name, + source=[source], + message=message, + description=message, + lastReceived=timestamp, + severity=severity, + status=status, + raw_email={**event}, + ) + + # Add all attributes from raw_email to the alert dto, except the ones that are already set + for key, value in event.items(): + # Avoid "-" in keys cuz CEL will fail [stripped-text screw CEL] + if not hasattr(alert, key) and "-" not in key: + setattr(alert, key, value) + + # Add email type as metadata + setattr(alert, "email_type", email_type) + + logger.info("Alert formatted", extra={"email_type": email_type, "alert_name": name}) + + # Apply extraction rules if configured + if provider_instance: + logger.info("Provider instance found") + extraction_rules = provider_instance.authentication_config.extraction + if extraction_rules: + logger.info("Extraction rules found") + for rule in extraction_rules: + key = rule.get("key") + regex = rule.get("value") + if key in dict(event): + try: + match = re.search(regex, event[key]) + if match: + for ( + group_name, + group_value, + ) in match.groupdict().items(): + setattr(alert, group_name, group_value) + except Exception as e: + logger.exception( + f"Error extracting key {key} with regex {regex}: {e}", + extra={ + "provider_id": provider_instance.provider_id, + "tenant_id": provider_instance.context_manager.tenant_id, + }, + ) + + logger.info("Alert extracted successfully", extra={"email_type": email_type}) + return alert + + except KeyError as e: + logger.error( + f"Missing required field in email event: {e}", + extra={ + "event_keys": list(event.keys()), + "missing_field": str(e) + } + ) + raise + except Exception as e: + logger.error( + f"Error formatting alert from email: {e}", + extra={ + "event_keys": list(event.keys()), + "from": event.get("from"), + "subject": event.get("subject"), + "error": str(e) + } + ) + raise if __name__ == "__main__":