Git-on-my-level · Nov 6, 2026 · Nov 5, 2026 · Nov 5, 2026 · Nov 5, 2026 · Nov 5, 2026
diff --git a/backend/charts/backend-listen/dev_omi_backend_listen_values.yaml b/backend/charts/backend-listen/dev_omi_backend_listen_values.yaml
@@ -138,6 +138,8 @@ env:
     value: "10"
   - name: SERVER_BATCH_UNSUPPORTED_LANGUAGES
     value: ""
+  - name: BATCH_HELD_DEAD_LETTER_ENABLED
+    value: "true"
   - name: BATCH_PIPELINE_ENABLED
     value: "false"
   - name: BATCH_PIPELINE_UID_ALLOWLIST

diff --git a/backend/charts/backend-listen/prod_omi_backend_listen_values.yaml b/backend/charts/backend-listen/prod_omi_backend_listen_values.yaml
@@ -137,6 +137,8 @@ env:
     value: "af,ar,az,be,bn,bs,ca,cy,eu,fa,gl,gu,he,hi,id,kk,kn,ko,mk,ml,mr,ms,no,pa,sq,sr,sw,ta,te,th,tl,tr,ur,vi,zh"
   - name: SERVER_BATCH_ONLY_AUTO_LANGUAGE_PERCENT
     value: "100"
+  - name: BATCH_HELD_DEAD_LETTER_ENABLED
+    value: "true"
   - name: BATCH_PIPELINE_ENABLED
     value: "true"
   - name: FIRST_SUMMARY_DEADLINE_ENABLED

diff --git a/backend/charts/batch-pipeline-worker/dev_omi_batch-pipeline-worker_values.yaml b/backend/charts/batch-pipeline-worker/dev_omi_batch-pipeline-worker_values.yaml
@@ -26,6 +26,7 @@ tolerations:
     value: present
     effect: NoSchedule
 worker:
+  heldDeadLetterEnabled: true
   servingMetricsMinRefreshSeconds: 30
   shadowRecoveryApiUrl: "https://backend-integration-dt5lrfkkoa-uc.a.run.app"
   headPriorityEnabled: false

diff --git a/backend/charts/batch-pipeline-worker/prod_omi_batch-pipeline-worker_values.yaml b/backend/charts/batch-pipeline-worker/prod_omi_batch-pipeline-worker_values.yaml
@@ -55,6 +55,7 @@ borrowedNodePools:
   enabled: false
   pools: [parakeet-pool, parakeet-pool-b, parakeet-pool-v2]
 worker:
+  heldDeadLetterEnabled: true
   # Two pump processes share each L4 (cl2-gpueff 2026-11-04, data/run-2026-11-04-gpueff
   # in omi-workspace): prod is claim-path bound (GPU util 16%, ~1.6 units per
   # 3.2s pull), so a second independent pump doubles concurrent claims per GPU.

diff --git a/backend/charts/batch-pipeline-worker/staging_omi_batch-pipeline-worker_values.yaml b/backend/charts/batch-pipeline-worker/staging_omi_batch-pipeline-worker_values.yaml
@@ -31,6 +31,7 @@ tolerations:
     value: present
     effect: NoSchedule
 worker:
+  heldDeadLetterEnabled: true
   apiUrl: https://backend-sync-staging-1031333818730.us-central1.run.app
   audience: https://backend-sync-staging-1031333818730.us-central1.run.app
   # Staging composes the prod manifest, so it inherits the shadow 25 pin; the

diff --git a/backend/charts/batch-pipeline-worker/templates/deployment.yaml b/backend/charts/batch-pipeline-worker/templates/deployment.yaml
@@ -128,6 +128,7 @@ spec:
             - {name: BATCH_PIPELINE_WORKER_VALIDATION_DISPOSITION_ENABLED, value: {{ .Values.worker.workerValidationDispositionEnabled | quote }}}
             - {name: BATCH_PIPELINE_DIARIZE_SPEECH_ONLY_ENABLED, value: {{ .Values.worker.diarizeSpeechOnlyEnabled | quote }}}
             - {name: BATCH_PIPELINE_SHORT_SPEECH_SINGLE_ENABLED, value: {{ .Values.worker.shortSpeechSingleEnabled | quote }}}
+            - {name: BATCH_HELD_DEAD_LETTER_ENABLED, value: {{ .Values.worker.heldDeadLetterEnabled | quote }}}
             - {name: BATCH_PIPELINE_EMPTY_ASR_DISPOSITION_ENABLED, value: {{ .Values.worker.emptyAsrDispositionEnabled | quote }}}
             - {name: BATCH_PIPELINE_EMPTY_ASR_SAFETY_NET_ENABLED, value: {{ .Values.worker.emptyAsrSafetyNetEnabled | quote }}}
             - {name: BATCH_PIPELINE_EMPTY_ASR_RECOVERY_ENABLED, value: {{ .Values.worker.emptyAsrRecoveryEnabled | quote }}}

diff --git a/backend/charts/batch-pipeline-worker/values.yaml b/backend/charts/batch-pipeline-worker/values.yaml
@@ -49,6 +49,7 @@ borrowedNodePools:
   # autoscaler's expendable cutoff (-10), so a Pending borrower is not ignored.
   priority: -1
 worker:
+  heldDeadLetterEnabled: true
   # Match the serving hosts; prod overrides this below the 120s guard blind window.
   servingMetricsMinRefreshSeconds: 30
   # Control-loop defaults; rollback independently with 64 / false / 2 / {}.

diff --git a/backend/charts/monitoring/alert-rules.json b/backend/charts/monitoring/alert-rules.json
@@ -28676,5 +28676,178 @@
     },
     "record": null,
     "missingSeriesEvalsToResolve": 2
+  },
+  {
+    "uid": "omi-batch-held-expired-volume",
+    "orgID": 1,
+    "folderUID": "betdycdziadc0e",
+    "ruleGroup": "Batch held dead letters",
+    "title": "Retained held batch dead-letter volume",
+    "condition": "E",
+    "data": [
+      {
+        "refId": "A",
+        "queryType": "",
+        "relativeTimeRange": {
+          "from": 900,
+          "to": 0
+        },
+        "datasourceUid": "prometheus",
+        "model": {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "expr": "max(omi_batch_held_expired_units and (time() - omi_batch_held_expired_observed_at_seconds < 900)) or vector(0)",
+          "instant": true,
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "refId": "A"
+        }
+      },
+      {
+        "refId": "B",
+        "queryType": "",
+        "relativeTimeRange": {
+          "from": 900,
+          "to": 0
+        },
+        "datasourceUid": "prometheus",
+        "model": {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "expr": "sum(increase(omi_batch_held_expired_total[30m])) or vector(0)",
+          "instant": true,
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "refId": "B"
+        }
+      },
+      {
+        "refId": "C",
+        "queryType": "",
+        "relativeTimeRange": {
+          "from": 900,
+          "to": 0
+        },
+        "datasourceUid": "prometheus",
+        "model": {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "expr": "(time() - max(omi_batch_held_expired_observed_at_seconds)) or vector(1e9)",
+          "instant": true,
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "refId": "C"
+        }
+      },
+      {
+        "refId": "D",
+        "queryType": "",
+        "relativeTimeRange": {
+          "from": 0,
+          "to": 0
+        },
+        "datasourceUid": "__expr__",
+        "model": {
+          "datasource": {
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "expression": "($A > 100 && $C < 900) || $B > 10",
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "refId": "D",
+          "type": "math"
+        }
+      },
+      {
+        "refId": "E",
+        "queryType": "",
+        "relativeTimeRange": {
+          "from": 0,
+          "to": 0
+        },
+        "datasourceUid": "__expr__",
+        "model": {
+          "conditions": [
+            {
+              "evaluator": {
+                "params": [],
+                "type": "gt"
+              },
+              "operator": {
+                "type": "and"
+              },
+              "query": {
+                "params": [
+                  "E"
+                ]
+              },
+              "reducer": {
+                "params": [],
+                "type": "last"
+              },
+              "type": "query"
+            }
+          ],
+          "datasource": {
+            "type": "__expr__",
+            "uid": "__expr__"
+          },
+          "expression": "D",
+          "intervalMs": 1000,
+          "maxDataPoints": 43200,
+          "reducer": "last",
+          "refId": "E",
+          "type": "reduce"
+        }
+      }
+    ],
+    "updated": "2026-11-06T00:00:00Z",
+    "noDataState": "OK",
+    "execErrState": "KeepLast",
+    "for": "5m",
+    "keep_firing_for": "0s",
+    "annotations": {
+      "dashboard_url": "https://monitor.omi.me/d/omi-core-features",
+      "summary": "Held batch dead letters exceed the retained-stock or new-expiry volume threshold.",
+      "user_impact": "Recorded audio remains retained but these units have no published transcript.",
+      "scope": "Stock is only state=dead with reason held_expired, a new explicit cohort; legacy unrelated dead rows never count. Rate counts unique CAS expirations over 30 minutes. Stock requires a sample younger than 15 minutes. Agent tier only. Each replica stock sample is filtered by its own timestamp before max aggregation, so another replica cannot refresh stale stock.",
+      "verification": "Read omi_batch_held_expired_units and omi_batch_held_expired_total, then inspect reasoned rows using an authorized operator read.",
+      "safe_next_action": "Check nightly re-drive and worker failure reasons. Use the private OIDC requeue-quarantined path after resolving the cause. Never delete audio or clear privacy fences.",
+      "runbook": "backend/docs/runbooks/batch-held-dead-letter.md",
+      "threshold": "Fresh retained stock >100 or >10 expirations in 30m for 5m; initial agent thresholds, not calibrated from production.",
+      "evaluated_bad": "Synthetic: stock 101 sampled 300s ago, or 11 new expirations in 30m.",
+      "evaluated_good": "Synthetic: stock 100 and 10 new expirations, or stale stock without a rate breach."
+    },
+    "labels": {
+      "alert_identity": "omi-batch-held-expired-volume",
+      "component": "batch-pipeline",
+      "impact": "product",
+      "severity": "warning",
+      "instatus_component": "speech-processing",
+      "tier": "agent",
+      "audience": "agent",
+      "owner": "uicleanup"
+    },
+    "isPaused": false,
+    "notification_settings": {
+      "receiver": "Omi - Services Alerting (Telegram)",
+      "group_by": [
+        "alertname"
+      ],
+      "group_wait": "5m",
+      "group_interval": "1h",
+      "repeat_interval": "24h",
+      "mute_time_intervals": [
+        "omi-tier-silent"
+      ]
+    },
+    "record": null
   }
 ]