Skip to content

Add DisableResolveMessage option for webhook alerts #66

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gluckzhang merged 1 commit into from
Jan 20, 2026
Merged

Add DisableResolveMessage option for webhook alerts #66

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions example-config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@ default_alert_config:
url: https://your-webhook-endpoint.example.com/alerts
# Severity threshold defines the minimum severity level at which the alerts are sent to this channel
severity_threshold: info
# When set to yes, resolved alert messages will not be sent to the webhook
# This is useful when you only want to be notified of firing alerts
disable_resolve_message: no

# Alert defaults shared by all chains
# If the chain stops seeing new blocks, should an alert be sent?
Expand Down
5 changes: 5 additions & 0 deletions td2/alert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,11 @@ func notifyWebhook(msg *alertMsg) (err error) {
return nil
}

// Skip sending resolved messages if disabled in config
if msg.resolved && boolVal(msg.alertConfig.Webhook.DisableResolveMessage) {
return nil
}

status := "firing"
if msg.resolved {
status = "resolved"
Expand Down
142 changes: 133 additions & 9 deletions td2/alert_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,10 +462,12 @@ func TestNotifySlack(t *testing.T) {

func TestNotifyWebhook(t *testing.T) {
tests := []struct {
name string
msg *alertMsg
serverResponse int
expectError bool
name string
msg *alertMsg
serverResponse int
expectError bool
expectServerCall bool
setupAlarms func(alarms *alarmCache) // optional setup for existing alarms
}{
{
name: "successful notification",
Expand All @@ -481,8 +483,9 @@ func TestNotifyWebhook(t *testing.T) {
Webhook: WebhookConfig{SeverityThreshold: "info"},
},
},
serverResponse: 200,
expectError: false,
serverResponse: 200,
expectError: false,
expectServerCall: true,
},
{
name: "server error",
Expand All @@ -498,15 +501,119 @@ func TestNotifyWebhook(t *testing.T) {
Webhook: WebhookConfig{SeverityThreshold: "info"},
},
},
serverResponse: 500,
expectError: true,
serverResponse: 500,
expectError: true,
expectServerCall: true,
},
{
name: "webhook disabled",
msg: &alertMsg{
wh: false,
},
expectError: false,
expectError: false,
expectServerCall: false,
},
{
name: "resolved message skipped when DisableResolveMessage is true",
msg: &alertMsg{
wh: true,
chain: "test-chain",
message: "test resolved message",
severity: "critical",
uniqueId: "test_alert_resolved_skip",
resolved: true,
whURL: "", // will be set to test server URL
alertConfig: &AlertConfig{
Webhook: WebhookConfig{
SeverityThreshold: "info",
DisableResolveMessage: &[]bool{true}[0],
},
},
},
serverResponse: 200,
expectError: false,
expectServerCall: false,
// Set up existing alert so the resolved message would normally be sent
setupAlarms: func(a *alarmCache) {
a.SentWHAlarms["test_alert_resolved_skip"] = alertMsgCache{
Message: "Previous alert",
SentTime: time.Now().Add(-1 * time.Hour),
}
},
},
{
name: "resolved message sent when DisableResolveMessage is false",
msg: &alertMsg{
wh: true,
chain: "test-chain",
message: "test resolved message",
severity: "critical",
uniqueId: "test_alert_resolved_send",
resolved: true,
whURL: "", // will be set to test server URL
alertConfig: &AlertConfig{
Webhook: WebhookConfig{
SeverityThreshold: "info",
DisableResolveMessage: &[]bool{false}[0],
},
},
},
serverResponse: 200,
expectError: false,
expectServerCall: true,
// Set up existing alert so the resolved message can be sent
setupAlarms: func(a *alarmCache) {
a.SentWHAlarms["test_alert_resolved_send"] = alertMsgCache{
Message: "Previous alert",
SentTime: time.Now().Add(-1 * time.Hour),
}
},
},
{
name: "resolved message sent when DisableResolveMessage is nil (default)",
msg: &alertMsg{
wh: true,
chain: "test-chain",
message: "test resolved message",
severity: "critical",
uniqueId: "test_alert_resolved_nil",
resolved: true,
whURL: "", // will be set to test server URL
alertConfig: &AlertConfig{
Webhook: WebhookConfig{SeverityThreshold: "info"},
},
},
serverResponse: 200,
expectError: false,
expectServerCall: true,
// Set up existing alert so the resolved message can be sent
setupAlarms: func(a *alarmCache) {
a.SentWHAlarms["test_alert_resolved_nil"] = alertMsgCache{
Message: "Previous alert",
SentTime: time.Now().Add(-1 * time.Hour),
}
},
},
{
name: "firing message sent even when DisableResolveMessage is true",
msg: &alertMsg{
wh: true,
chain: "test-chain",
message: "test firing message",
severity: "critical",
uniqueId: "test_alert_firing",
resolved: false,
whURL: "", // will be set to test server URL
alertConfig: &AlertConfig{
Webhook: WebhookConfig{
SeverityThreshold: "info",
DisableResolveMessage: &[]bool{true}[0],
},
},
},
serverResponse: 200,
expectError: false,
expectServerCall: true,
},
}

Expand All @@ -530,9 +637,18 @@ func TestNotifyWebhook(t *testing.T) {
// Reset alarms for each test
testAlarms.SentWHAlarms = make(map[string]alertMsgCache)

// Run optional alarm setup (e.g., to set up existing alerts for resolved message tests)
if tt.setupAlarms != nil {
tt.setupAlarms(testAlarms)
}

// Track whether server was called
serverCalled := false

if tt.msg.wh {
// Create test server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
serverCalled = true
w.WriteHeader(tt.serverResponse)
}))
defer server.Close()
Expand All @@ -546,6 +662,14 @@ func TestNotifyWebhook(t *testing.T) {
if !tt.expectError && err != nil {
t.Errorf("Expected no error but got: %v", err)
}

// Verify server call expectation
if tt.expectServerCall && !serverCalled {
t.Errorf("Expected server to be called but it was not")
}
if !tt.expectServerCall && serverCalled {
t.Errorf("Expected server NOT to be called but it was")
}
})
}
}
Expand Down
7 changes: 4 additions & 3 deletions td2/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,9 +359,10 @@ type SlackConfig struct {
// WebhookConfig holds the information needed to send alerts to a generic webhook endpoint
// The payload follows a Grafana-like format for broad compatibility
type WebhookConfig struct {
Enabled *bool `yaml:"enabled"`
URL string `yaml:"url"`
SeverityThreshold string `yaml:"severity_threshold"`
Enabled *bool `yaml:"enabled"`
URL string `yaml:"url"`
SeverityThreshold string `yaml:"severity_threshold"`
DisableResolveMessage *bool `yaml:"disable_resolve_message"`
}

// HealthcheckConfig holds the information needed to send pings to a healthcheck endpoint
Expand Down
Loading