Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,18 +1,19 @@
module github.com/firstset/tenderduty/v2

go 1.18
go 1.21

require (
github.com/PagerDuty/go-pagerduty v1.5.1
github.com/cosmos/btcutil v1.0.4
github.com/cosmos/cosmos-sdk v0.45.11
github.com/go-passwd/validator v0.0.0-20180902184246-0b4c967e436b
github.com/go-telegram-bot-api/telegram-bot-api/v5 v5.5.1
github.com/go-yaml/yaml v2.1.0+incompatible
github.com/gorilla/websocket v1.5.0
github.com/near/borsh-go v0.3.1
github.com/prometheus/client_golang v1.12.2
github.com/tendermint/tendermint v0.34.24
github.com/textileio/go-threads v1.1.5
github.com/near/borsh-go v0.3.1
golang.org/x/crypto v0.1.0
golang.org/x/term v0.1.0
)
Expand All @@ -29,7 +30,6 @@ require (
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/confio/ics23/go v0.7.0 // indirect
github.com/cosmos/btcutil v1.0.4 // indirect
github.com/cosmos/go-bip39 v1.0.0 // indirect
github.com/cosmos/iavl v0.19.4 // indirect
github.com/cosmos/ledger-cosmos-go v0.11.1 // indirect
Expand Down
39 changes: 26 additions & 13 deletions go.sum

Large diffs are not rendered by default.

10 changes: 6 additions & 4 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
_ "embed"
"flag"
"fmt"
"log"
"log/slog"
"os"
"syscall"

Expand Down Expand Up @@ -48,7 +48,8 @@ func main() {
fmt.Print("Please enter the encryption password: ")
pass, err := term.ReadPassword(int(syscall.Stdin))
if err != nil {
log.Fatal(err)
slog.Error("failed to read password", "err", err)
os.Exit(1)
}
fmt.Println("")
password = string(pass)
Expand All @@ -62,13 +63,14 @@ func main() {
e = td2.EncryptedConfig(configFile, encryptedFile, password, true)
}
if e != nil {
log.Fatalln(e)
slog.Error("failed to process encrypted config", "err", e)
os.Exit(1)
}
os.Exit(0)
}

err := td2.Run(configFile, stateFile, chainConfigDirectory, &password, devMode)
if err != nil {
log.Println(err.Error(), "... exiting.")
slog.Error("tenderduty exiting", "err", err)
}
}
67 changes: 30 additions & 37 deletions td2/alert.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"log/slog"
"math"
"net/http"
"slices"
Expand Down Expand Up @@ -184,7 +184,7 @@ func shouldNotify(msg *alertMsg, dest notifyDest) bool {
if strings.HasPrefix(msg.uniqueId, "UnvotedGovernanceProposal") {
// Check if it has been 6 hours since the last (re-)send
if whichMap[msg.uniqueId].SentTime.Before(time.Now().Add(-1 * time.Duration(td.GovernanceAlertsReminderInterval) * time.Hour)) {
l(fmt.Sprintf("🔄 RE-SENDING ALERT on %s (%s) - notifying %s", msg.chain, msg.message, service))
l(slog.LevelInfo, fmt.Sprintf("🔄 RE-SENDING ALERT on %s (%s) - notifying %s", msg.chain, msg.message, service))
cache := alertMsgCache{
Message: msg.message,
SentTime: time.Now(),
Expand All @@ -197,11 +197,11 @@ func shouldNotify(msg *alertMsg, dest notifyDest) bool {
case !whichMap[msg.uniqueId].SentTime.IsZero() && msg.resolved:
// alarm is cleared
delete(whichMap, msg.uniqueId)
l(fmt.Sprintf("💜 Resolved alarm on %s (%s) - notifying %s", msg.chain, msg.message, service))
l(slog.LevelInfo, fmt.Sprintf("💜 Resolved alarm on %s (%s) - notifying %s", msg.chain, msg.message, service))
return true
case msg.resolved:
// it looks like we got a duplicate resolution or suppressed it. Note it and move on:
l(fmt.Sprintf("😕 Not clearing alarm on %s (%s) - no corresponding alert %s", msg.chain, msg.message, service))
l(slog.LevelWarn, fmt.Sprintf("😕 Not clearing alarm on %s (%s) - no corresponding alert %s", msg.chain, msg.message, service))
return false
}

Expand All @@ -212,7 +212,7 @@ func shouldNotify(msg *alertMsg, dest notifyDest) bool {

// for pagerduty we perform some basic flap detection
if dest == pd && msg.pd && alarms.flappingAlarms[msg.chain][msg.uniqueId].SentTime.After(time.Now().Add(-5*time.Minute)) {
l("🛑 flapping detected - suppressing pagerduty notification:", msg.chain, msg.message)
l(slog.LevelWarn, "🛑 flapping detected - suppressing pagerduty notification:", msg.chain, msg.message)
return false
} else if dest == pd && msg.pd {
cache := alertMsgCache{
Expand All @@ -222,7 +222,7 @@ func shouldNotify(msg *alertMsg, dest notifyDest) bool {
alarms.flappingAlarms[msg.chain][msg.uniqueId] = cache
}

l(fmt.Sprintf("🚨 ALERT new alarm on %s (%s) - notifying %s", msg.chain, msg.message, service))
l(slog.LevelInfo, fmt.Sprintf("🚨 ALERT new alarm on %s (%s) - notifying %s", msg.chain, msg.message, service))
cache := alertMsgCache{
Message: msg.message,
SentTime: time.Now(),
Expand Down Expand Up @@ -301,27 +301,27 @@ func notifyDiscord(msg *alertMsg) (err error) {
client := &http.Client{}
data, err := json.MarshalIndent(discPost, "", " ")
if err != nil {
l("⚠️ Could not notify discord!", err)
l(slog.LevelWarn, "⚠️ Could not notify discord!", err)
return err
}

req, err := http.NewRequest("POST", msg.discHook, bytes.NewBuffer(data))
if err != nil {
l("⚠️ Could not notify discord!", err)
l(slog.LevelWarn, "⚠️ Could not notify discord!", err)
return err
}
req.Header.Set("Content-Type", "application/json")

resp, err := client.Do(req)
if err != nil {
l("⚠️ Could not notify discord!", err)
l(slog.LevelWarn, "⚠️ Could not notify discord!", err)
return err
}
_ = resp.Body.Close()

if resp.StatusCode != 204 {
log.Println(resp)
l("⚠️ Could not notify discord! Returned", resp.StatusCode)
slog.Warn("discord webhook returned non-success response", "status", resp.StatusCode)
l(slog.LevelWarn, "⚠️ Could not notify discord! Returned", resp.StatusCode)
return err
}
return nil
Expand Down Expand Up @@ -364,7 +364,7 @@ func notifyTg(msg *alertMsg) (err error) {
}
bot, err := tgbotapi.NewBotAPI(msg.tgKey)
if err != nil {
l("notify telegram:", err)
l(slog.LevelWarn, "notify telegram:", err)
return
}

Expand All @@ -376,7 +376,7 @@ func notifyTg(msg *alertMsg) (err error) {
mc := tgbotapi.NewMessageToChannel(msg.tgChannel, fmt.Sprintf("%s: %s: %s", msg.chain, prefix, msg.message))
_, err = bot.Send(mc)
if err != nil {
l("telegram send:", err)
l(slog.LevelWarn, "telegram send:", err)
}
return err
}
Expand All @@ -390,7 +390,7 @@ func notifyPagerduty(msg *alertMsg) (err error) {
}
// key from the example, don't spam their api
if msg.key == "aaaaaaaaaaaabbbbbbbbbbbbbcccccccccccc" {
l("invalid pagerduty key")
l(slog.LevelWarn, "invalid pagerduty key")
return
}
action := "trigger"
Expand Down Expand Up @@ -485,27 +485,27 @@ func notifyWebhook(msg *alertMsg) (err error) {

data, err := json.Marshal(payload)
if err != nil {
l("⚠️ Could not marshal webhook payload!", err)
l(slog.LevelWarn, "⚠️ Could not marshal webhook payload!", err)
return err
}

req, err := http.NewRequest("POST", msg.whURL, bytes.NewBuffer(data))
if err != nil {
l("⚠️ Could not create webhook request!", err)
l(slog.LevelWarn, "⚠️ Could not create webhook request!", err)
return err
}
req.Header.Set("Content-Type", "application/json")

client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
l("⚠️ Could not send webhook!", err)
l(slog.LevelWarn, "⚠️ Could not send webhook!", err)
return err
}
_ = resp.Body.Close()

if resp.StatusCode < 200 || resp.StatusCode >= 300 {
l("⚠️ Webhook returned non-success status:", resp.StatusCode)
l(slog.LevelWarn, "⚠️ Webhook returned non-success status:", resp.StatusCode)
return fmt.Errorf("webhook returned status %d for %s", resp.StatusCode, msg.chain)
}

Expand Down Expand Up @@ -660,7 +660,7 @@ func evaluateNoRPCEndpointsAlert(cc *ChainConfig, noNodesSec *int) (bool, bool)
*noNodesSec += 2
if *noNodesSec <= 60*td.NodeDownMin {
if *noNodesSec%20 == 0 {
l(fmt.Sprintf("no nodes available on %s for %d seconds, deferring alarm", cc.ChainId, *noNodesSec))
l(slog.LevelInfo, fmt.Sprintf("no nodes available on %s for %d seconds, deferring alarm", cc.ChainId, *noNodesSec))
}
} else {
if !alarms.exist(cc.name, alertID) {
Expand Down Expand Up @@ -1006,12 +1006,14 @@ func evaluateUnclaimedRewardsAlert(cc *ChainConfig) (bool, bool) {
return alert, resolved
}

convertedToDisplay := false
if cc.denomMetadata != nil {
convertedCoins, err := utils.ConvertDecCoinToDisplayUnit(nativeCoins, *cc.denomMetadata)
if err == nil {
nativeCoins = *convertedCoins
convertedToDisplay = true
} else {
l(fmt.Errorf("cannot convert rewards/commission to display unit for %s, err: %w", cc.name, err))
l(slog.LevelDebug, fmt.Errorf("cannot convert rewards/commission to display unit for %s, err: %w", cc.name, err))
}
}

Expand All @@ -1024,26 +1026,17 @@ func evaluateUnclaimedRewardsAlert(cc *ChainConfig) (bool, bool) {
return alert, resolved
}

if err := github_com_cosmos_cosmos_sdk_types.ValidateDenom(targetDenom); err != nil {
fallback := ""
if len(nativeCoins) > 0 {
if errFallback := github_com_cosmos_cosmos_sdk_types.ValidateDenom(nativeCoins[0].Denom); errFallback == nil {
fallback = nativeCoins[0].Denom
}
}
if fallback == "" {
l(fmt.Errorf("invalid target denom %q for %s: %w", targetDenom, cc.name, err))
return alert, resolved
}
l(fmt.Errorf("invalid target denom %q for %s: %w; falling back to %q", targetDenom, cc.name, err, fallback))
targetDenom = fallback
// coinPrice is for the chain display token. If metadata exists but conversion to display
// failed, the amount may still be in base units (e.g. uom), which would misprice by 10^exp.
// Skip this alert evaluation rather than raising a false amount-based alert.
if cc.denomMetadata != nil && !convertedToDisplay {
l(slog.LevelDebug, fmt.Errorf("skipping unclaimed rewards pricing for %s because rewards are not confirmed in display units", cc.name))
return alert, resolved
}

totalRewards := github_com_cosmos_cosmos_sdk_types.NewDecCoinFromDec(targetDenom, totalAmount)

coinPrice, err := td.coinMarketCapClient.GetPrice(td.ctx, cc.Slug)
if err == nil {
totalRewardsConverted := totalRewards.Amount.MustFloat64() * coinPrice.Price
if err == nil && convertedToDisplay {
totalRewardsConverted := totalAmount.MustFloat64() * coinPrice.Price
threshold := floatVal(cc.Alerts.UnclaimedRewardsThreshold)

alertID := fmt.Sprintf("UnclaimedRewards_%s", cc.ValAddress)
Expand Down
9 changes: 6 additions & 3 deletions td2/dashboard/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ import (
"embed"
"encoding/json"
"io/fs"
"log"
"log/slog"
"net/http"
"os"
"regexp"
"sort"
"sync"
Expand All @@ -27,7 +28,8 @@ func Serve(port string, updates chan *ChainStatus, logs chan LogMessage, hideLog
var err error
rootDir, err = fs.Sub(Content, "static")
if err != nil {
log.Fatalln(err)
slog.Error("failed to load embedded static content", "err", err)
os.Exit(1)
}
var cast broadcast.Broadcaster

Expand Down Expand Up @@ -150,7 +152,8 @@ func Serve(port string, updates chan *ChainStatus, logs chan LogMessage, hideLog
}
err = server.ListenAndServe()
cast.Discard()
log.Fatal("tenderduty dashboard server failed", err)
slog.Error("tenderduty dashboard server failed", "err", err)
os.Exit(1)
}

// CacheHandler implements the Handler interface with a Cache-Control set on responses
Expand Down
4 changes: 2 additions & 2 deletions td2/encryption.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"errors"
"fmt"
"io"
"log"
"log/slog"
"os"

"github.com/go-passwd/validator"
Expand Down Expand Up @@ -227,6 +227,6 @@ func EncryptedConfig(plaintext, ciphertext, pass string, decrypting bool) error
if decrypting {
fileType = "decrypted"
}
log.Printf("wrote %d bytes to %s file %s\n", size, fileType, outfile)
slog.Info("wrote file", "bytes", size, "type", fileType, "file", outfile)
return nil
}
Loading
Loading