Add prometheus metrics for monitoring

cmd/channel_account.go

@@ -59,7 +59,7 @@ func (c *channelAccountCmd) Command() *cobra.Command {
 				return fmt.Errorf("setting values of config options: %w", err)
 			}
 
-			dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL)
+			dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL, nil)
 			if err != nil {
 				return fmt.Errorf("opening connection pool: %w", err)
 			}

cmd/distribution_account.go

@@ -62,7 +62,7 @@ func kmsCommand() *cobra.Command {
 				return fmt.Errorf("setting values of config options: %w", err)
 			}
 
-			dbConnectionPool, err := db.OpenDBConnectionPool(cfg.databaseURL)
+			dbConnectionPool, err := db.OpenDBConnectionPool(cfg.databaseURL, nil)
 			if err != nil {
 				return fmt.Errorf("opening connection pool: %w", err)
 			}

cmd/serve.go

@@ -113,7 +113,7 @@ func (c *serveCmd) Command() *cobra.Command {
 				return fmt.Errorf("setting values of config options: %w", err)
 			}
 
-			dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL)
+			dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL, nil)
 			if err != nil {
 				return fmt.Errorf("opening connection pool: %w", err)
 			}

go.mod

@@ -5,12 +5,14 @@ go 1.23.2
 require (
 	github.com/alitto/pond v1.9.2
 	github.com/aws/aws-sdk-go v1.55.5
+	github.com/dlmiddlecote/sqlstats v1.0.2
 	github.com/getsentry/sentry-go v0.29.1
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-playground/validator/v10 v10.22.1
 	github.com/google/uuid v1.6.0
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/lib/pq v1.10.9
+	github.com/prometheus/client_golang v1.17.0
 	github.com/rubenv/sql-migrate v1.7.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.1
@@ -49,7 +51,6 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_golang v1.17.0 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect

internal/data/accounts_test.go

@@ -16,7 +16,7 @@ func TestAccountModelInsert(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -41,7 +41,7 @@ func TestAccountModelDelete(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -69,7 +69,7 @@ func TestAccountModelIsAccountFeeBumpEligible(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 

internal/data/payments_test.go

@@ -18,7 +18,7 @@ func TestPaymentModelAddPayment(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -142,7 +142,7 @@ func TestPaymentModelAddPayment(t *testing.T) {
 func TestPaymentModelGetLatestLedgerSynced(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -167,7 +167,7 @@ func TestPaymentModelGetLatestLedgerSynced(t *testing.T) {
 func TestPaymentModelUpdateLatestLedgerSynced(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -189,7 +189,7 @@ func TestPaymentModelUpdateLatestLedgerSynced(t *testing.T) {
 func TestPaymentModelGetPaymentsPaginated(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 

internal/db/db.go

@@ -7,6 +7,8 @@ import (
 	"time"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/dlmiddlecote/sqlstats"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stellar/go/support/log"
 )
 
@@ -31,14 +33,19 @@ const (
 	MaxOpenDBConns    = 30
 )
 
-func OpenDBConnectionPool(dataSourceName string) (ConnectionPool, error) {
+func OpenDBConnectionPool(dataSourceName string, metricsRegistry *prometheus.Registry) (ConnectionPool, error) {
 	sqlxDB, err := sqlx.Open("postgres", dataSourceName)
 	if err != nil {
 		return nil, fmt.Errorf("error creating app DB connection pool: %w", err)
 	}
 	sqlxDB.SetConnMaxIdleTime(MaxDBConnIdleTime)
 	sqlxDB.SetMaxOpenConns(MaxOpenDBConns)
 
+	if metricsRegistry != nil {
+		collector := sqlstats.NewStatsCollector("wallet-backend-db", sqlxDB)
+		metricsRegistry.MustRegister(collector)
+	}
+
 	err = sqlxDB.Ping()
 	if err != nil {
 		return nil, fmt.Errorf("error pinging app DB connection pool: %w", err)

internal/db/db_test.go

@@ -13,7 +13,7 @@ func TestOpenDBConnectionPool(t *testing.T) {
 	dbt := dbtest.Postgres(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 

internal/db/migrate.go

@@ -10,7 +10,7 @@ import (
 )
 
 func Migrate(ctx context.Context, databaseURL string, direction migrate.MigrationDirection, count int) (int, error) {
-	dbConnectionPool, err := OpenDBConnectionPool(databaseURL)
+	dbConnectionPool, err := OpenDBConnectionPool(databaseURL, nil)
 	if err != nil {
 		return 0, fmt.Errorf("connecting to the database: %w", err)
 	}

internal/db/migrate_test.go

@@ -16,7 +16,7 @@ func TestMigrate_up_1(t *testing.T) {
 	dbt := dbtest.OpenWithoutMigrations(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -37,7 +37,7 @@ func TestMigrate_up_2_down_1(t *testing.T) {
 	dbt := dbtest.OpenWithoutMigrations(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -72,7 +72,7 @@ func TestMigrate_upall_down_all(t *testing.T) {
 	db := dbtest.OpenWithoutMigrations(t)
 	defer db.Close()
 
-	dbConnectionPool, err := OpenDBConnectionPool(db.DSN)
+	dbConnectionPool, err := OpenDBConnectionPool(db.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 

internal/ingest/ingest.go

@@ -6,6 +6,8 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"github.com/stellar/go/support/log"
 
@@ -49,8 +51,10 @@ func Ingest(cfg Configs) error {
 }
 
 func setupDeps(cfg Configs) (services.IngestService, error) {
+	ingestMetricsRegistry := prometheus.NewRegistry()
+
 	// Open DB connection pool
-	dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL)
+	dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL, ingestMetricsRegistry)
 	if err != nil {
 		return nil, fmt.Errorf("connecting to the database: %w", err)
 	}
@@ -75,10 +79,18 @@ func setupDeps(cfg Configs) (services.IngestService, error) {
 	router := tssrouter.NewRouter(tssRouterConfig)
 
 	ingestService, err := services.NewIngestService(
-		models, cfg.LedgerCursorName, cfg.AppTracker, rpcService, router, tssStore)
+		models, cfg.LedgerCursorName, cfg.AppTracker, rpcService, router, tssStore, ingestMetricsRegistry)
 	if err != nil {
 		return nil, fmt.Errorf("instantiating ingest service: %w", err)
 	}
 
+	http.Handle("/ingest-metrics", promhttp.HandlerFor(ingestMetricsRegistry, promhttp.HandlerOpts{}))
+	go func() {
+		err := http.ListenAndServe(":8002", nil)
+		if err != nil {
+			log.Ctx(context.Background()).Fatalf("starting ingest metrics server: %v", err)
+		}
+	}()
+
 	return ingestService, nil
 }

internal/serve/httphandler/account_handler_test.go

@@ -31,7 +31,7 @@ func TestAccountHandlerRegisterAccount(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -130,7 +130,7 @@ func TestAccountHandlerDeregisterAccount(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 

internal/serve/httphandler/payment_handler_test.go

@@ -23,7 +23,7 @@ func TestPaymentHandlerGetPayments(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 

internal/serve/httphandler/tss_handler_test.go

@@ -34,7 +34,7 @@ func TestBuildTransactions(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 	store, _ := store.NewStore(dbConnectionPool)
@@ -132,7 +132,7 @@ func TestSubmitTransactions(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 	store, _ := store.NewStore(dbConnectionPool)
@@ -227,7 +227,7 @@ func TestGetTransaction(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 	store, _ := store.NewStore(dbConnectionPool)

internal/serve/serve.go

@@ -7,6 +7,8 @@ import (
 	"time"
 
 	"github.com/go-chi/chi"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	supporthttp "github.com/stellar/go/support/http"
 	"github.com/stellar/go/support/log"
@@ -106,6 +108,8 @@ type handlerDeps struct {
 	TSSTransactionService tssservices.TransactionService
 	// Error Tracker
 	AppTracker apptracker.AppTracker
+	// Prometheus Metrics
+	MetricsRegistry *prometheus.Registry
 }
 
 func Serve(cfg Configs) error {
@@ -135,7 +139,8 @@ func Serve(cfg Configs) error {
 }
 
 func initHandlerDeps(cfg Configs) (handlerDeps, error) {
-	dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL)
+	metricsRegistry := prometheus.NewRegistry()
+	dbConnectionPool, err := db.OpenDBConnectionPool(cfg.DatabaseURL, metricsRegistry)
 	if err != nil {
 		return handlerDeps{}, fmt.Errorf("connecting to the database: %w", err)
 	}
@@ -273,6 +278,7 @@ func initHandlerDeps(cfg Configs) (handlerDeps, error) {
 		PaymentService:            paymentService,
 		AppTracker:                cfg.AppTracker,
 		NetworkPassphrase:         cfg.NetworkPassphrase,
+		MetricsRegistry:           metricsRegistry,
 		// TSS
 		RPCCallerChannel:      rpcCallerChannel,
 		ErrorJitterChannel:    errorJitterChannel,
@@ -313,6 +319,7 @@ func handler(deps handlerDeps) http.Handler {
 	mux.Use(middleware.RecoverHandler(deps.AppTracker))
 
 	mux.Get("/health", health.PassHandler{}.ServeHTTP)
+	mux.Get("/api-metrics", promhttp.HandlerFor(deps.MetricsRegistry, promhttp.HandlerOpts{}).ServeHTTP)
 
 	// Authenticated routes
 	mux.Group(func(r chi.Router) {

internal/services/account_service_test.go

@@ -17,7 +17,7 @@ func TestAccountRegister(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -43,7 +43,7 @@ func TestAccountDeregister(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 

internal/services/account_sponsorship_service_test.go

@@ -22,7 +22,7 @@ import (
 func TestAccountSponsorshipServiceSponsorAccountCreationTransaction(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -299,7 +299,7 @@ func TestAccountSponsorshipServiceSponsorAccountCreationTransaction(t *testing.T
 func TestAccountSponsorshipServiceWrapTransaction(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 

internal/services/channel_account_service_test.go

@@ -24,7 +24,7 @@ func TestChannelAccountServiceEnsureChannelAccounts(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -293,7 +293,7 @@ func TestSubmitTransaction(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()
 
@@ -348,7 +348,7 @@ func TestWaitForTransactionConfirmation(t *testing.T) {
 	dbt := dbtest.Open(t)
 	defer dbt.Close()
 
-	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN)
+	dbConnectionPool, err := db.OpenDBConnectionPool(dbt.DSN, nil)
 	require.NoError(t, err)
 	defer dbConnectionPool.Close()