From 4d92b567d0c8c16b0a2d5b5a24c0e9862a0caa2f Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sat, 11 Oct 2025 18:12:18 +0300 Subject: [PATCH 1/7] PMM-14305 Fix ClickHouse dirty state --- qan-api2/db.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/qan-api2/db.go b/qan-api2/db.go index 022f7982ada..55888ab76ea 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -113,9 +113,31 @@ func runMigrations(dsn string) error { // run up to the latest migration err = m.Up() - if errors.Is(err, migrate.ErrNoChange) { + if err == nil || errors.Is(err, migrate.ErrNoChange) { return nil } + + // If the database is in dirty state, try to fix it (PMM-14305) + var errDirty migrate.ErrDirty + if errors.As(err, &errDirty) { + log.Printf("Migration %d was unsuccesful, trying to fix it...", errDirty.Version) + + // Note: -1 is a valid version to force to + ver := int(errDirty.Version) - 1 + fixErr := m.Force(ver) + if fixErr != nil { + return fmt.Errorf("can't force the migration %d: %w", ver, fixErr) + } + + // try to run migrations again + err = m.Up() + if err == nil || errors.Is(err, migrate.ErrNoChange) { + return nil + } + + return err + } + return err } From 57f94e307ebff335ec225b9999fbe26cb276236e Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sat, 11 Oct 2025 18:35:15 +0300 Subject: [PATCH 2/7] PMM-14305 Refactor err handling --- qan-api2/db.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qan-api2/db.go b/qan-api2/db.go index 55888ab76ea..118cbe1c9f3 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -124,14 +124,14 @@ func runMigrations(dsn string) error { // Note: -1 is a valid version to force to ver := int(errDirty.Version) - 1 - fixErr := m.Force(ver) - if fixErr != nil { - return fmt.Errorf("can't force the migration %d: %w", ver, fixErr) + fErr := m.Force(ver) + if fErr != nil { + return fmt.Errorf("can't force the migration %d: %w", ver, fErr) } // try to run migrations again err = m.Up() - if err == nil || errors.Is(err, migrate.ErrNoChange) { + if errors.Is(err, migrate.ErrNoChange) { return nil } From acc18de20b87d76f99b8326bef59017a9e6b8c2e Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sat, 11 Oct 2025 18:36:31 +0300 Subject: [PATCH 3/7] PMM-14305 Fix linter errors --- qan-api2/db.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/qan-api2/db.go b/qan-api2/db.go index 118cbe1c9f3..fa7a404c8e4 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -120,10 +120,10 @@ func runMigrations(dsn string) error { // If the database is in dirty state, try to fix it (PMM-14305) var errDirty migrate.ErrDirty if errors.As(err, &errDirty) { - log.Printf("Migration %d was unsuccesful, trying to fix it...", errDirty.Version) + log.Printf("Migration %d was unsuccessful, trying to fix it...", errDirty.Version) // Note: -1 is a valid version to force to - ver := int(errDirty.Version) - 1 + ver := errDirty.Version - 1 fErr := m.Force(ver) if fErr != nil { return fmt.Errorf("can't force the migration %d: %w", ver, fErr) @@ -134,8 +134,6 @@ func runMigrations(dsn string) error { if errors.Is(err, migrate.ErrNoChange) { return nil } - - return err } return err From 4b7ee6a2eb8f537d16a657cdf5ac9d3d13cd80c1 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sat, 11 Oct 2025 22:25:30 +0300 Subject: [PATCH 4/7] PMM-14305 Migrate to stdlib errors pkg --- qan-api2/db.go | 2 +- qan-api2/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qan-api2/db.go b/qan-api2/db.go index fa7a404c8e4..3b00c22d253 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -17,6 +17,7 @@ package main import ( "embed" + "errors" "fmt" "log" "net/url" @@ -28,7 +29,6 @@ import ( "github.com/golang-migrate/migrate/v4/source/iofs" "github.com/jmoiron/sqlx" // TODO: research alternatives. Ex.: https://github.com/go-reform/reform "github.com/jmoiron/sqlx/reflectx" - "github.com/pkg/errors" ) const ( diff --git a/qan-api2/main.go b/qan-api2/main.go index bd504387677..c258729c3e9 100644 --- a/qan-api2/main.go +++ b/qan-api2/main.go @@ -18,6 +18,7 @@ package main import ( "bytes" "context" + "errors" _ "expvar" // register /debug/vars "fmt" "html/template" @@ -38,7 +39,6 @@ import ( grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" grpc_gateway "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" "github.com/jmoiron/sqlx" - "github.com/pkg/errors" prom "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" From 4aa0c04d0dfa7a8b89bebc09330a827319582f0e Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Sun, 12 Oct 2025 12:39:13 +0300 Subject: [PATCH 5/7] PMM-14305 Remove a redundant method --- qan-api2/utils/logger/logger.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/qan-api2/utils/logger/logger.go b/qan-api2/utils/logger/logger.go index f9d2b2b5ce4..90aafd631d0 100644 --- a/qan-api2/utils/logger/logger.go +++ b/qan-api2/utils/logger/logger.go @@ -28,7 +28,7 @@ type keyStruct struct{} var key = keyStruct{} -// Get returns logrus entry for given context. Set must be called before this method is called. +// Get returns logrus entry for given context. SetEntry must be called before this method is called. func Get(ctx context.Context) *logrus.Entry { v := ctx.Value(key) if v == nil { @@ -37,11 +37,6 @@ func Get(ctx context.Context) *logrus.Entry { return v.(*logrus.Entry) //nolint:forcetypeassert } -// Set returns derived context with set logrus entry with given request ID. -func Set(ctx context.Context, requestID string) context.Context { - return SetEntry(ctx, logrus.WithField("request", requestID)) -} - // SetEntry returns derived context with set given logrus entry. func SetEntry(ctx context.Context, l *logrus.Entry) context.Context { if ctx.Value(key) != nil { From a51192803fcbbe18bef7a2271952627530b51aff Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Fri, 17 Oct 2025 23:51:46 +0300 Subject: [PATCH 6/7] PMM-14305 Fix an adge case when the first migration fails --- qan-api2/db.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/qan-api2/db.go b/qan-api2/db.go index 3b00c22d253..1dd76feb612 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -122,8 +122,11 @@ func runMigrations(dsn string) error { if errors.As(err, &errDirty) { log.Printf("Migration %d was unsuccessful, trying to fix it...", errDirty.Version) - // Note: -1 is a valid version to force to ver := errDirty.Version - 1 + if ver == 0 { + // Note: since 0th migration does not exist, we set it to -1, which means "start from scratch" + ver = -1 + } fErr := m.Force(ver) if fErr != nil { return fmt.Errorf("can't force the migration %d: %w", ver, fErr) From da8cc9e5b7bd38fea930306e84511a595cea3009 Mon Sep 17 00:00:00 2001 From: Alex Demidoff Date: Tue, 21 Oct 2025 23:35:32 +0300 Subject: [PATCH 7/7] PMM-14305 Minor update --- qan-api2/db.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qan-api2/db.go b/qan-api2/db.go index 1dd76feb612..31797ee321b 100644 --- a/qan-api2/db.go +++ b/qan-api2/db.go @@ -127,12 +127,12 @@ func runMigrations(dsn string) error { // Note: since 0th migration does not exist, we set it to -1, which means "start from scratch" ver = -1 } - fErr := m.Force(ver) - if fErr != nil { - return fmt.Errorf("can't force the migration %d: %w", ver, fErr) + err = m.Force(ver) + if err != nil { + return fmt.Errorf("can't force the migration %d: %w", ver, err) } - // try to run migrations again + // try to run migrations again, starting from the forced version err = m.Up() if errors.Is(err, migrate.ErrNoChange) { return nil