PMM-14305 Fix ClickHouse dirty migration state

qan-api2/db.go

@@ -17,6 +17,7 @@ package main
 
 import (
 	"embed"
+	"errors"
 	"fmt"
 	"log"
 	"net/url"
@@ -28,7 +29,6 @@ import (
 	"github.com/golang-migrate/migrate/v4/source/iofs"
 	"github.com/jmoiron/sqlx" // TODO: research alternatives. Ex.: https://github.com/go-reform/reform
 	"github.com/jmoiron/sqlx/reflectx"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -113,9 +113,29 @@ func runMigrations(dsn string) error {
 
 	// run up to the latest migration
 	err = m.Up()
-	if errors.Is(err, migrate.ErrNoChange) {
+	if err == nil || errors.Is(err, migrate.ErrNoChange) {
 		return nil
 	}
+
+	// If the database is in dirty state, try to fix it (PMM-14305)
+	var errDirty migrate.ErrDirty
+	if errors.As(err, &errDirty) {
+		log.Printf("Migration %d was unsuccessful, trying to fix it...", errDirty.Version)
+
+		// Note: -1 is a valid version to force to
+		ver := errDirty.Version - 1
+		fErr := m.Force(ver)
+		if fErr != nil {
+			return fmt.Errorf("can't force the migration %d: %w", ver, fErr)
+		}
+
+		// try to run migrations again
+		err = m.Up()
+		if errors.Is(err, migrate.ErrNoChange) {
+			return nil
+		}
+	}
+
 	return err
 }
 

qan-api2/main.go

@@ -18,6 +18,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	_ "expvar" // register /debug/vars
 	"fmt"
 	"html/template"
@@ -38,7 +39,6 @@ import (
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	grpc_gateway "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/jmoiron/sqlx"
-	"github.com/pkg/errors"
 	prom "github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"

qan-api2/utils/logger/logger.go

@@ -28,7 +28,7 @@ type keyStruct struct{}
 
 var key = keyStruct{}
 
-// Get returns logrus entry for given context. Set must be called before this method is called.
+// Get returns logrus entry for given context. SetEntry must be called before this method is called.
 func Get(ctx context.Context) *logrus.Entry {
 	v := ctx.Value(key)
 	if v == nil {
@@ -37,11 +37,6 @@ func Get(ctx context.Context) *logrus.Entry {
 	return v.(*logrus.Entry) //nolint:forcetypeassert
 }
 
-// Set returns derived context with set logrus entry with given request ID.
-func Set(ctx context.Context, requestID string) context.Context {
-	return SetEntry(ctx, logrus.WithField("request", requestID))
-}
-
 // SetEntry returns derived context with set given logrus entry.
 func SetEntry(ctx context.Context, l *logrus.Entry) context.Context {
 	if ctx.Value(key) != nil {