diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2189755..c804724 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -29,7 +29,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v2 with: - go-version: "1.20" + go-version: "1.24.1" - name: Test run: go test -v ./... diff --git a/.gitignore b/.gitignore index 1243afb..2a937b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -bin/sakura-controller +bin/db-controller diff --git a/AUTHORS b/AUTHORS index fa4878d..627d8f2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,5 +4,4 @@ # some cases, their employer may be the copyright holder. To see the full list # of contributors, see the revision history in source control. -Yamato Sugawara Shuichi Ohkubo diff --git a/Makefile b/Makefile index 9754ca7..445f0ef 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,5 @@ .PHONY: all -all: format test vet - -.PHONY: sakura-all -sakura-all: all sakura-build +all: format test vet build .PHONY: format format: @@ -19,18 +16,18 @@ ci: format test vet vet: go vet ./... -.PHONY: sakura-build -sakura-build: - go build -o bin/sakura-controller ./cmd/sakura-controller +.PHONY: build +build: + go build -o bin/db-controller ./cmd/db-controller .PHONY: check-license check-license: - go-licenses check ./cmd/sakura-controller + go-licenses check ./cmd/db-controller .PHONY: add-license add-license: addlicense -c "The distributed-mariadb-controller Authors" . - go-licenses check ./cmd/sakura-controller + go-licenses check ./cmd/db-controller .PHONY: tool tool: diff --git a/cmd/sakura-controller/api/v0/controller-middleware.go b/cmd/db-controller/api/v0/controller-middleware.go similarity index 85% rename from cmd/sakura-controller/api/v0/controller-middleware.go rename to cmd/db-controller/api/v0/controller-middleware.go index c2e139c..c4ad527 100644 --- a/cmd/sakura-controller/api/v0/controller-middleware.go +++ b/cmd/db-controller/api/v0/controller-middleware.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -19,7 +19,6 @@ import ( "github.com/labstack/echo/v4" "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller/sakura" ) const ( @@ -27,7 +26,7 @@ const ( ) // UseControllerState is an echo middleware that injects the current state of the db-controller into othe request context. -func UseControllerState(ctrler *sakura.SAKURAController) func(echo.HandlerFunc) echo.HandlerFunc { +func UseControllerState(ctrler *controller.Controller) func(echo.HandlerFunc) echo.HandlerFunc { return func(next echo.HandlerFunc) echo.HandlerFunc { return func(c echo.Context) error { c.Set(controllerStateCtxKey, ctrler.GetState()) diff --git a/cmd/sakura-controller/api/v0/controller-status.go b/cmd/db-controller/api/v0/controller-status.go similarity index 95% rename from cmd/sakura-controller/api/v0/controller-status.go rename to cmd/db-controller/api/v0/controller-status.go index a2d9251..2430c8e 100644 --- a/cmd/sakura-controller/api/v0/controller-status.go +++ b/cmd/db-controller/api/v0/controller-status.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/sakura-controller/api/v0/error.go b/cmd/db-controller/api/v0/error.go similarity index 91% rename from cmd/sakura-controller/api/v0/error.go rename to cmd/db-controller/api/v0/error.go index ee2520c..24f91fd 100644 --- a/cmd/sakura-controller/api/v0/error.go +++ b/cmd/db-controller/api/v0/error.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/sakura-controller/api/v0/gslb.go b/cmd/db-controller/api/v0/gslb.go similarity index 94% rename from cmd/sakura-controller/api/v0/gslb.go rename to cmd/db-controller/api/v0/gslb.go index dc5bb0f..1c79b27 100644 --- a/cmd/sakura-controller/api/v0/gslb.go +++ b/cmd/db-controller/api/v0/gslb.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/db-controller/cli.go b/cmd/db-controller/cli.go new file mode 100644 index 0000000..769f013 --- /dev/null +++ b/cmd/db-controller/cli.go @@ -0,0 +1,91 @@ +// Copyright 2025 The distributed-mariadb-controller Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" +) + +var ( + // logLevelFlag is a cli-flag that specifies the log level on the db-controller. + logLevelFlag string + // lockFilePathFlag is a cli-flag that specifies the filepath of the exclusive lock. + lockFilePathFlag string + // dbServingPortFlag is a cli-flag that specifies portnumber of database service + dbServingPortFlag int + // dbReplicaUserNameFlag is a cli-flag that specifies the username for replication + dbReplicaUserNameFlag string + // dbReplicaPasswordFilePathFlag is a cli-flag that specifies the filepath of the DB replica password. + dbReplicaPasswordFilePathFlag string + // globalInterfaceNameFlag is a cli-flag that specifies the global network interface for get my IPaddress. + globalInterfaceNameFlag string + // chainNameForDBAclFlag is a cli-flag that specifies the nftables chain name for DB access control list. + chainNameForDBAclFlag string + + // mainPollingSpanSecondFlag is a cli-flag that specifies the span seconds of the loop in main.go. + mainPollingSpanSecondFlag int + // httpAPIServerPortFlag is a cli-flag that specifies the port the HTTP API server listens. + httpAPIServerPortFlag int + // prometheusExporterPortFlag is a cli-flag that specifies the port the prometheus exporter listens. + prometheusExporterPortFlag int + // dbReplicaSourcePortFlag is a cli-flag that specifies the port of primary as replication source. + dbReplicaSourcePortFlag int + + // enablePrometheusExporterFlag is a cli-flag that enables the prometheus exporter. + enablePrometheusExporterFlag bool + // enableHTTPAPIFlag is a cli-flag that enables the http api server. + enableHTTPAPIFlag bool +) + +// ParseAllFlags parses all defined cmd-flags. +func parseAllFlags(args []string) error { + fs := flag.NewFlagSet("db-controller", flag.PanicOnError) + + fs.StringVar(&logLevelFlag, "log-level", "warning", "the log level(debug/info/warning/error)") + fs.StringVar(&lockFilePathFlag, "lock-filepath", "/var/run/db-controller/lock", "the filepath of the exclusive lock") + fs.StringVar(&dbReplicaPasswordFilePathFlag, "db-repilica-password-filepath", "/var/run/db-controller/.db-replica-password", "the filepath of the DB replica password") + fs.StringVar(&globalInterfaceNameFlag, "global-interface-name", "eth0", "the interface name of global") + fs.StringVar(&chainNameForDBAclFlag, "chain-name-for-db-acl", "mariadb", "the chain name for DB access control") + fs.StringVar(&dbReplicaUserNameFlag, "db-replica-user-name", "repl", "the username for replication") + + fs.IntVar(&mainPollingSpanSecondFlag, "main-polling-span-second", 4, "the span seconds of the loop in main.go") + fs.IntVar(&httpAPIServerPortFlag, "http-api-server-port", 54545, "the port the http api server listens") + fs.IntVar(&prometheusExporterPortFlag, "prometheus-exporter-port", 50505, "the port the prometheus exporter listens") + fs.IntVar(&dbReplicaSourcePortFlag, "db-replica-source-port", 13306, "the port of primary as replication source") + fs.IntVar(&dbServingPortFlag, "db-serving-port", 3306, "the port of database service") + + fs.BoolVar(&enablePrometheusExporterFlag, "prometheus-exporter", true, "enables the prometheus exporter") + fs.BoolVar(&enableHTTPAPIFlag, "http-api", true, "enables the http api server") + + return fs.Parse(args) +} + +// ValidateAllFlags validates all cmd flags. +func validateAllFlags() error { + if !isValidLogLevelFlag(logLevelFlag) { + return fmt.Errorf("--log-level must be one of debug/info/warning/error") + } + + if prometheusExporterPortFlag < 0 || 65535 < prometheusExporterPortFlag { + return fmt.Errorf("--prometheus-exporter-port must be the range of uint16(tcp port)") + } + + return nil +} + +func isValidLogLevelFlag(l string) bool { + return l == "debug" || l == "info" || l == "warning" || l == "error" +} diff --git a/cmd/sakura-controller/main.go b/cmd/db-controller/main.go similarity index 63% rename from cmd/sakura-controller/main.go rename to cmd/db-controller/main.go index 2d92dd2..d8aa7cf 100644 --- a/cmd/sakura-controller/main.go +++ b/cmd/db-controller/main.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ import ( "context" "fmt" "io" - "io/ioutil" + "log/slog" "net/http" "os" "os/signal" @@ -32,21 +32,13 @@ import ( "github.com/labstack/echo/v4/middleware" "github.com/labstack/gommon/log" "github.com/prometheus/client_golang/prometheus/promhttp" - apiv0 "github.com/sakura-internet/distributed-mariadb-controller/cmd/sakura-controller/api/v0" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/bash" + apiv0 "github.com/sakura-internet/distributed-mariadb-controller/cmd/db-controller/api/v0" "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller/sakura" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" "github.com/vishvananda/netlink" - - "golang.org/x/exp/rand" - "golang.org/x/exp/slog" ) func main() { - rand.Seed(uint64(time.Now().UnixNano())) - - ctx, cancel := context.WithCancel(context.Background()) - if err := parseAllFlags(os.Args[1:]); err != nil { panic(err) } @@ -54,14 +46,14 @@ func main() { panic(err) } - logger := setupGlobalLogger(os.Stderr, LogLevelFlag) + logger := setupGlobalLogger(os.Stderr, logLevelFlag) // mkdir for lock file - if err := os.MkdirAll(filepath.Dir(LockFilePathFlag), 0o755); err != nil { + if err := os.MkdirAll(filepath.Dir(lockFilePathFlag), 0o755); err != nil { panic(err) } - lockf, err := tryToGetTheExclusiveLockWithoutBlocking(LockFilePathFlag) + lockf, err := tryToGetTheExclusiveLockWithoutBlocking(lockFilePathFlag) if err != nil { panic(err) } @@ -69,63 +61,66 @@ func main() { // for controlling the traffics that they're to the DB server port. // the function returns nil if the expected chain is already exist. - if err := createNftablesChain(logger); err != nil { + nftConnect := nftables.NewDefaultConnector(logger) + if err := nftConnect.CreateChain(chainNameForDBAclFlag); err != nil { panic(err) } - c := sakura.NewSAKURAController(logger) - - { - eth0Address, err := getEth0NetIFAddress() - if err != nil { - panic(err) - } - - logger.Debug("eth0 address", "address", eth0Address) - c.HostAddress = eth0Address + // prepare controller instance + myHostAddress, err := getNetIFAddress(globalInterfaceNameFlag) + if err != nil { + panic(err) } - { - dbReplicaPassword, err := readDBReplicaPassword(DBReplicaPasswordFilePathFlag) - if err != nil { - panic(err) - } + logger.Debug("host address", "address", myHostAddress) - c.MariaDBReplicaPassword = dbReplicaPassword + dbReplicaPassword, err := readDBReplicaPassword(dbReplicaPasswordFilePathFlag) + if err != nil { + panic(err) } - c.MariaDBReplicaSourcePort = DBReplicaSourcePortFlag + c := controller.NewController( + logger, + controller.WithGlobalInterfaceName(globalInterfaceNameFlag), + controller.WithHostAddress(myHostAddress), + controller.WithDBServingPort(uint16(dbServingPortFlag)), + controller.WithDBReplicaUserName(dbReplicaUserNameFlag), + controller.WithDBReplicaPassword(dbReplicaPassword), + controller.WithDBReplicaSourcePort(uint16(dbReplicaSourcePortFlag)), + controller.WithDBAclChainName(chainNameForDBAclFlag), + ) + + // start goroutines + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() wg := new(sync.WaitGroup) wg.Add(1) - go func(ctx context.Context, wg *sync.WaitGroup, c *sakura.SAKURAController) { + go func(ctx context.Context, wg *sync.WaitGroup, c *controller.Controller) { defer wg.Done() - controller.Start(ctx, logger, controller.Controller(c), time.Second*time.Duration(MainPollingSpanSecondFlag)) + c.Start(ctx, time.Second*time.Duration(mainPollingSpanSecondFlag)) }(ctx, wg, c) - if EnablePrometheusExporterFlag { + if enablePrometheusExporterFlag { wg.Add(1) go startPrometheusExporterServer(ctx, wg) } - if EnableHTTPAPIFlag { + if enableHTTPAPIFlag { wg.Add(1) go startHTTPAPIServer(ctx, wg, c) } + // wait for receive signal signal.Ignore(syscall.SIGHUP, syscall.SIGPIPE) stopSigCh := make(chan os.Signal, 3) signal.Notify(stopSigCh, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM) -mainLoop: - for range stopSigCh { - logger.Info("got stop signal. exiting.") - - // for stopping all goroutine. - cancel() - break mainLoop - } + <-stopSigCh + logger.Info("got stop signal. exiting.") + // for stopping all goroutine. + cancel() wg.Wait() logger.Info("db-controller exited. see you again, bye.") @@ -141,7 +136,7 @@ func startPrometheusExporterServer( // Setup e := echo.New() - switch LogLevelFlag { + switch logLevelFlag { case "info": e.Logger.SetLevel(log.INFO) case "debug": @@ -153,11 +148,11 @@ func startPrometheusExporterServer( e.Logger.SetLevel(log.ERROR) } - reg := sakura.NewPrometheusMetricRegistry() + reg := controller.NewPrometheusMetricRegistry() e.GET("/metrics", echo.WrapHandler(promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))) // Start server - addr := fmt.Sprintf(":%d", PrometheusExporterPortFlag) + addr := fmt.Sprintf(":%d", prometheusExporterPortFlag) ch := make(chan bool, 1) go func(ch chan<- bool) { @@ -168,24 +163,18 @@ func startPrometheusExporterServer( ch <- true }(ch) -waitLoop: - for { - select { - case <-ctx.Done(): - if err := e.Shutdown(ctx); err != nil { - e.Logger.Fatal(err) - } - <-ch - break waitLoop - } + <-ctx.Done() + if err := e.Shutdown(ctx); err != nil { + e.Logger.Fatal(err) } + <-ch } // startHTTPAPIServer starts the HTTP API server that serves the controller status responder. func startHTTPAPIServer( ctx context.Context, wg *sync.WaitGroup, - c *sakura.SAKURAController, + c *controller.Controller, ) { defer wg.Done() @@ -193,7 +182,7 @@ func startHTTPAPIServer( e := echo.New() e.Use(apiv0.UseControllerState(c)) - switch LogLevelFlag { + switch logLevelFlag { case "info": e.Logger.SetLevel(log.INFO) case "debug": @@ -209,7 +198,7 @@ func startHTTPAPIServer( e.GET("/healthcheck", apiv0.GSLBHealthCheckEndpoint) e.GET("/status", apiv0.GetDBControllerStatus) // Start server - addr := fmt.Sprintf(":%d", HTTPAPIServerPortFlag) + addr := fmt.Sprintf(":%d", httpAPIServerPortFlag) ch := make(chan bool, 1) go func(ch chan<- bool) { @@ -220,39 +209,16 @@ func startHTTPAPIServer( ch <- true }(ch) -waitLoop: - for { - select { - case <-ctx.Done(): - if err := e.Shutdown(ctx); err != nil { - e.Logger.Fatal(err) - } - <-ch - break waitLoop - } - } -} - -// createNftablesChain tries to create an nftables chain on filter table. -func createNftablesChain( - logger *slog.Logger, -) error { - const ( - chainName = "mariadb" - ) - // nft add chain comand returns ok if the chain is already exist. - cmd := fmt.Sprintf("nft add chain filter %s { type filter hook input priority 0\\; }", chainName) - logger.Info("execute command", "command", cmd, "callerFn", "createNftablesChain") - if _, err := bash.RunCommand(cmd); err != nil { - return fmt.Errorf("failed to add nft chain: %w", err) + <-ctx.Done() + if err := e.Shutdown(ctx); err != nil { + e.Logger.Fatal(err) } - - return nil + <-ch } -// getEth0NetIFAddress tries to get the IP address of the eth0 I/F using Netlink messages. -func getEth0NetIFAddress() (string, error) { - eth, err := netlink.LinkByName("eth0") +// getNetIFAddress tries to get the IP address of the specified interface name I/F using Netlink messages. +func getNetIFAddress(intfname string) (string, error) { + eth, err := netlink.LinkByName(intfname) if err != nil { return "", err } @@ -263,7 +229,7 @@ func getEth0NetIFAddress() (string, error) { } if len(addrs) == 0 { - return "", fmt.Errorf("eth0 doesn't have any IP addresses") + return "", fmt.Errorf("%s doesn't have any IP addresses", intfname) } return addrs[0].IP.String(), nil @@ -285,7 +251,7 @@ func tryToGetTheExclusiveLockWithoutBlocking(path string) (*os.File, error) { // setupGlobalLogger setups a slog.Logger and sets it as the global logger of the slog packages. func setupGlobalLogger(w io.Writer, level string) *slog.Logger { - opts := slog.HandlerOptions{ + opts := &slog.HandlerOptions{ AddSource: true, } @@ -300,7 +266,7 @@ func setupGlobalLogger(w io.Writer, level string) *slog.Logger { opts.Level = slog.LevelError } - return slog.New(opts.NewTextHandler(w)) + return slog.New(slog.NewTextHandler(w, opts)) } // readDBReplicaPassword reads the contents from db replica password file. @@ -311,7 +277,7 @@ func readDBReplicaPassword(path string) (string, error) { } defer f.Close() - b, err := ioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return "", err } diff --git a/cmd/sakura-controller/cli.go b/cmd/sakura-controller/cli.go deleted file mode 100644 index e03ea53..0000000 --- a/cmd/sakura-controller/cli.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" -) - -var ( - // LogLevelFlag is a cli-flag that specifies the log level on the db-controller. - LogLevelFlag string - // LockFilePathFlag is a cli-flag that specifies the filepath of the exclusive lock. - LockFilePathFlag string - // DBReplicaPasswordFilePathFlag is a cli-flag that specifies the filepath of the DB replica password. - DBReplicaPasswordFilePathFlag string - - // MainPollingSpanSecondFlag is a cli-flag that specifies the span seconds of the loop in main.go. - MainPollingSpanSecondFlag int - // HTTPAPIServerPortFlag is a cli-flag that specifies the port the HTTP API server listens. - HTTPAPIServerPortFlag int - // PrometheusExporterPortFlag is a cli-flag that specifies the port the prometheus exporter listens. - PrometheusExporterPortFlag int - // DBReplicaSourcePortFlag is a cli-flag that specifies the port of primary as replication source. - DBReplicaSourcePortFlag int - - // EnablePrometheusExporterFlag is a cli-flag that enables the prometheus exporter. - EnablePrometheusExporterFlag bool - // EnableHTTPAPIFlag is a cli-flag that enables the http api server. - EnableHTTPAPIFlag bool -) - -// ParseAllFlags parses all defined cmd-flags. -func parseAllFlags(args []string) error { - fs := flag.NewFlagSet("db-controller", flag.PanicOnError) - - fs.StringVar(&LockFilePathFlag, "lock-filepath", "/var/run/db-controller/lock", "the filepath of the exclusive lock") - fs.StringVar(&DBReplicaPasswordFilePathFlag, "db-repilica-password-filepath", "/var/run/db-controller/.db-replica-password", "the filepath of the DB replica password") - fs.StringVar(&LogLevelFlag, "log-level", "warning", "the log level(debug/info/warning/error)") - - fs.IntVar(&MainPollingSpanSecondFlag, "main-polling-span-second", 4, "the span seconds of the loop in main.go") - fs.IntVar(&PrometheusExporterPortFlag, "prometheus-exporter-port", 50505, "the port the prometheus exporter listens") - fs.IntVar(&HTTPAPIServerPortFlag, "http-api-server-port", 54545, "the port the http api server listens") - fs.IntVar(&DBReplicaSourcePortFlag, "db-replica-source-port", 3306, "the port of primary as replication source") - - fs.BoolVar(&EnablePrometheusExporterFlag, "prometheus-exporter", true, "enables the prometheus exporter") - fs.BoolVar(&EnableHTTPAPIFlag, "http-api", true, "enables the http api server") - - return fs.Parse(args) -} - -// ValidateAllFlags validates all cmd flags. -func validateAllFlags() error { - if invalidLogLevelFlag(LogLevelFlag) { - return fmt.Errorf("--log-level must be one of debug/info/warning/error") - } - - if PrometheusExporterPortFlag < 0 || 65535 < PrometheusExporterPortFlag { - return fmt.Errorf("--prometheus-exporter-port must be the range of uint16(tcp port)") - } - - return nil -} - -func invalidLogLevelFlag(l string) bool { - valid := l == "debug" || l == "info" || l == "warning" || l == "error" - return !valid -} diff --git a/go.mod b/go.mod index 3d55664..a60d656 100644 --- a/go.mod +++ b/go.mod @@ -1,39 +1,36 @@ module github.com/sakura-internet/distributed-mariadb-controller -go 1.20 +go 1.24.1 require ( - github.com/labstack/echo/v4 v4.10.2 - github.com/labstack/gommon v0.4.0 - github.com/prometheus/client_golang v1.15.1 - github.com/stretchr/testify v1.8.1 - github.com/vishvananda/netlink v1.1.0 - golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb + github.com/labstack/echo/v4 v4.13.3 + github.com/labstack/gommon v0.4.2 + github.com/prometheus/client_golang v1.21.1 + github.com/stretchr/testify v1.10.0 + github.com/vishvananda/netlink v1.3.0 ) require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang-jwt/jwt v3.2.2+incompatible // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasttemplate v1.2.2 // indirect - github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df // indirect - golang.org/x/crypto v0.6.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/time v0.3.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + github.com/vishvananda/netns v0.0.4 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/time v0.8.0 // indirect + google.golang.org/protobuf v1.36.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3b45ac8..d399057 100644 --- a/go.sum +++ b/go.sum @@ -1,91 +1,71 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= -github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= -github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= -github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= -github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.13.3 h1:pwhpCPrTl5qry5HRdM5FwdXnhXSLSY+WE+YQSeCaafY= +github.com/labstack/echo/v4 v4.13.3/go.mod h1:o90YNEeQWjDozo584l7AwhJMHN0bOC4tAfg+Xox9q5g= +github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= +github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= -github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= +github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w= -golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= +github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/bash/run.go b/pkg/command/command.go similarity index 58% rename from pkg/bash/run.go rename to pkg/command/command.go index db523ff..d3629c5 100644 --- a/pkg/bash/run.go +++ b/pkg/command/command.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,11 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package bash +package command -import "os/exec" +import ( + "context" + "os/exec" + "time" +) -// RunCommand executes a shell command. -func RunCommand(cmd string) ([]byte, error) { - return exec.Command("sh", "-c", cmd).Output() +// RunWithTimeout executes a command with timeout. +func RunWithTimeout(timeout time.Duration, name string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return exec.CommandContext(ctx, name, args...).Output() } diff --git a/pkg/controller/candidate_state.go b/pkg/controller/candidate_state.go new file mode 100644 index 0000000..6ec8d8d --- /dev/null +++ b/pkg/controller/candidate_state.go @@ -0,0 +1,66 @@ +// Copyright 2025 The distributed-mariadb-controller Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "fmt" +) + +// decideNextStateOnCandidate determines the next state on candidate state. +func (c *Controller) decideNextStateOnCandidate() State { + if c.currentMariaDBHealth == dbHealthCheckResultNG { + c.logger.Warn("MariaDB is down. falling back to fault state.") + return StateFault + } + + if c.currentNeighbors.candidateNodeExists() || c.currentNeighbors.primaryNodeExists() { + c.logger.Info("another candidate or primary exists. falling back to fault state.") + return StateFault + } + + if c.readyToPrimary == readytoPrimaryJudgeOK { + return StatePrimary + } + + c.logger.Info("I'm not ready to primary. staying candidate state.") + return StateCandidate +} + +// triggerRunOnStateChangesToCandidate transition to candidate in main loop. +func (c *Controller) triggerRunOnStateChangesToCandidate() error { + // [STEP1]: setting MariaDB State. + if err := c.startMariaDBService(); err != nil { + return err + } + if health := c.checkMariaDBHealth(); health == dbHealthCheckResultNG { + return fmt.Errorf("MariaDB instance is down") + } + if err := c.syncReadOnlyVariable( /* read_only=1 */ true); err != nil { + return err + } + + // [STEP2]: setting Nftables State. + if err := c.rejectDatabaseServiceTraffic(); err != nil { + return err + } + + // [STEP3]: configurating frrouting. + if err := c.advertiseSelfNetIFAddress(); err != nil { + return err + } + + c.logger.Info("candidate state handler succeed") + return nil +} diff --git a/pkg/controller/candidate_state_whitebox_test.go b/pkg/controller/candidate_state_whitebox_test.go new file mode 100644 index 0000000..1bbd82e --- /dev/null +++ b/pkg/controller/candidate_state_whitebox_test.go @@ -0,0 +1,77 @@ +// Copyright 2025 The distributed-mariadb-controller Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDecideNextStateOnCandidate_MariaDBIsUnhealthy(t *testing.T) { + c := _newFakeController() + c.currentMariaDBHealth = dbHealthCheckResultNG + c.readyToPrimary = readytoPrimaryJudgeNG + + nextState := c.decideNextStateOnCandidate() + assert.Equal(t, StateFault, nextState) +} + +func TestDecideNextStateOnCandidate_InMultiCandidateSituation(t *testing.T) { + c := _newFakeController() + c.currentNeighbors[StateCandidate] = []neighbor{""} + c.currentMariaDBHealth = dbHealthCheckResultOK + c.readyToPrimary = readytoPrimaryJudgeNG + + nextState := c.decideNextStateOnCandidate() + assert.Equal(t, StateFault, nextState) +} + +func TestDecideNextStateOnCandidate_PrimaryIsAlreadyExist(t *testing.T) { + c := _newFakeController() + c.currentNeighbors[StatePrimary] = []neighbor{""} + c.currentMariaDBHealth = dbHealthCheckResultOK + c.readyToPrimary = readytoPrimaryJudgeNG + + nextState := c.decideNextStateOnCandidate() + assert.Equal(t, StateFault, nextState) +} + +func TestDecideNextStateOnCandidate_ToBePromotedToPrimary(t *testing.T) { + c := _newFakeController() + c.currentNeighbors[StateFault] = []neighbor{""} + c.currentMariaDBHealth = dbHealthCheckResultOK + c.readyToPrimary = readytoPrimaryJudgeOK + + nextState := c.decideNextStateOnCandidate() + assert.Equal(t, StatePrimary, nextState) +} + +func TestDecideNextStateCandidate_RemainCandidate(t *testing.T) { + c := _newFakeController() + c.currentNeighbors[StateFault] = []neighbor{""} + c.currentMariaDBHealth = dbHealthCheckResultOK + c.readyToPrimary = readytoPrimaryJudgeNG + + nextState := c.decideNextStateOnCandidate() + assert.Equal(t, StateCandidate, nextState) +} + +func TestTriggerRunOnStateChangesToCandidate_OKPath(t *testing.T) { + c := _newFakeController() + + err := c.triggerRunOnStateChangesToCandidate() + assert.NoError(t, err) +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 7ddde3d..f435bf0 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,59 +14,497 @@ package controller -// Controller manages distributed MariaDB cluster in each database server. -// The controller forms as a state machine. -type Controller interface { - /// GetState returns the current state of the controller. - GetState() State - // PreDecideNextStateHandler is triggered before calling MakeDecision() - PreDecideNextStateHandler() error - // DecideNextState determines next state that the controller should transition. - DecideNextState() State - // OnStateHandler is an implementation of the root state handler. - // All controller must trigget the state handler on the given state. - OnStateHandler(nextState State) error - // OnExit is triggered when the Start() received context.Context.Done(). - OnExit() error -} +import ( + "context" + "fmt" + "log/slog" + "math/rand" + "net" + "sync" + "time" + + "github.com/sakura-internet/distributed-mariadb-controller/pkg/frrouting/bgpd" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/frrouting/vtysh" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/mariadb" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/systemd" +) // State specifies the controller state. type State string const ( - StateInitial State = "initial" - StatePrimary State = "primary" - StateReplica State = "replica" - StateFault State = "fault" + StateInitial State = "initial" + StateFault State = "fault" + StateCandidate State = "candidate" + StatePrimary State = "primary" + StateReplica State = "replica" + StateAnchor State = "anchor" +) + +var ( + controllerAllStates = map[State]bool{ + StateInitial: true, + StateFault: true, + StatePrimary: true, + StateCandidate: true, + StateReplica: true, + } +) + +// dbHealthCheckResult is the result of the mariadb's healthcheck +type dbHealthCheckResult uint + +const ( + dbHealthCheckResultOK dbHealthCheckResult = iota + dbHealthCheckResultNG ) -// UnimplementedController implements Controller interface. -// each method of the Controller does nothing. -type UnimplementedController struct{} +// readyToPrimaryJudge is the result of the judgement to be promoted to primary state. +type readyToPrimaryJudge uint + +const ( + // readytoPrimaryJudgeOK is OK for being promoted to primary state + readytoPrimaryJudgeOK readyToPrimaryJudge = iota + // readytoPrimaryJudgeNG is NG for being promoted to primary state + readytoPrimaryJudgeNG +) -// DecideNextState implements Controller -func (*UnimplementedController) DecideNextState() State { - return StateFault +type Controller struct { + logger *slog.Logger + // globalInterfaceName is DB service interface name. + globalInterfaceName string + // hostAddress is an IP address of the global interface. + hostAddress string + // dbServingPort is the port number of database service + dbServingPort uint16 + // dbReplicaUserName is the username for replication + dbReplicaUserName string + // dbReplicaSourcePort is the port of primary as replication source. + dbReplicaSourcePort uint16 + // dbReplicaPassword is credential used by replica to establish replication link for primary + dbReplicaPassword string + // dbAclChainName is the nftables chain name for database access control. + dbAclChainName string + + // currentState is the current state of the controller. + // for prevending unexpected transition, the state isn't exposed. + currentState State + // prevState is the previous state of the controller. + prevState State + // m is a read-write-mutex that is used for sharing controller's state btw controller/http-api goroutines. + m sync.RWMutex + // replicationStatusCheckFailCount is a counter of the MariaDB's replication status checker in replica state. + replicationStatusCheckFailCount uint + // writeTestDataFailCount is a counter that the controller tries to write test data to MariaDB. + // if the count overs the pre-declared threshold, the controller urgently exits. + writeTestDataFailCount uint + // currentNeighbors holds the current BGP neighbors of the dbserver. + // that discovered in each loop of the controller. + currentNeighbors neighborSet + // currentMariaDBHealth holds the most recent healthcheck's result. + currentMariaDBHealth dbHealthCheckResult + // readyToPrimary + readyToPrimary readyToPrimaryJudge + + // nftablesConnector communicates with FRRouting BGPd via vtysh. + nftablesConnector nftables.Connector + // bgpdConnector communicates with FRRouting BGPd via vtysh. + bgpdConnector bgpd.BGPdConnector + // systemdConnector manages the systemd services. + systemdConnector systemd.Connector + // mariaDBConnector communicates with MariaDB via mysql-client. + mariaDBConnector mariadb.Connector } -// PreDecideNextStateHandler implements Controller -func (*UnimplementedController) PreDecideNextStateHandler() error { +func NewController( + logger *slog.Logger, + configs ...ControllerConfig, +) *Controller { + c := &Controller{ + logger: logger, + + currentState: StateInitial, + currentNeighbors: newNeighborSet(), + + nftablesConnector: nftables.NewDefaultConnector(logger), + bgpdConnector: vtysh.NewDefaultBGPdConnector(logger), + mariaDBConnector: mariadb.NewDefaultConnector(logger), + systemdConnector: systemd.NewDefaultConnector(logger), + } + + for _, cfg := range configs { + cfg(c) + } + return c +} + +// Start starts the controller loop. +// the function recognizes a done signal from the given context. +func (c *Controller) Start( + ctx context.Context, + ctrlerLoopInterval time.Duration, +) { + c.logger.Info("Hello, Starting db-controller.") + + ticker := time.NewTicker(ctrlerLoopInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + c.forceTransitionToFault() + return + case <-ticker.C: + // random sleep to avoid global synchronization + time.Sleep(time.Second * time.Duration(rand.Intn(2)+1)) + + if err := c.preDecideNextStateHandler(); err != nil { + c.logger.Error("preDecideNextStateHandler", "error", err, "state", string(c.GetState())) + // we urgently transition to fault state + c.forceTransitionToFault() + continue + } + + nextState := c.decideNextState() + c.logger.Debug("controller decided next state", "next state", nextState) + + if err := c.onStateHandler(nextState); err != nil { + c.logger.Error("onStateHandler", "error", err, "next state", nextState) + } + } + } +} + +// GetState returns the current state of the controller. +func (c *Controller) GetState() State { + c.m.RLock() + defer c.m.RUnlock() + + return c.currentState +} + +// decideNextState determines next state that the controller should transition. +func (c *Controller) decideNextState() State { + c.logger.Debug("decide next state", "current state", c.GetState()) + if c.currentNeighbors.isNetworkParted() { + c.logger.Info("detected network partition", "neighbors", c.currentNeighbors.neighborAddresses()) + return StateFault + } + + switch c.GetState() { + case StateFault: + return c.decideNextStateOnFault() + case StateCandidate: + return c.decideNextStateOnCandidate() + case StatePrimary: + return c.decideNextStateOnPrimary() + case StateReplica: + return c.decideNextStateOnReplica() + case StateInitial: + // just initialized controller take this case. + return StateFault + default: + panic("unreachable") + } +} + +func (c *Controller) onStateHandler(nextState State) error { + if cannotTransitionTo(c.GetState(), nextState) { + panic("unreachable controller state was picked") + } + c.setState(nextState) + + if c.keepStateInPrevTransition() { + if err := c.triggerRunOnStateKeeps(); err != nil { + c.logger.Error("failed to triggerRunOnStateKeeps. transition to fault state and exit", "error", err, "state", string(c.GetState())) + c.forceTransitionToFault() + panic("urgently exit") + } + + return nil + } + + if err := c.triggerRunOnStateChanges(); err != nil { + // we urgently transition to fault state + c.logger.Error("failed to TriggerRunOnStateChanges. transition to fault state.", "error", err, "state", string(c.GetState())) + c.forceTransitionToFault() + } + return nil } -var _ Controller = &UnimplementedController{} +// preDecideNextStateHandler is triggered before calling MakeDecision() +func (c *Controller) preDecideNextStateHandler() error { + prevNeighbors := c.currentNeighbors + prefixes, err := c.collectStateCommunityRoutePrefixes() + if err != nil { + return fmt.Errorf("failed to collect BGP routes: %w", err) + } + + c.currentNeighbors = c.extractNeighborAddresses(prefixes) + // to avoiding unnecessary calculation, we checks the logger's level. + if prevNeighbors.different(c.currentNeighbors) { + addrs := c.currentNeighbors.neighborAddresses() + c.logger.Info("neighbor set is updated", "addresses", addrs) + } + + c.currentMariaDBHealth = c.checkMariaDBHealth() + + // judging "not ready" to primary when mariadb is not healthy + if c.currentMariaDBHealth == dbHealthCheckResultNG { + c.readyToPrimary = readytoPrimaryJudgeNG + return nil + } + c.readyToPrimary = c.readyToBePromotedToPrimary() + + return nil +} + +// setState sets the given state as the current state of the controller. +func (c *Controller) setState(nextState State) { + c.prevState = c.GetState() + { + c.m.Lock() + c.currentState = nextState + c.m.Unlock() + } + + if c.prevState == nextState { + c.logger.Debug("controller transitions the state(unchanged)", "from", c.prevState, "to", nextState) + } else { + c.logger.Info("controller transitions the state(changed)", "from", c.prevState, "to", nextState) + } + + // modify state metric(s) + dbControllerStateTransitionCounterVec.WithLabelValues(string(nextState)).Inc() + for s := range controllerAllStates { + // clear flag of all state + dbControllerStateGaugeVec.WithLabelValues(string(s)).Set(0) + } + // set flag of next state + dbControllerStateGaugeVec.WithLabelValues(string(nextState)).Set(1) +} + +// triggerRunOnStateChanges triggers the state handler if the previous state is not the current state. +func (c *Controller) triggerRunOnStateChanges() error { + switch c.GetState() { + case StatePrimary: + return c.triggerRunOnStateChangesToPrimary() + case StateFault: + return c.triggerRunOnStateChangesToFault() + case StateCandidate: + return c.triggerRunOnStateChangesToCandidate() + case StateReplica: + return c.triggerRunOnStateChangesToReplica() + } -// GetState implements Controller -func (*UnimplementedController) GetState() State { - return StateFault + panic("unreachable") } -// OnExit implements Controller -func (*UnimplementedController) OnExit() error { +// triggerRunOnStateKeeps triggers the state handler if the previous state is same as the current state. +func (c *Controller) triggerRunOnStateKeeps() error { + switch c.GetState() { + case StatePrimary: + return c.triggerRunOnStateKeepsPrimary() + case StateReplica: + return c.triggerRunOnStateKeepsReplica() + } + return nil } -// OnStateHandler implements Controller -func (*UnimplementedController) OnStateHandler(nextState State) error { +// advertiseSelfNetIFAddress updates the configuration of the advertising route. +// the BGP community of the advertising route will be updated with the current controller-state. +func (c *Controller) advertiseSelfNetIFAddress() error { + _, selfAddr, err := net.ParseCIDR(c.hostAddress + "/32") + if err != nil { + return err + } + return c.bgpdConnector.ConfigureRouteWithRouteMap(*selfAddr, string(c.GetState())) +} + +// forceTransitionToFault set state to fault and triggers fault handler +func (c *Controller) forceTransitionToFault() { + // do nothing when already state is fault + if c.GetState() == StateFault { + return + } + + c.setState(StateFault) + if err := c.triggerRunOnStateChanges(); err != nil { + c.logger.Info("failed to TriggerRunOnStateChanges while going to fault. Ignore errors.", "error", err) + } +} + +// keepStateInPrevTransition determins wheather a state transition has occurred +func (c *Controller) keepStateInPrevTransition() bool { + prev := c.getPreviousState() + cur := c.GetState() + return prev == cur +} + +// getPreviousState returns the controller's previous state. +func (c *Controller) getPreviousState() State { + c.m.RLock() + defer c.m.RUnlock() + + return c.prevState +} + +// checkMariaDBHealth checks whether the MariaDB server is healthy or not. +func (c *Controller) checkMariaDBHealth() dbHealthCheckResult { + if err := c.systemdConnector.CheckServiceStatus(mariadb.SystemdSerivceName); err != nil { + c.logger.Debug("'systemctl status mariadb' exit with returning error", "error", err) + return dbHealthCheckResultNG + } + + return dbHealthCheckResultOK +} + +// readyToBePromotedToPrimary returns true when the controller satisfies the conditions to be promoted to primary state. +func (c *Controller) readyToBePromotedToPrimary() readyToPrimaryJudge { + status, err := c.mariaDBConnector.ShowReplicationStatus() + if err != nil { + c.logger.Debug("failed to show replication status", "error", err) + return readytoPrimaryJudgeNG + } + + readMasterLogPos, ok := status[mariadb.ReplicationStatusReadMasterLogPos] + if !ok { + return readytoPrimaryJudgeOK + } + + if readMasterLogPos == status[mariadb.ReplicationStatusExecMasterLogPos] && + status[mariadb.ReplicationStatusMasterLogFile] == status[mariadb.ReplicationStatusRelayMasterLogFile] { + return readytoPrimaryJudgeOK + } + + return readytoPrimaryJudgeNG +} + +// CollectStateCommunityRoutePrefixes collects the BGP route-prefix that they have a community of a controller-state. +func (c *Controller) collectStateCommunityRoutePrefixes() (map[State][]net.IP, error) { + routes := make(map[State][]net.IP) + + // StateInitial is not needed in the below slice because the state doesn't advertise any routes. + states := []State{ + StateCandidate, + StateFault, + StatePrimary, + StateReplica, + StateAnchor, + } + + for _, state := range states { + if routes[state] == nil { + routes[state] = make([]net.IP, 0) + } + + bgp, err := c.bgpdConnector.ShowRoutesWithBGPCommunityList(string(state)) + if err != nil { + return nil, err + } + + for routePrefix := range bgp.Routes { + // NOTE: we recommend you use net/netip instead of net package + // because the netip.Addr is the most prefered way to present an IP address in Go. + // but the net/netip package doesn't have the way to parse CIDR notation. + addr, _, err := net.ParseCIDR(routePrefix) + if err != nil { + c.logger.Error("failed to parse route prefix", "error", err) + } + + routes[state] = append(routes[state], addr) + } + } + + return routes, nil +} + +// ExtractNeighborAddresses get only the addresses of the neighbors from the given prefixes. +func (c *Controller) extractNeighborAddresses( + prefixMatrix map[State][]net.IP, +) neighborSet { + neighbors := newNeighborSet() + + for state, prefixes := range prefixMatrix { + + for _, prefix := range prefixes { + + // each prefix of the advertised BGP route is the unicast address of other DB instances. + // if the route prefix(unicast IP) and my address are same, + // the route is advertised from me so it should be ignored. + if prefix.String() == c.hostAddress { + continue + } + + if neighbors[state] == nil { + neighbors[state] = make([]neighbor, 0) + } + + neighbors[state] = append( + neighbors[state], + neighbor(prefix.String()), + ) + } + } + + return neighbors +} + +// syncReadOnlyVariable updates the read_only variable to the given expected value. +// if the current value equals the given value, the variable is already synced. +// otherwise, the function tries to sync the variable. +func (c *Controller) syncReadOnlyVariable(readOnlyToBeTrue bool) error { + isOn := c.mariaDBConnector.IsReadOnly() + + if readOnlyToBeTrue == isOn { + // the variable is already the expected value. + // nothing to do. + return nil + } + + if readOnlyToBeTrue { + return c.mariaDBConnector.TurnOnReadOnly() + } + + return c.mariaDBConnector.TurnOffReadOnly() +} + +// startMariaDBService starts the mariadb service of systemd. +func (c *Controller) startMariaDBService() error { + if err := c.mariaDBConnector.RemoveMasterInfo(); err != nil { + return err + } + if err := c.mariaDBConnector.RemoveRelayInfo(); err != nil { + return err + } + if err := c.systemdConnector.StartService(mariadb.SystemdSerivceName); err != nil { + return err + } + return nil } + +// cannotTransitionTo checks whether the state machine doesn't have the edge from current to next. +func cannotTransitionTo( + currentState State, + nextState State, +) bool { + switch currentState { + case StateFault: + return nextState == StatePrimary + case StateCandidate: + return nextState == StateReplica + case StatePrimary: + return nextState == StateCandidate || nextState == StateReplica + case StateReplica: + return nextState == StatePrimary + case StateInitial: + return nextState != StateFault + default: + // unreachable + return true + } +} diff --git a/pkg/controller/controller_config.go b/pkg/controller/controller_config.go new file mode 100644 index 0000000..1148a68 --- /dev/null +++ b/pkg/controller/controller_config.go @@ -0,0 +1,95 @@ +// Copyright 2025 The distributed-mariadb-controller Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "github.com/sakura-internet/distributed-mariadb-controller/pkg/frrouting/bgpd" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/mariadb" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/systemd" +) + +// ControllerConfig is the configuration that is applied into Controller. +type ControllerConfig func(c *Controller) + +func WithGlobalInterfaceName(globalInterfaceName string) ControllerConfig { + return func(c *Controller) { + c.globalInterfaceName = globalInterfaceName + } +} + +func WithHostAddress(hostAddress string) ControllerConfig { + return func(c *Controller) { + c.hostAddress = hostAddress + } +} + +func WithDBServingPort(dbServingPort uint16) ControllerConfig { + return func(c *Controller) { + c.dbServingPort = dbServingPort + } +} + +func WithDBReplicaUserName(dbReplicaUserName string) ControllerConfig { + return func(c *Controller) { + c.dbReplicaUserName = dbReplicaUserName + } +} + +func WithDBReplicaPassword(dbReplicaPassword string) ControllerConfig { + return func(c *Controller) { + c.dbReplicaPassword = dbReplicaPassword + } +} + +func WithDBReplicaSourcePort(dbReplicaSourcePort uint16) ControllerConfig { + return func(c *Controller) { + c.dbReplicaSourcePort = dbReplicaSourcePort + } +} + +func WithDBAclChainName(dbAclChainName string) ControllerConfig { + return func(c *Controller) { + c.dbAclChainName = dbAclChainName + } +} + +// WithSystemdConnector generates a config that sets the systemd.Connector into Controller. +func WithSystemdConnector(connector systemd.Connector) ControllerConfig { + return func(c *Controller) { + c.systemdConnector = connector + } +} + +// WithMariaDBConnector generates a config that sets the mariadb.Connector into Controller. +func WithMariaDBConnector(connector mariadb.Connector) ControllerConfig { + return func(c *Controller) { + c.mariaDBConnector = connector + } +} + +// WithNftablesConnector generates a config that sets the nftables.Connector into Controller. +func WithNftablesConnector(connector nftables.Connector) ControllerConfig { + return func(c *Controller) { + c.nftablesConnector = connector + } +} + +// WithBGPdConnector generates a config that sets the vtysh.WithBGPdConnector into Controller. +func WithBGPdConnector(connector bgpd.BGPdConnector) ControllerConfig { + return func(c *Controller) { + c.bgpdConnector = connector + } +} diff --git a/pkg/controller/fault_state.go b/pkg/controller/fault_state.go new file mode 100644 index 0000000..4598fcc --- /dev/null +++ b/pkg/controller/fault_state.go @@ -0,0 +1,93 @@ +// Copyright 2025 The distributed-mariadb-controller Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "github.com/sakura-internet/distributed-mariadb-controller/pkg/mariadb" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" +) + +// decideNextStateOnFault determines the next state on fault state +func (c *Controller) decideNextStateOnFault() State { + if c.currentNeighbors.primaryNodeExists() { + return StateReplica + } + + if c.currentNeighbors.candidateNodeExists() || c.currentNeighbors.replicaNodeExists() { + c.logger.Info("another candidate or replica exists") + return StateFault + } + + // the fault controller is ready to transition to candidate state + // because network reachability is ok and no one candidate is here. + return StateCandidate +} + +// triggerRunOnStateChangesToFault transition to fault state in main loop. +// In fault state, the controller just reflect the fault state to external resources. +func (c *Controller) triggerRunOnStateChangesToFault() error { + // [STEP1]: configurating frrouting + if err := c.advertiseSelfNetIFAddress(); err != nil { + c.logger.Warn("failed to advertise self-address in BGP but ignored because i'm fault", "error", err) + } + + // [STEP2]: setting nftables state + if err := c.rejectDatabaseServiceTraffic(); err != nil { + c.logger.Warn("failed to reject tcp traffic to database serving port but ignored because i'm fault", "error", err) + } + + // [STEP3]: setting MariaDB state + if err := c.systemdConnector.KillService(mariadb.SystemdSerivceName); err != nil { + c.logger.Warn("failed to kill db service but ignored because i'm fault", "error", err) + } + if err := c.stopMariaDBService(); err != nil { + c.logger.Warn("failed to stop systemd mariadb process but ignored because i'm fault", "error", err) + } + + c.logger.Info("fault state handler succeed") + return nil +} + +// rejectDatabaseServiceTraffic sets the reject rule that denies the inbound communication from the outsider of the network. +func (c *Controller) rejectDatabaseServiceTraffic() error { + if err := c.nftablesConnector.FlushChain( + c.dbAclChainName, + ); err != nil { + return err + } + + rejectMatches := []nftables.Match{ + nftables.IFNameMatch(c.globalInterfaceName), + nftables.TCPDstPortMatch(uint16(c.dbServingPort)), + } + if err := c.nftablesConnector.AddRule( + c.dbAclChainName, + rejectMatches, + nftables.RejectStatement(), + ); err != nil { + return err + } + + return nil +} + +// stopMariaDBService stops the mariadb's systemd service. +func (c *Controller) stopMariaDBService() error { + if err := c.systemdConnector.StopService(mariadb.SystemdSerivceName); err != nil { + return err + } + + return nil +} diff --git a/pkg/controller/sakura/fault_state_whitebox_test.go b/pkg/controller/fault_state_whitebox_test.go similarity index 50% rename from pkg/controller/sakura/fault_state_whitebox_test.go rename to pkg/controller/fault_state_whitebox_test.go index 3f1c09b..75f1e90 100644 --- a/pkg/controller/sakura/fault_state_whitebox_test.go +++ b/pkg/controller/fault_state_whitebox_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,20 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sakura +package controller import ( - "os" "testing" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" "github.com/sakura-internet/distributed-mariadb-controller/pkg/systemd" "github.com/stretchr/testify/assert" - "golang.org/x/exp/slog" ) func TestTriggerRunOnStateChangesToFault_OKPath(t *testing.T) { - c := _newFakeSAKURAController() + c := _newFakeController() err := c.triggerRunOnStateChangesToFault() assert.NoError(t, err) @@ -37,37 +34,33 @@ func TestTriggerRunOnStateChangesToFault_OKPath(t *testing.T) { } func TestDecideNextStateOnFault_WithPrimaryNeighbors(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StatePrimary] = append(ns.NeighborMatrix[controller.StatePrimary], Neighbor{}) + c := _newFakeController() + c.currentNeighbors[StatePrimary] = []neighbor{""} - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnFault(logger, ns) - assert.Equal(t, controller.StateReplica, nextState) + nextState := c.decideNextStateOnFault() + assert.Equal(t, StateReplica, nextState) } func TestDecideNextStateOnFault_WithCandidateNeighbors(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[SAKURAControllerStateCandidate] = append(ns.NeighborMatrix[SAKURAControllerStateCandidate], Neighbor{}) + c := _newFakeController() + c.currentNeighbors[StateCandidate] = []neighbor{""} - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnFault(logger, ns) - assert.Equal(t, controller.StateFault, nextState) + nextState := c.decideNextStateOnFault() + assert.Equal(t, StateFault, nextState) } func TestDecideNextStateOnFault_WithReplicaNeighbors(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StateReplica] = append(ns.NeighborMatrix[controller.StateReplica], Neighbor{}) + c := _newFakeController() + c.currentNeighbors[StateReplica] = []neighbor{""} - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnFault(logger, ns) - assert.Equal(t, controller.StateFault, nextState) + nextState := c.decideNextStateOnFault() + assert.Equal(t, StateFault, nextState) } func TestDecideNextStateOnFault_WithoutNeighbors(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StateFault] = append(ns.NeighborMatrix[controller.StateFault], Neighbor{}) + c := _newFakeController() + c.currentNeighbors[StateFault] = []neighbor{""} - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnFault(logger, ns) - assert.Equal(t, SAKURAControllerStateCandidate, nextState) + nextState := c.decideNextStateOnFault() + assert.Equal(t, StateCandidate, nextState) } diff --git a/pkg/controller/neighbor.go b/pkg/controller/neighbor.go new file mode 100644 index 0000000..bff95e6 --- /dev/null +++ b/pkg/controller/neighbor.go @@ -0,0 +1,111 @@ +// Copyright 2025 The distributed-mariadb-controller Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "fmt" + "reflect" + "strings" +) + +// neighbor is the BGP neighbor. +type neighbor string + +// neighborSet holds the set of the BGP neighbors. +type neighborSet map[State][]neighbor + +// newNeighborSet initializes the empty NeighborSet. +func newNeighborSet() neighborSet { + return neighborSet{ + StateFault: make([]neighbor, 0), + StateCandidate: make([]neighbor, 0), + StatePrimary: make([]neighbor, 0), + StateReplica: make([]neighbor, 0), + } +} + +// different returns true if the n and other is differenct. +func (n neighborSet) different(other neighborSet) bool { + if len(n) != len(other) { + return true + } + + for k, nNeighbors := range n { + oNeighbors, ok := other[k] + if !ok { + return true + } + + if !reflect.DeepEqual(nNeighbors, oNeighbors) { + return true + } + } + + return false +} + +// neighborAddresses construct the addresses of the neighbors into a string. +func (n neighborSet) neighborAddresses() string { + addressesByState := []string{} + + for state, neighbors := range n { + addrs := make([]string, len(neighbors)) + for i, neighbor := range neighbors { + addrs[i] = string(neighbor) + } + + addressesByState = append(addressesByState, fmt.Sprintf("%s: [%s]", state, strings.Join(addrs, ","))) + } + + return strings.Join(addressesByState, ", ") +} + +// primaryNodeExists returns true if the set contains primary-state node(s). +func (n neighborSet) primaryNodeExists() bool { + return len(n[StatePrimary]) != 0 +} + +// replicaNodeExists returns true if the set contains replica-state node(s). +func (n neighborSet) replicaNodeExists() bool { + return len(n[StateReplica]) != 0 +} + +// candidateNodeExists returns true if the set contains candidate-state node(s). +func (n neighborSet) candidateNodeExists() bool { + return len(n[StateCandidate]) != 0 +} + +// faultNodeExists returns true if the set contains fault-state node(s). +func (n neighborSet) faultNodeExists() bool { + return len(n[StateFault]) != 0 +} + +// anchorNodeExists returns true if the set contains anchor-mode node(s). +func (n neighborSet) anchorNodeExists() bool { + return len(n[StateAnchor]) != 0 +} + +// isNetworkParted returns true if there is no neighbor on the network. +func (n neighborSet) isNetworkParted() bool { + if n.primaryNodeExists() || + n.candidateNodeExists() || + n.replicaNodeExists() || + n.faultNodeExists() || + n.anchorNodeExists() { + return false + } + + return true +} diff --git a/pkg/controller/neighbor_test.go b/pkg/controller/neighbor_test.go new file mode 100644 index 0000000..ec5c7e2 --- /dev/null +++ b/pkg/controller/neighbor_test.go @@ -0,0 +1,44 @@ +// Copyright 2025 The distributed-mariadb-controller Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDifferent_Same(t *testing.T) { + a := newNeighborSet() + b := newNeighborSet() + assert.False(t, a.different(b)) +} + +func TestDifferent_DiffLen(t *testing.T) { + a := newNeighborSet() + b := newNeighborSet() + b[StateInitial] = []neighbor{""} + assert.True(t, a.different(b)) +} + +func TestDifferent_DiffNeigh(t *testing.T) { + a := newNeighborSet() + a[StateInitial] = []neighbor{""} + + b := newNeighborSet() + b[StateInitial] = []neighbor{"10.0.0.1"} + assert.True(t, a.different(b)) + +} diff --git a/pkg/controller/sakura/primary_state.go b/pkg/controller/primary_state.go similarity index 61% rename from pkg/controller/sakura/primary_state.go rename to pkg/controller/primary_state.go index 08517dd..571ba7e 100644 --- a/pkg/controller/sakura/primary_state.go +++ b/pkg/controller/primary_state.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,14 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sakura +package controller import ( "fmt" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" - "golang.org/x/exp/slog" ) const ( @@ -34,36 +32,32 @@ const ( ) // decideNextStateOnPrimary determines the next state on primary state -func decideNextStateOnPrimary( - logger *slog.Logger, - neighbors *NeighborSet, - mariaDBHealth MariaDBHealthCheckResult, -) controller.State { - if mariaDBHealth == MariaDBHealthCheckResultNG { - logger.Warn("MariaDB instance is down") - return controller.StateFault +func (c *Controller) decideNextStateOnPrimary() State { + if c.currentMariaDBHealth == dbHealthCheckResultNG { + c.logger.Warn("MariaDB instance is down") + return StateFault } // found dual-primary situation. - if neighbors.primaryNodeExists() { - logger.Warn("dual primary detected") - return controller.StateFault + if c.currentNeighbors.primaryNodeExists() { + c.logger.Warn("dual primary detected") + return StateFault } // won't transition to other state. - return controller.StatePrimary + return StatePrimary } // triggerRunOnStateChangesToPrimary processes transition to primary state in main loop. -func (c *SAKURAController) triggerRunOnStateChangesToPrimary() error { - if health := c.checkMariaDBHealth(); health == MariaDBHealthCheckResultNG { +func (c *Controller) triggerRunOnStateChangesToPrimary() error { + if health := c.checkMariaDBHealth(); health == dbHealthCheckResultNG { return fmt.Errorf("MariaDB instance is down") } - if c.CurrentNeighbors.primaryNodeExists() { + if c.currentNeighbors.primaryNodeExists() { return fmt.Errorf("dual primary detected") } - // [STEP1]: START of setting MariaDB state + // [STEP1]: setting MariaDB state if err := c.mariaDBConnector.StopReplica(); err != nil { return err } @@ -73,56 +67,54 @@ func (c *SAKURAController) triggerRunOnStateChangesToPrimary() error { if err := c.syncReadOnlyVariable( /* read_only=0 */ false); err != nil { return err } - // [STEP1]: END of setting MariaDB state - // [STEP2]: START of setting nftables state - if err := c.acceptTCP3306Traffic(); err != nil { + // [STEP2]: setting nftables state + if err := c.acceptDatabaseServiceTraffic(); err != nil { return err } - // [STEP2]: END of setting nftables state - // [STEP3]: START of configurating frrouting + // [STEP3]: configurating frrouting if err := c.advertiseSelfNetIFAddress(); err != nil { return err } - // [STEP3]: END of configurating frrouting // reset the count because the controller is healthy. c.writeTestDataFailCount = 0 - c.Logger.Info("primary state handler succeed") + c.logger.Info("primary state handler succeed") return nil } // triggerRunOnStateKeepsPrimary is the handler that is triggered when the prev/current state is different. -func (c *SAKURAController) triggerRunOnStateKeepsPrimary() error { +func (c *Controller) triggerRunOnStateKeepsPrimary() error { if c.writeTestDataFailCount >= writeTestDataFailCountThreshold { return fmt.Errorf("reached the maximum fail count of write test data") } if err := c.writeTestDataToMariaDB(); err != nil { c.writeTestDataFailCount++ - c.Logger.Warn("failed to write test data to mariadb", "error", err, "failedCount", c.writeTestDataFailCount) - } else { - // reset the count because the controller is healthy. - c.writeTestDataFailCount = 0 + c.logger.Warn("failed to write test data to mariadb", "error", err, "failedCount", c.writeTestDataFailCount) + // return noerror because this is soft fail + return nil } + // reset the count because the controller is healthy. + c.writeTestDataFailCount = 0 return nil } -// acceptTCP3306Traffic sets the rule that accepts the inbound communication. -func (c *SAKURAController) acceptTCP3306Traffic() error { - if err := c.nftablesConnector.FlushChain(nftables.BuiltinTableFilter, nftablesMariaDBChain); err != nil { +// acceptDatabaseServiceTraffic sets the rule that accepts the inbound communication. +func (c *Controller) acceptDatabaseServiceTraffic() error { + if err := c.nftablesConnector.FlushChain(c.dbAclChainName); err != nil { return err } acceptMatches := []nftables.Match{ - nftables.IFNameMatch(mariaDBServerDefaultIFName), - nftables.TCPDstPortMatch(mariaDBServerDefaultPort), + nftables.IFNameMatch(c.globalInterfaceName), + nftables.TCPDstPortMatch(uint16(c.dbServingPort)), } - if err := c.nftablesConnector.AddRule(nftables.BuiltinTableFilter, nftablesMariaDBChain, acceptMatches, nftables.AcceptStatement()); err != nil { + if err := c.nftablesConnector.AddRule(c.dbAclChainName, acceptMatches, nftables.AcceptStatement()); err != nil { return err } @@ -130,7 +122,7 @@ func (c *SAKURAController) acceptTCP3306Traffic() error { } // writeTestDataToMariaDB tries to write the testdata to MariaDB. -func (c *SAKURAController) writeTestDataToMariaDB() error { +func (c *Controller) writeTestDataToMariaDB() error { if err := c.createManagementDatabase(); err != nil { return err } @@ -144,32 +136,28 @@ func (c *SAKURAController) writeTestDataToMariaDB() error { return err } - if err := c.systemdConnector.CheckServiceStatus("mariadb"); err != nil { - return err - } - return nil } // createManagementDatabase tries to create the management database. // if the management database is already exist, the function does nothing. -func (c *SAKURAController) createManagementDatabase() error { +func (c *Controller) createManagementDatabase() error { return c.mariaDBConnector.CreateDatabase(managementDatabaseName) } // createManagementDatabase tries to create alive-check table on the management database. // if the alive-check table is already exist, the function does nothing. -func (c *SAKURAController) createAliveCheckTableOnManagementDB() error { +func (c *Controller) createAliveCheckTableOnManagementDB() error { return c.mariaDBConnector.CreateIDTable(managementDatabaseName, aliveCheckTableName) } // insertTemporaryRecordToAliveCheck tries to insert temporary record to alive-check table. -func (c *SAKURAController) insertTemporaryRecordToAliveCheck() error { +func (c *Controller) insertTemporaryRecordToAliveCheck() error { // id has no meaning. return c.mariaDBConnector.InsertIDRecord(managementDatabaseName, aliveCheckTableName, 1) } // deleteTemporaryRecordOnAliveCheck tries to delete records on alive-check table. -func (c *SAKURAController) deleteTemporaryRecordOnAliveCheck() error { +func (c *Controller) deleteTemporaryRecordOnAliveCheck() error { return c.mariaDBConnector.DeleteRecords(managementDatabaseName, aliveCheckTableName) } diff --git a/pkg/controller/sakura/primary_state_whitebox_test.go b/pkg/controller/primary_state_whitebox_test.go similarity index 69% rename from pkg/controller/sakura/primary_state_whitebox_test.go rename to pkg/controller/primary_state_whitebox_test.go index dea7cf9..bfc3feb 100644 --- a/pkg/controller/sakura/primary_state_whitebox_test.go +++ b/pkg/controller/primary_state_whitebox_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,46 +12,43 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sakura +package controller import ( - "os" "testing" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" "github.com/sakura-internet/distributed-mariadb-controller/pkg/mariadb" "github.com/stretchr/testify/assert" - "golang.org/x/exp/slog" ) func TestDecideNextStateOnPrimary_MariaDBIsUnhealthy(t *testing.T) { - ns := NewNeighborSet() + c := _newFakeController() + c.currentMariaDBHealth = dbHealthCheckResultNG - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnPrimary(logger, ns, MariaDBHealthCheckResultNG) - assert.Equal(t, controller.StateFault, nextState) + nextState := c.decideNextStateOnPrimary() + assert.Equal(t, StateFault, nextState) } func TestDecideNextStateOnPrimary_InDualPrimarySituation(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StatePrimary] = append(ns.NeighborMatrix[controller.StatePrimary], Neighbor{}) + c := _newFakeController() + c.currentNeighbors[StatePrimary] = []neighbor{""} + c.currentMariaDBHealth = dbHealthCheckResultOK - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnPrimary(logger, ns, MariaDBHealthCheckResultOK) - assert.Equal(t, controller.StateFault, nextState) + nextState := c.decideNextStateOnPrimary() + assert.Equal(t, StateFault, nextState) } func TestDecideNextStateOnPrimary_OKPath(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StateReplica] = append(ns.NeighborMatrix[controller.StateReplica], Neighbor{}) + c := _newFakeController() + c.currentNeighbors[StateReplica] = []neighbor{""} + c.currentMariaDBHealth = dbHealthCheckResultOK - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnPrimary(logger, ns, MariaDBHealthCheckResultOK) - assert.Equal(t, controller.StatePrimary, nextState) + nextState := c.decideNextStateOnPrimary() + assert.Equal(t, StatePrimary, nextState) } func TestTriggerRunOnStateKeepsPrimary_WriteTestDataFailPath(t *testing.T) { - c := _newFakeSAKURAController() + c := _newFakeController() // inject the mariadb connector that fails to write testdata. c.mariaDBConnector = mariadb.NewFakeMariaDBFailWriteTestDataConnector() @@ -63,7 +60,7 @@ func TestTriggerRunOnStateKeepsPrimary_WriteTestDataFailPath(t *testing.T) { } func TestTriggerRunOnStateKeepsPrimary_WriteTestDataFailedCountOversThreshold(t *testing.T) { - c := _newFakeSAKURAController() + c := _newFakeController() // inject the mariadb connector that fails to write testdata. c.mariaDBConnector = mariadb.NewFakeMariaDBFailWriteTestDataConnector() c.writeTestDataFailCount = writeTestDataFailCountThreshold @@ -73,7 +70,7 @@ func TestTriggerRunOnStateKeepsPrimary_WriteTestDataFailedCountOversThreshold(t } func TestTriggerRunOnStateChangesToPrimary_OKPath(t *testing.T) { - c := _newFakeSAKURAController() + c := _newFakeController() // for checking the triggerRunOnStateChangesToPrimary() resets this count to 0 c.writeTestDataFailCount = 5 diff --git a/pkg/controller/sakura/prometheus_metric.go b/pkg/controller/prometheus_metric.go similarity index 62% rename from pkg/controller/sakura/prometheus_metric.go rename to pkg/controller/prometheus_metric.go index d0cce58..402dc75 100644 --- a/pkg/controller/sakura/prometheus_metric.go +++ b/pkg/controller/prometheus_metric.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,27 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sakura +package controller import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" ) var ( - // DBControllerStateGaugeVec is the gauge-vec metric in prometheus + // dbControllerStateGaugeVec is the gauge-vec metric in prometheus // that holds the current state of the controller. - DBControllerStateGaugeVec = prometheus.NewGaugeVec( + dbControllerStateGaugeVec = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "edb_db_controller_state", Help: "the controller state of db-controller", }, []string{"state"}, ) - // DBControllerStateTransitionCounterVec is the counter-vec metric in prometheus + // dbControllerStateTransitionCounterVec is the counter-vec metric in prometheus // that holds the transition count of the controller. - DBControllerStateTransitionCounterVec = prometheus.NewCounterVec( + dbControllerStateTransitionCounterVec = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "edb_db_controller_state_transition_count", Help: "the counter of the controller state transition", @@ -42,11 +41,11 @@ var ( ) func init() { - DBControllerStateGaugeVec.WithLabelValues(string(controller.StateInitial)).Set(1) - DBControllerStateGaugeVec.WithLabelValues(string(controller.StateFault)).Set(0) - DBControllerStateGaugeVec.WithLabelValues(string(SAKURAControllerStateCandidate)).Set(0) - DBControllerStateGaugeVec.WithLabelValues(string(controller.StateReplica)).Set(0) - DBControllerStateGaugeVec.WithLabelValues(string(controller.StatePrimary)).Set(0) + dbControllerStateGaugeVec.WithLabelValues(string(StateInitial)).Set(1) + dbControllerStateGaugeVec.WithLabelValues(string(StateFault)).Set(0) + dbControllerStateGaugeVec.WithLabelValues(string(StateCandidate)).Set(0) + dbControllerStateGaugeVec.WithLabelValues(string(StateReplica)).Set(0) + dbControllerStateGaugeVec.WithLabelValues(string(StatePrimary)).Set(0) } func NewPrometheusMetricRegistry() *prometheus.Registry { @@ -58,8 +57,8 @@ func NewPrometheusMetricRegistry() *prometheus.Registry { collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), // db-controller - DBControllerStateGaugeVec, - DBControllerStateTransitionCounterVec, + dbControllerStateGaugeVec, + dbControllerStateTransitionCounterVec, ) return reg } diff --git a/pkg/controller/sakura/replica_state.go b/pkg/controller/replica_state.go similarity index 63% rename from pkg/controller/sakura/replica_state.go rename to pkg/controller/replica_state.go index db2ee8e..4763e6c 100644 --- a/pkg/controller/sakura/replica_state.go +++ b/pkg/controller/replica_state.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sakura +package controller import ( "fmt" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" "github.com/sakura-internet/distributed-mariadb-controller/pkg/mariadb" ) @@ -25,40 +24,35 @@ const ( // replicationStatusCheckThreshold is a threshold. // if the counter of the controller overs this, the controller goes panic. replicationStatusCheckThreshold = 20 - - mariaDBMasterDefaultUser = "repl" ) // decideNextStateOnReplica determines the next state on replica state. -func decideNextStateOnReplica( - neighbors *NeighborSet, - mariaDBHealth MariaDBHealthCheckResult, -) controller.State { - if mariaDBHealth == MariaDBHealthCheckResultNG { - return controller.StateFault +func (c *Controller) decideNextStateOnReplica() State { + if c.currentMariaDBHealth == dbHealthCheckResultNG { + return StateFault } - noPrimary := !neighbors.primaryNodeExists() - noCandidate := !neighbors.candidateNodeExists() + noPrimary := !c.currentNeighbors.primaryNodeExists() + noCandidate := !c.currentNeighbors.candidateNodeExists() if noPrimary && noCandidate { // you may be the next primary node! - return SAKURAControllerStateCandidate + return StateCandidate } - return controller.StateReplica + return StateReplica } // triggerRunOnStateChangesToReplica transition to replica state in main loop. -func (c *SAKURAController) triggerRunOnStateChangesToReplica() error { - // [STEP1]: START of setting MariaDB State. +func (c *Controller) triggerRunOnStateChangesToReplica() error { + // [STEP1]: setting MariaDB State. if err := c.startMariaDBService(); err != nil { return err } - if health := c.checkMariaDBHealth(); health == MariaDBHealthCheckResultNG { + if health := c.checkMariaDBHealth(); health == dbHealthCheckResultNG { return fmt.Errorf("MariaDB instance is down") } - if !c.CurrentNeighbors.primaryNodeExists() { + if !c.currentNeighbors.primaryNodeExists() { return fmt.Errorf("there is no primary neighbor in replica mode") } @@ -72,12 +66,12 @@ func (c *SAKURAController) triggerRunOnStateChangesToReplica() error { return err } - primaryNode := c.CurrentNeighbors.NeighborMatrix[controller.StatePrimary][0] + primaryNode := c.currentNeighbors[StatePrimary][0] master := mariadb.MasterInstance{ - Host: primaryNode.Address, - Port: c.MariaDBReplicaSourcePort, - User: mariaDBMasterDefaultUser, - Password: c.MariaDBReplicaPassword, + Host: string(primaryNode), + Port: c.dbReplicaSourcePort, + User: c.dbReplicaUserName, + Password: c.dbReplicaPassword, UseGTID: mariadb.MasterUseGTIDValueCurrentPos, } if err := c.mariaDBConnector.ChangeMasterTo(master); err != nil { @@ -87,28 +81,25 @@ func (c *SAKURAController) triggerRunOnStateChangesToReplica() error { if err := c.mariaDBConnector.StartReplica(); err != nil { return err } - // [STEP1]: END of setting MariaDB State. - // [STEP2]: START of setting Nftables State. - if err := c.rejectTCP3306TrafficFromExternal(); err != nil { + // [STEP2]: setting Nftables State. + if err := c.rejectDatabaseServiceTraffic(); err != nil { return err } - // [STEP2]: END of setting Nftables State. - // [STEP3]: START of configurating frrouting. + // [STEP3]: configurating frrouting. if err := c.advertiseSelfNetIFAddress(); err != nil { return err } - // [STEP3]: END of configurating frrouting. // reset the count because the controller is healthy for replica mode. c.replicationStatusCheckFailCount = 0 - c.Logger.Info("replica state handler succeed") + c.logger.Info("replica state handler succeed") return nil } -func (c *SAKURAController) triggerRunOnStateKeepsReplica() error { +func (c *Controller) triggerRunOnStateKeepsReplica() error { if c.replicationStatusCheckFailCount >= replicationStatusCheckThreshold { // we should manually operate the case for recovering. return fmt.Errorf("reached the maximum retry limit for replication") @@ -117,22 +108,25 @@ func (c *SAKURAController) triggerRunOnStateKeepsReplica() error { if err := c.checkMariaDBReplicationStatus(); err != nil { // we should keep trying to challenge that the replication status satisfies our conditions. c.replicationStatusCheckFailCount++ - c.Logger.Warn("failed to satisfy replication conditions", "error", err, "replicationCount", c.replicationStatusCheckFailCount) + c.logger.Warn("failed to satisfy replication conditions", "error", err, "replicationCount", c.replicationStatusCheckFailCount) if err := c.restartMariaDBReplica(); err != nil { - c.Logger.Warn("failed to restart replica", "error", err) + c.logger.Warn("failed to restart replica", "error", err) } - } else { - // reset the count because the controller is healthy. - c.replicationStatusCheckFailCount = 0 + + // return noerror because this is soft fail + return nil } + // reset the count because the controller is healthy. + c.replicationStatusCheckFailCount = 0 + return nil } // checkMariaDBReplicationStatus returns true if the status of replication is satisfied. // if the challenge failed to satisfy the conditions, this function returns false. -func (c *SAKURAController) checkMariaDBReplicationStatus() error { +func (c *Controller) checkMariaDBReplicationStatus() error { status, err := c.mariaDBConnector.ShowReplicationStatus() if err != nil { return err @@ -142,13 +136,11 @@ func (c *SAKURAController) checkMariaDBReplicationStatus() error { return fmt.Errorf("failed to satisfy the replication conditions") } - // lastNotifiedReplicationDelay := time.Unix(0, 0) - // c.validateReplicationDelaySeconds(replicaStatus, lastNotifiedReplicationDelay) return nil } // checkRequiredReplicationStatusIsOK checks the replication status satisfies the required conditions. -func (c *SAKURAController) checkRequiredReplicationStatusIsOK(status mariadb.ReplicationStatus) bool { +func (c *Controller) checkRequiredReplicationStatusIsOK(status mariadb.ReplicationStatus) bool { ioRunning, ok1 := status[mariadb.ReplicationStatusSlaveIORunning] sqlRunning, ok2 := status[mariadb.ReplicationStatusSlaveSQLRunning] @@ -156,26 +148,26 @@ func (c *SAKURAController) checkRequiredReplicationStatusIsOK(status mariadb.Rep msg := fmt.Sprintf("failed to retrieve %s or %s", mariadb.ReplicationStatusSlaveIORunning, mariadb.ReplicationStatusSlaveSQLRunning) - c.Logger.Debug(msg) + c.logger.Debug(msg) return false } if ioRunning != mariadb.ReplicationStatusSlaveIORunningYes { msg := fmt.Sprintf("unexpected %s status", mariadb.ReplicationStatusSlaveIORunning) - c.Logger.Debug(msg, "expected", "Yes", "actual", ioRunning) + c.logger.Debug(msg, "expected", "Yes", "actual", ioRunning) return false } if sqlRunning != mariadb.ReplicationStatusSlaveSQLRunningYes { msg := fmt.Sprintf("unexpected %s status", mariadb.ReplicationStatusSlaveSQLRunning) - c.Logger.Debug(msg, "expected", "Yes", "actual", sqlRunning) + c.logger.Debug(msg, "expected", "Yes", "actual", sqlRunning) return false } return true } -func (c *SAKURAController) restartMariaDBReplica() error { +func (c *Controller) restartMariaDBReplica() error { if err := c.mariaDBConnector.StopReplica(); err != nil { return err } diff --git a/pkg/controller/sakura/replica_state_whitebox_test.go b/pkg/controller/replica_state_whitebox_test.go similarity index 75% rename from pkg/controller/sakura/replica_state_whitebox_test.go rename to pkg/controller/replica_state_whitebox_test.go index 1f44f29..0e85e26 100644 --- a/pkg/controller/sakura/replica_state_whitebox_test.go +++ b/pkg/controller/replica_state_whitebox_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sakura +package controller import ( - "fmt" "testing" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" "github.com/sakura-internet/distributed-mariadb-controller/pkg/frrouting/bgpd" "github.com/sakura-internet/distributed-mariadb-controller/pkg/mariadb" "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" @@ -27,39 +25,41 @@ import ( ) func TestDecideNextStateOnReplica_MariaDBIsUnhealthy(t *testing.T) { - ns := NewNeighborSet() - nextState := decideNextStateOnReplica(ns, MariaDBHealthCheckResultNG) - assert.Equal(t, controller.StateFault, nextState) + c := _newFakeController() + c.currentMariaDBHealth = dbHealthCheckResultNG + + nextState := c.decideNextStateOnReplica() + assert.Equal(t, StateFault, nextState) } func TestDecisionNextState_OnReplica_RemainReplica(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StatePrimary] = append(ns.NeighborMatrix[controller.StatePrimary], Neighbor{}) + c := _newFakeController() + c.currentNeighbors[StatePrimary] = []neighbor{""} + c.currentMariaDBHealth = dbHealthCheckResultOK - nextState := decideNextStateOnReplica(ns, MariaDBHealthCheckResultOK) - assert.Equal(t, controller.StateReplica, nextState) + nextState := c.decideNextStateOnReplica() + assert.Equal(t, StateReplica, nextState) } func TestDecisionNextState_OnReplica_NoOnePrimaryAndCandidate(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StateFault] = append(ns.NeighborMatrix[controller.StateFault], Neighbor{}) + c := _newFakeController() + c.currentNeighbors[StateFault] = []neighbor{""} + c.currentMariaDBHealth = dbHealthCheckResultOK - nextState := decideNextStateOnReplica(ns, MariaDBHealthCheckResultOK) - assert.Equal(t, SAKURAControllerStateCandidate, nextState) + nextState := c.decideNextStateOnReplica() + assert.Equal(t, StateCandidate, nextState) } func TestTriggerRunOnStateChangesToReplica_OKPath(t *testing.T) { - c := _newFakeSAKURAController() + c := _newFakeController() // for checking the triggerRunOnStateChangesToReplica() resets this count to 0 c.replicationStatusCheckFailCount = 5 - primaryNeighbor := Neighbor{ - Address: "10.0.0.2", - } - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StatePrimary] = append(ns.NeighborMatrix[controller.StatePrimary], primaryNeighbor) - c.CurrentNeighbors = ns + primaryNeighbor := neighbor("10.0.0.2") + ns := newNeighborSet() + ns[StatePrimary] = append(ns[StatePrimary], primaryNeighbor) + c.currentNeighbors = ns err := c.triggerRunOnStateChangesToReplica() assert.NoError(t, err) @@ -67,9 +67,9 @@ func TestTriggerRunOnStateChangesToReplica_OKPath(t *testing.T) { // test with MariaDB Connector fakeMariaDBConn := c.mariaDBConnector.(*mariadb.FakeMariaDBConnector) t.Run("TestTriggerRunOnStateChangesToReplica_OKPath_shouldResetReplicationStatusCheckCount", _shouldResetReplicationStatusCheckCount(c)) - t.Run("TestTriggerRunOnStateChangesToReplica_OKPath_mustTurnOnMariaDBReadOnlyVariable", _mustTurnOnMariaDBReadOnlyVairable(fakeMariaDBConn)) + t.Run("TestTriggerRunOnStateChangesToReplica_OKPath_mustTurnOnMariaDBReadOnlyVariable", _mustTurnOnMariaDBReadOnlyVariable(fakeMariaDBConn)) t.Run("TestTriggerRunOnStateChangesToReplica_OKPath_shouldBeCorrectReplicationCommandsExecutionOrder", _shouldBeCorrectReplicationCommandsExecutionOrder(fakeMariaDBConn)) - t.Run("TestTriggerRunOnStateChangesToReplica_OKPath_mustCallChangeMasterToWithCorrectArgs", _mustCallChangeMasterToWithCorrectArgs(fakeMariaDBConn, primaryNeighbor.Address, "dummy-db-replica-password")) + t.Run("TestTriggerRunOnStateChangesToReplica_OKPath_mustCallChangeMasterToWithCorrectArgs", _mustCallChangeMasterToWithCorrectArgs(fakeMariaDBConn, string(primaryNeighbor), "dummy-db-replica-password")) // test with Nftables Connector fakeNftablesConn := c.nftablesConnector.(*nftables.FakeNftablesConnector) @@ -81,14 +81,14 @@ func TestTriggerRunOnStateChangesToReplica_OKPath(t *testing.T) { } func TestTriggerRunOnStateKeepsReplica_CheckReplicationStatusFailPath(t *testing.T) { - c := _newFakeSAKURAController() + c := _newFakeController() // inject the mariadb connector that fails to check replication status. c.mariaDBConnector = mariadb.NewFakeMariaDBFailedReplicationConnector() { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StatePrimary] = append(ns.NeighborMatrix[controller.StatePrimary], Neighbor{}) - c.CurrentNeighbors = ns + ns := newNeighborSet() + ns[StatePrimary] = []neighbor{""} + c.currentNeighbors = ns } err := c.triggerRunOnStateKeepsReplica() assert.NoError(t, err) @@ -103,22 +103,22 @@ func TestTriggerRunOnStateKeepsReplica_CheckReplicationStatusFailPath(t *testing } func TestTriggerRunOnStateKeepsReplica_ReplicationStatusCheckCountOversThreshold(t *testing.T) { - c := _newFakeSAKURAController() + c := _newFakeController() // inject the mariadb connector that fails to check replication status. c.mariaDBConnector = mariadb.NewFakeMariaDBFailedReplicationConnector() c.replicationStatusCheckFailCount = replicationStatusCheckThreshold { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StatePrimary] = append(ns.NeighborMatrix[controller.StatePrimary], Neighbor{}) - c.CurrentNeighbors = ns + ns := newNeighborSet() + ns[StatePrimary] = []neighbor{""} + c.currentNeighbors = ns } err := c.triggerRunOnStateKeepsReplica() assert.Error(t, err) } -func _shouldResetReplicationStatusCheckCount(c *SAKURAController) func(*testing.T) { +func _shouldResetReplicationStatusCheckCount(c *Controller) func(*testing.T) { return func(t *testing.T) { t.Parallel() @@ -127,14 +127,14 @@ func _shouldResetReplicationStatusCheckCount(c *SAKURAController) func(*testing. } } -func _mustTurnOnMariaDBReadOnlyVairable( +func _mustTurnOnMariaDBReadOnlyVariable( conn *mariadb.FakeMariaDBConnector, ) func(*testing.T) { return func(t *testing.T) { t.Parallel() // check whether the controller turn on the read_only variable - _, ok := conn.Timestamp[fmt.Sprintf("TurnOnBoolVariable(%s)", mariadb.ReadOnlyVariableName)] + _, ok := conn.Timestamp["TurnOnReadOnly"] assert.True(t, ok) } } @@ -164,8 +164,9 @@ func _mustCallChangeMasterToWithCorrectArgs( // check whether the ChangeMasterTo() is called with the properties of a primary neighbor. // FakeMariaDBConnector holds the argument of ChangeMasterTo() to .MasterConfig directly. + expectedReplicaUserName := "repl" assert.Equal(t, expectedPrimaryAddress, conn.MasterConfig.Host) - assert.Equal(t, mariaDBMasterDefaultUser, conn.MasterConfig.User) + assert.Equal(t, expectedReplicaUserName, conn.MasterConfig.User) assert.Equal(t, expectedPassword, conn.MasterConfig.Password) assert.Equal(t, mariadb.MasterUseGTIDValueCurrentPos, conn.MasterConfig.UseGTID) } diff --git a/pkg/controller/sakura/candidate_state.go b/pkg/controller/sakura/candidate_state.go deleted file mode 100644 index 1d33a96..0000000 --- a/pkg/controller/sakura/candidate_state.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sakura - -import ( - "fmt" - - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" - "golang.org/x/exp/slog" -) - -// decideNextStateOnCandidate determines the next state on candidate state. -func decideNextStateOnCandidate( - logger *slog.Logger, - neighbors *NeighborSet, - mariaDBHealth MariaDBHealthCheckResult, - readyToPrimaryJudge ReadyToPrimaryJudge, -) controller.State { - if mariaDBHealth == MariaDBHealthCheckResultNG { - logger.Warn("MariaDB is down. falling back to fault state.") - return controller.StateFault - } - - if neighbors.candidateNodeExists() || neighbors.primaryNodeExists() { - logger.Info("another candidate or primary exists. falling back to fault state.") - return controller.StateFault - } - - if readyToPrimaryJudge == ReadytoPrimaryJudgeOK { - return controller.StatePrimary - } - - logger.Info("I'm not ready to primary. staying candidate state.") - return SAKURAControllerStateCandidate -} - -// triggerRunOnStateChangesToCandidate transition to candidate in main loop. -func (c *SAKURAController) triggerRunOnStateChangesToCandidate() error { - // [STEP1]: START of setting MariaDB State. - if err := c.startMariaDBService(); err != nil { - return err - } - if health := c.checkMariaDBHealth(); health == MariaDBHealthCheckResultNG { - return fmt.Errorf("MariaDB instance is down") - } - - if err := c.syncReadOnlyVariable( /* read_only=1 */ true); err != nil { - return err - } - - // [STEP1]: END of setting MariaDB State. - - // [STEP2]: START of setting Nftables State. - if err := c.rejectTCP3306TrafficFromExternal(); err != nil { - return err - } - // [STEP2]: END of setting Nftables State. - - // [STEP3]: START of configurating frrouting. - if err := c.advertiseSelfNetIFAddress(); err != nil { - return err - } - // [STEP3]: END of configurating frrouting. - - c.Logger.Info("candidate state handler succeed") - return nil -} diff --git a/pkg/controller/sakura/candidate_state_whitebox_test.go b/pkg/controller/sakura/candidate_state_whitebox_test.go deleted file mode 100644 index 9981fa6..0000000 --- a/pkg/controller/sakura/candidate_state_whitebox_test.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sakura - -import ( - "os" - "testing" - - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" - "github.com/stretchr/testify/assert" - "golang.org/x/exp/slog" -) - -func TestDecideNextStateOnCandidate_MariaDBIsUnhealthy(t *testing.T) { - ns := NewNeighborSet() - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnCandidate(logger, ns, MariaDBHealthCheckResultNG, ReadytoPrimaryJudgeNG) - assert.Equal(t, controller.StateFault, nextState) -} - -func TestDecideNextStateOnCandidate_InMultiCandidateSituation(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[SAKURAControllerStateCandidate] = append(ns.NeighborMatrix[SAKURAControllerStateCandidate], Neighbor{}) - - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnCandidate(logger, ns, MariaDBHealthCheckResultOK, ReadytoPrimaryJudgeNG) - assert.Equal(t, controller.StateFault, nextState) -} - -func TestDecideNextStateOnCandidate_PrimaryIsAlreadyExist(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StatePrimary] = append(ns.NeighborMatrix[SAKURAControllerStateCandidate], Neighbor{}) - - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnCandidate(logger, ns, MariaDBHealthCheckResultOK, ReadytoPrimaryJudgeNG) - assert.Equal(t, controller.StateFault, nextState) -} - -func TestDecideNextStateOnCandidate_ToBePromotedToPrimary(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StateFault] = append(ns.NeighborMatrix[controller.StateFault], Neighbor{}) - - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnCandidate(logger, ns, MariaDBHealthCheckResultOK, ReadytoPrimaryJudgeOK) - assert.Equal(t, controller.StatePrimary, nextState) -} - -func TestDecideNextStateCandidate_RemainCandidate(t *testing.T) { - ns := NewNeighborSet() - ns.NeighborMatrix[controller.StateFault] = append(ns.NeighborMatrix[controller.StateFault], Neighbor{}) - - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - nextState := decideNextStateOnCandidate(logger, ns, MariaDBHealthCheckResultOK, ReadytoPrimaryJudgeNG) - assert.Equal(t, SAKURAControllerStateCandidate, nextState) -} - -func TestTriggerRunOnStateChangesToCandidate_OKPath(t *testing.T) { - c := _newFakeSAKURAController() - - err := c.triggerRunOnStateChangesToCandidate() - assert.NoError(t, err) -} diff --git a/pkg/controller/sakura/controller.go b/pkg/controller/sakura/controller.go deleted file mode 100644 index a8df15d..0000000 --- a/pkg/controller/sakura/controller.go +++ /dev/null @@ -1,534 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sakura - -import ( - "math/rand" - "net" - "os" - "sync" - "time" - - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/frrouting/bgpd" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/frrouting/vtysh" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/mariadb" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/process" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/systemd" - "golang.org/x/exp/slog" -) - -const ( - // SAKURAControllerStateCandidate is the controller state that is ready to be promoted to Primary state. - SAKURAControllerStateCandidate controller.State = "candidate" - // SAKURAControllerStateAnchor indicates the controller is not exist and the node is under the anchor mode. - // The State is not used in db-controller. - SAKURAControllerStateAnchor controller.State = "anchor" - mariaDBServerDefaultIFName = "eth0" - mariaDBServerDefaultPort = 3306 - nftablesMariaDBChain = "mariadb" - mariaDBSerivceName = "mariadb" -) - -var ( - SAKURAControllerAllStates = map[controller.State]bool{ - controller.StateInitial: true, - controller.StateFault: true, - controller.StatePrimary: true, - SAKURAControllerStateCandidate: true, - controller.StateReplica: true, - } -) - -type SAKURAController struct { - Logger *slog.Logger - // currentState is the current state of the controller. - // for prevending unexpected transition, the state isn't exposed. - currentState controller.State - // prevState is the previous state of the controller. - prevState controller.State - // m is a read-write-mutex that is used for sharing controller's state btw controller/http-api goroutines. - m sync.RWMutex - // HostAddress is an IP address of the eth0. - HostAddress string - // DBReplicaSourcePort is the port of primary as replication source. - MariaDBReplicaSourcePort int - // MariaDBReplicaPassword is credential used by replica to establish replication link for primary - MariaDBReplicaPassword string - // replicationStatusCheckFailCount is a counter of the MariaDB's replication status checker in replica state. - replicationStatusCheckFailCount uint - // writeTestDataFailCount is a counter that the controller tries to write test data to MariaDB. - // if the count overs the pre-declared threshold, the controller urgently exits. - writeTestDataFailCount uint - // CurrentNeighbors holds the current BGP neighbors of the dbserver. - // that discovered in each loop of the controller. - CurrentNeighbors *NeighborSet - // CurrentMariaDBHealth holds the most recent healthcheck's result. - CurrentMariaDBHealth MariaDBHealthCheckResult - // ReadyToPrimary - ReadyToPrimary ReadyToPrimaryJudge - - // nftablesConnector communicates with FRRouting BGPd via vtysh. - nftablesConnector nftables.Connector - // bgpdConnector communicates with FRRouting BGPd via vtysh. - bgpdConnector bgpd.BGPdConnector - // processControlConnector manages the linux process. - processControlConnector process.ProcessControlConnector - // systemdConnector manages the systemd services. - systemdConnector systemd.Connector - // mariaDBConnector communicates with MariaDB via mysql-client. - mariaDBConnector mariadb.Connector -} - -// for guarding that the sakura controller implements -var _ controller.Controller = &SAKURAController{} - -// GetState implements controller.Controller -func (c *SAKURAController) GetState() controller.State { - c.m.RLock() - defer c.m.RUnlock() - - return c.currentState -} - -// DecideNextState implements controller.Controller -func (c *SAKURAController) DecideNextState() controller.State { - c.Logger.Debug("decide next state", "current state", c.GetState()) - if networkIsParted(c.CurrentNeighbors) { - c.Logger.Info("detected network partition", "neighbors", c.CurrentNeighbors.NeighborAddresses()) - return controller.StateFault - } - - switch c.GetState() { - case controller.StateFault: - return decideNextStateOnFault(c.Logger, c.CurrentNeighbors) - case SAKURAControllerStateCandidate: - return decideNextStateOnCandidate( - c.Logger, - c.CurrentNeighbors, - c.CurrentMariaDBHealth, - c.ReadyToPrimary, - ) - case controller.StatePrimary: - return decideNextStateOnPrimary( - c.Logger, - c.CurrentNeighbors, - c.CurrentMariaDBHealth, - ) - case controller.StateReplica: - return decideNextStateOnReplica( - c.CurrentNeighbors, - c.CurrentMariaDBHealth, - ) - case controller.StateInitial: - // just initialized controller take this case. - return controller.StateFault - default: - panic("unreachable") - } -} - -// OnExit implements controller.Controller -func (c *SAKURAController) OnExit() error { - c.SetState(controller.StateFault) - if err := c.triggerRunOnStateChangesToFault(); err != nil { - c.Logger.Info("failed to TriggerRunOnStateChanges while going to fault. Ignore errors.", err) - } - - return nil -} - -// OnStateHandler implements controller.Controller -func (c *SAKURAController) OnStateHandler(nextState controller.State) error { - time.Sleep(time.Second * time.Duration(rand.Intn(2)+1)) - - if cannotTransitionTo(c.GetState(), nextState) { - panic("unreachable controller state was picked") - } - c.SetState(nextState) - - if c.keepStateInPrevTransition() { - if err := c.triggerRunOnStateKeeps(); err != nil { - c.Logger.Error("failed to triggerRunOnStateKeeps. transition to fault state and exit", err, "state", string(c.GetState())) - c.forceTransitionToFault() - panic("urgently exit") - } - - return nil - } - - if err := c.triggerRunOnStateChanges(); err != nil { - // we urgently transition to fault state - c.Logger.Error("failed to TriggerRunOnStateChanges. transition to fault state.", err, "state", string(c.GetState())) - c.forceTransitionToFault() - } - - return nil -} - -// PreDecideNextStateHandler implements controller.Controller -func (c *SAKURAController) PreDecideNextStateHandler() error { - prevNeighbors := c.CurrentNeighbors - prefixes, err := c.collectStateCommunityRoutePrefixes() - if err != nil { - // we urgently transition to fault state - c.Logger.Error("failed to collect BGP routes", err, "state", c.GetState()) - c.forceTransitionToFault() - - return nil - } - - c.CurrentNeighbors = c.extractNeighborAddresses(prefixes) - // to avoiding unnecessary calculation, we checks the logger's level. - if c.Logger.Enabled(slog.LevelInfo) { - - if prevNeighbors.Different(c.CurrentNeighbors) { - addrs := c.CurrentNeighbors.NeighborAddresses() - c.Logger.Info("neighbor set is updated", "addresses", addrs) - } - } - - c.CurrentMariaDBHealth = c.checkMariaDBHealth() - c.ReadyToPrimary = c.readyToBePromotedToPrimary() - return nil -} - -// SetState sets the given state as the current state of the controller. -func (c *SAKURAController) SetState(nextState controller.State) { - c.prevState = c.GetState() - { - c.m.Lock() - c.currentState = nextState - c.m.Unlock() - } - - curState := c.GetState() - if c.prevState == curState { - c.Logger.Debug("controller transitions the state", "from", c.prevState, "to", curState) - } else { - c.Logger.Info("controller transitions the state", "from", c.prevState, "to", curState) - } - - // modify state metric(s) - DBControllerStateTransitionCounterVec.WithLabelValues(string(curState)).Inc() - - DBControllerStateGaugeVec.WithLabelValues(string(curState)).Set(1) - - for s := range SAKURAControllerAllStates { - if s == curState { - continue - } - - DBControllerStateGaugeVec.WithLabelValues(string(s)).Set(0) - } -} - -func NewSAKURAController(logger *slog.Logger, configs ...ControllerConfig) *SAKURAController { - c := &SAKURAController{ - currentState: controller.StateInitial, - Logger: logger, - CurrentNeighbors: NewNeighborSet(), - nftablesConnector: nftables.NewDefaultConnector(logger), - bgpdConnector: vtysh.NewDefaultBGPdConnector(logger), - processControlConnector: process.NewDefaultConnector(logger), - mariaDBConnector: mariadb.NewDefaultConnector(logger), - systemdConnector: systemd.NewDefaultConnector(logger), - } - - for _, cfg := range configs { - cfg(c) - } - return c -} - -// triggerRunOnStateChanges triggers the state handler if the previous state is not the current state. -func (c *SAKURAController) triggerRunOnStateChanges() error { - switch c.GetState() { - case controller.StatePrimary: - if err := c.triggerRunOnStateChangesToPrimary(); err != nil { - return err - } - case controller.StateFault: - if err := c.triggerRunOnStateChangesToFault(); err != nil { - return err - } - case SAKURAControllerStateCandidate: - if err := c.triggerRunOnStateChangesToCandidate(); err != nil { - return err - } - case controller.StateReplica: - if err := c.triggerRunOnStateChangesToReplica(); err != nil { - return err - } - case SAKURAControllerStateAnchor: - panic("unreachable") - } - - return nil -} - -// triggerRunOnStateKeeps triggers the state handler if the previous state is same as the current state. -func (c *SAKURAController) triggerRunOnStateKeeps() error { - switch c.GetState() { - case controller.StatePrimary: - if err := c.triggerRunOnStateKeepsPrimary(); err != nil { - return err - } - case controller.StateReplica: - if err := c.triggerRunOnStateKeepsReplica(); err != nil { - return err - } - } - - return nil -} - -// advertiseSelfNetIFAddress updates the configuration of the advertising route. -// the BGP community of the advertising route will be updated with the current controller-state. -func (c *SAKURAController) advertiseSelfNetIFAddress() error { - _, selfAddr, err := net.ParseCIDR(c.HostAddress + "/32") - if err != nil { - return err - } - return c.bgpdConnector.ConfigureRouteWithRouteMap(*selfAddr, string(c.GetState())) -} - -// forceTransitionToFault set state to fault and triggers fault handler -func (c *SAKURAController) forceTransitionToFault() { - c.SetState(controller.StateFault) - if err := c.triggerRunOnStateChanges(); err != nil { - c.Logger.Info("failed to TriggerRunOnStateChanges while going to fault. Ignore errors.", err) - } -} - -// keepStateInPrevTransition determins wheather a state transition has occurred -func (c *SAKURAController) keepStateInPrevTransition() bool { - prev := c.getPreviousState() - cur := c.GetState() - return prev == cur -} - -// getPreviousState returns the controller's previous state. -func (c *SAKURAController) getPreviousState() controller.State { - c.m.RLock() - defer c.m.RUnlock() - - return c.prevState -} - -// MariaDBHealthCheckResult is the result of the mariadb's healthcheck -type MariaDBHealthCheckResult uint - -const ( - MariaDBHealthCheckResultOK MariaDBHealthCheckResult = iota - MariaDBHealthCheckResultNG -) - -// checkMariaDBHealth checks whether the MariaDB server is healthy or not. -func (c *SAKURAController) checkMariaDBHealth() MariaDBHealthCheckResult { - if err := c.systemdConnector.CheckServiceStatus(mariaDBSerivceName); err != nil { - c.Logger.Debug("'systemctl status mariadb' exit with returning error", "error", err) - return MariaDBHealthCheckResultNG - } - - return MariaDBHealthCheckResultOK -} - -// ReadyToPrimaryJudge is the result of the judgement to be promoted to primary state. -type ReadyToPrimaryJudge uint - -const ( - // ReadytoPrimaryJudgeOK is OK for being promoted to primary state - ReadytoPrimaryJudgeOK ReadyToPrimaryJudge = iota - // ReadytoPrimaryJudgeNG is NG for being promoted to primary state - ReadytoPrimaryJudgeNG -) - -// readyToBePromotedToPrimary returns true when the controller satisfies the conditions to be promoted to primary state. -func (c *SAKURAController) readyToBePromotedToPrimary() ReadyToPrimaryJudge { - status, err := c.mariaDBConnector.ShowReplicationStatus() - if err != nil { - c.Logger.Debug("failed to show replication status", "error", err) - return ReadytoPrimaryJudgeNG - } - - readMasterLogPos, ok := status[mariadb.ReplicationStatusReadMasterLogPos] - if !ok { - return ReadytoPrimaryJudgeOK - } - - if readMasterLogPos == status[mariadb.ReplicationStatusExecMasterLogPos] && - status[mariadb.ReplicationStatusMasterLogFile] == status[mariadb.ReplicationStatusRelayMasterLogFile] { - return ReadytoPrimaryJudgeOK - } - - return ReadytoPrimaryJudgeNG -} - -// CollectStateCommunityRoutePrefixes collects the BGP route-prefix that they have a community of a controller-state. -func (c *SAKURAController) collectStateCommunityRoutePrefixes() (map[controller.State][]net.IP, error) { - routes := make(map[controller.State][]net.IP) - - // StateInitial is not needed in the below slice because the state doesn't advertise any routes. - states := []controller.State{ - SAKURAControllerStateCandidate, - controller.StateFault, - controller.StatePrimary, - controller.StateReplica, - SAKURAControllerStateAnchor, - } - - for _, state := range states { - if routes[state] == nil { - routes[state] = make([]net.IP, 0) - } - - bgp, err := c.bgpdConnector.ShowRoutesWithBGPCommunityList(string(state)) - if err != nil { - return nil, err - } - - for routePrefix := range bgp.Routes { - // NOTE: we recommend you use net/netip instead of net package - // because the netip.Addr is the most prefered way to present an IP address in Go. - // but the net/netip package doesn't have the way to parse CIDR notation. - addr, _, err := net.ParseCIDR(routePrefix) - if err != nil { - c.Logger.Error("failed to parse route prefix", err) - } - - routes[state] = append(routes[state], addr) - } - - } - - return routes, nil -} - -// ExtractNeighborAddresses get only the addresses of the neighbors from the given prefixes. -func (c *SAKURAController) extractNeighborAddresses( - prefixMatrix map[controller.State][]net.IP, -) *NeighborSet { - neighbors := NewNeighborSet() - - for state, prefixes := range prefixMatrix { - - for _, prefix := range prefixes { - - // each prefix of the advertised BGP route is the unicast address of other DB instances. - // if the route prefix(unicast IP) and my address are same, - // the route is advertised from me so it should be ignored. - if prefix.String() == c.HostAddress { - continue - } - - if neighbors.NeighborMatrix[state] == nil { - neighbors.NeighborMatrix[state] = make([]Neighbor, 0) - } - - neighbors.NeighborMatrix[state] = append( - neighbors.NeighborMatrix[state], - Neighbor{ - Address: prefix.String(), - }, - ) - } - } - - return neighbors -} - -// syncReadOnlyVariable updates the read_only variable to the given expected value. -// if the current value equals the given value, the variable is already synced. -// otherwise, the function tries to sync the variable. -func (c *SAKURAController) syncReadOnlyVariable(readOnlyToBeTrue bool) error { - isOn := c.mariaDBConnector.CheckBoolVariableIsON(mariadb.ReadOnlyVariableName) - - if readOnlyToBeTrue == isOn { - // the variable is already the expected value. - // nothing to do. - return nil - } - - if readOnlyToBeTrue { - return c.mariaDBConnector.TurnOnBoolVariable(mariadb.ReadOnlyVariableName) - } - - return c.mariaDBConnector.TurnOffBoolVariable(mariadb.ReadOnlyVariableName) -} - -// startMariaDBService starts the mariadb service of systemd. -func (c *SAKURAController) startMariaDBService() error { - const ( - mysqlMasterInfoFilePath = "/var/lib/mysql/master.info" - mysqllRelayInfoFilePath = "/var/lib/mysql/relay-log.info" - ) - - preHook := func() error { - if err := os.RemoveAll(mysqlMasterInfoFilePath); err != nil { - return err - } - - if err := os.RemoveAll(mysqllRelayInfoFilePath); err != nil { - return err - } - - return nil - } - - if err := c.systemdConnector.StartService(mariaDBSerivceName, preHook, nil); err != nil { - return err - } - - return nil -} - -// cannotTransitionTo checks whether the state machine doesn't have the edge from current to next. -func cannotTransitionTo( - currentState controller.State, - nextState controller.State, -) bool { - switch currentState { - case controller.StateFault: - return nextState == controller.StatePrimary - case SAKURAControllerStateCandidate: - return nextState == controller.StateReplica - case controller.StatePrimary: - return nextState == SAKURAControllerStateCandidate || nextState == controller.StateReplica - case controller.StateReplica: - return nextState == controller.StatePrimary - case controller.StateInitial: - return nextState != controller.StateFault - default: - // unreachable - return true - } -} - -// networkIsParted returns true if there is no neighbor on the network. -func networkIsParted( - neighbors *NeighborSet, -) bool { - if neighbors.primaryNodeExists() || - neighbors.candidateNodeExists() || - neighbors.replicaNodeExists() || - neighbors.faultNodeExists() || - neighbors.anchorNodeExists() { - return false - } - - return true -} diff --git a/pkg/controller/sakura/controller_config.go b/pkg/controller/sakura/controller_config.go deleted file mode 100644 index 3ab2b02..0000000 --- a/pkg/controller/sakura/controller_config.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sakura - -import ( - "github.com/sakura-internet/distributed-mariadb-controller/pkg/frrouting/bgpd" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/mariadb" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/process" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/systemd" -) - -// ControllerConfig is the configuration that is applied into SAKURAController. -type ControllerConfig func(c *SAKURAController) - -// SystemdConnector generates a config that sets the systemd.Connector into SAKURAController. -func SystemdConnector(connector systemd.Connector) ControllerConfig { - return func(c *SAKURAController) { - c.systemdConnector = connector - } -} - -func MariaDBConnector(connector mariadb.Connector) ControllerConfig { - return func(c *SAKURAController) { - c.mariaDBConnector = connector - } -} - -// NftablesConnector generates a config that sets the nftables.Connector into SAKURAController. -func NftablesConnector(connector nftables.Connector) ControllerConfig { - return func(c *SAKURAController) { - c.nftablesConnector = connector - } -} - -// BGPdConnector generates a config that sets the vtysh.BGPdConnector into SAKURAController. -func BGPdConnector(connector bgpd.BGPdConnector) ControllerConfig { - return func(c *SAKURAController) { - c.bgpdConnector = connector - } -} - -// ProcessControlConnector generates a config that sets the process.ProcessControlConnector into SAKURAController. -func ProcessControlConnector(connector process.ProcessControlConnector) ControllerConfig { - return func(c *SAKURAController) { - c.processControlConnector = connector - } -} diff --git a/pkg/controller/sakura/fault_state.go b/pkg/controller/sakura/fault_state.go deleted file mode 100644 index 9184d08..0000000 --- a/pkg/controller/sakura/fault_state.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sakura - -import ( - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" - "golang.org/x/exp/slog" -) - -const ( - MariaDBDaemonProcessPath = "/usr/sbin/mariadbd" -) - -// decideNextStateOnFault determines the next state on fault state -func decideNextStateOnFault( - logger *slog.Logger, - neighbors *NeighborSet, -) controller.State { - if neighbors.primaryNodeExists() { - return controller.StateReplica - } - - if neighbors.candidateNodeExists() || neighbors.replicaNodeExists() { - logger.Info("another candidate or replica exists") - return controller.StateFault - } - - // the fault controller is ready to transition to candidate state - // because network reachability is ok and no one candidate is here. - return SAKURAControllerStateCandidate -} - -// triggerRunOnStateChangesToFault transition to fault state in main loop. -// In fault state, the controller just reflect the fault state to external resources. -func (c *SAKURAController) triggerRunOnStateChangesToFault() error { - // [STEP1]: START of configurating frrouting - if err := c.advertiseSelfNetIFAddress(); err != nil { - c.Logger.Warn("failed to advertise self-address in BGP but ignored because i'm fault", "error", err) - } - // [STEP1]: END of configurating frrouting - - // [STEP2]: START of setting nftables state - if err := c.rejectTCP3306TrafficFromExternal(); err != nil { - c.Logger.Warn("failed to reject tcp traffic to 3306 but ignored because i'm fault", "error", err) - } - // [STEP2]: END of setting nftables state - - // [STEP3]: START of setting MariaDB state - if err := c.processControlConnector.KillProcessWithFullName(MariaDBDaemonProcessPath); err != nil { - c.Logger.Warn("failed to kill mariadb daemon process but ignored because i'm fault", "error", err) - } - if err := c.stopMariaDBService(); err != nil { - c.Logger.Warn("failed to stop systemd mariadb process but ignored because i'm fault", "error", err) - } - // [STEP3]: END of setting MariaDB state - - c.Logger.Info("fault state handler succeed") - return nil -} - -// rejectTCP3306TrafficFromExternal sets the reject rule that denies the inbound communication from the outsider of the network. -func (c *SAKURAController) rejectTCP3306TrafficFromExternal() error { - if err := c.nftablesConnector.FlushChain( - nftables.BuiltinTableFilter, - nftablesMariaDBChain, - ); err != nil { - return err - } - - rejectMatches := []nftables.Match{ - nftables.IFNameMatch(mariaDBServerDefaultIFName), - nftables.TCPDstPortMatch(mariaDBServerDefaultPort), - } - if err := c.nftablesConnector.AddRule( - nftables.BuiltinTableFilter, - nftablesMariaDBChain, - rejectMatches, - nftables.RejectStatement(), - ); err != nil { - return err - } - - return nil -} - -// stopMariaDBService stops the mariadb's systemd service. -func (c *SAKURAController) stopMariaDBService() error { - if err := c.systemdConnector.StopService(mariaDBSerivceName); err != nil { - return err - } - - return nil -} diff --git a/pkg/controller/sakura/neighbor.go b/pkg/controller/sakura/neighbor.go deleted file mode 100644 index 77278ec..0000000 --- a/pkg/controller/sakura/neighbor.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sakura - -import ( - "fmt" - "reflect" - "strings" - - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" -) - -// NeighborSet holds the set of the BGP neighbors. -type NeighborSet struct { - // Neighbors is the set of the BGP neighbors. - NeighborMatrix map[controller.State][]Neighbor -} - -// NewNeighborSet initializes the empty NeighborSet. -func NewNeighborSet() *NeighborSet { - return &NeighborSet{ - NeighborMatrix: map[controller.State][]Neighbor{ - controller.StateFault: make([]Neighbor, 0), - SAKURAControllerStateCandidate: make([]Neighbor, 0), - controller.StatePrimary: make([]Neighbor, 0), - controller.StateReplica: make([]Neighbor, 0), - }, - } -} - -// Different returns true if the n and other is differenct. -func (n NeighborSet) Different(other *NeighborSet) bool { - if len(n.NeighborMatrix) != len(other.NeighborMatrix) { - return true - } - - for k, nNeighbors := range n.NeighborMatrix { - oNeighbors, ok := other.NeighborMatrix[k] - if !ok { - return true - } - - if !reflect.DeepEqual(nNeighbors, oNeighbors) { - return true - } - } - - return false -} - -// NeighborAddresses construct the addresses of the neighbors into a string. -func (n NeighborSet) NeighborAddresses() string { - addressesByState := []string{} - - for state, neighbors := range n.NeighborMatrix { - addrs := make([]string, len(neighbors)) - for i, neighbor := range neighbors { - addrs[i] = neighbor.Address - } - - addressesByState = append(addressesByState, fmt.Sprintf("%s: [%s]", state, strings.Join(addrs, ","))) - } - - return strings.Join(addressesByState, ", ") -} - -// primaryNodeExists returns true if the set contains primary-state node(s). -func (n *NeighborSet) primaryNodeExists() bool { - return len(n.NeighborMatrix[controller.StatePrimary]) != 0 -} - -// replicaNodeExists returns true if the set contains replica-state node(s). -func (n *NeighborSet) replicaNodeExists() bool { - return len(n.NeighborMatrix[controller.StateReplica]) != 0 -} - -// candidateNodeExists returns true if the set contains candidate-state node(s). -func (n *NeighborSet) candidateNodeExists() bool { - return len(n.NeighborMatrix[SAKURAControllerStateCandidate]) != 0 -} - -// faultNodeExists returns true if the set contains fault-state node(s). -func (n *NeighborSet) faultNodeExists() bool { - return len(n.NeighborMatrix[controller.StateFault]) != 0 -} - -// anchorNodeExists returns true if the set contains anchor-mode node(s). -func (n *NeighborSet) anchorNodeExists() bool { - return len(n.NeighborMatrix[SAKURAControllerStateAnchor]) != 0 -} - -// Neighbor is the BGP neighbor. -type Neighbor struct { - // Address is the address of the BGP speaker. - // e.g., "192.168.0.1" - Address string -} diff --git a/pkg/controller/sakura/neighbor_test.go b/pkg/controller/sakura/neighbor_test.go deleted file mode 100644 index 64f664a..0000000 --- a/pkg/controller/sakura/neighbor_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sakura_test - -import ( - "testing" - - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/controller/sakura" - "github.com/stretchr/testify/assert" -) - -func TestDifferent_Same(t *testing.T) { - a := sakura.NewNeighborSet() - b := sakura.NewNeighborSet() - assert.False(t, a.Different(b)) -} - -func TestDifferent_DiffLen(t *testing.T) { - a := sakura.NewNeighborSet() - b := sakura.NewNeighborSet() - b.NeighborMatrix[controller.StateInitial] = append(b.NeighborMatrix[controller.StateInitial], sakura.Neighbor{}) - assert.True(t, a.Different(b)) -} - -func TestDifferent_DiffNeigh(t *testing.T) { - a := sakura.NewNeighborSet() - a.NeighborMatrix[controller.StateInitial] = append(a.NeighborMatrix[controller.StateInitial], sakura.Neighbor{}) - - b := sakura.NewNeighborSet() - b.NeighborMatrix[controller.StateInitial] = append(b.NeighborMatrix[controller.StateInitial], sakura.Neighbor{Address: "10.0.0.1"}) - assert.True(t, a.Different(b)) - -} diff --git a/pkg/controller/start.go b/pkg/controller/start.go deleted file mode 100644 index 86a4014..0000000 --- a/pkg/controller/start.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package controller - -import ( - "context" - "time" - - "golang.org/x/exp/slog" -) - -// Start starts the controller loop. -// the function recognizes a done signal from the given context. -func Start( - ctx context.Context, - logger *slog.Logger, - ctrler Controller, - ctrlerLoopInterval time.Duration, -) { - ticker := time.NewTicker(ctrlerLoopInterval) - defer ticker.Stop() - -controllerLoop: - for { - select { - case <-ctx.Done(): - ctrler.OnExit() - break controllerLoop - case <-ticker.C: - if err := ctrler.PreDecideNextStateHandler(); err != nil { - logger.Error("controller.PreMakeDecisionHandler()", err, "state", ctrler.GetState()) - continue - } - - nextState := ctrler.DecideNextState() - - logger.Debug("controller decides next state", "state", nextState) - if err := ctrler.OnStateHandler(nextState); err != nil { - logger.Error("controller.OnStateHandler()", err, "state", nextState) - } - } - } -} diff --git a/pkg/controller/sakura/test_helper.go b/pkg/controller/test_helper.go similarity index 55% rename from pkg/controller/sakura/test_helper.go rename to pkg/controller/test_helper.go index 9d6bc05..99cfc5c 100644 --- a/pkg/controller/sakura/test_helper.go +++ b/pkg/controller/test_helper.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,31 +12,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sakura +package controller import ( + "log/slog" "os" "github.com/sakura-internet/distributed-mariadb-controller/pkg/frrouting/bgpd" "github.com/sakura-internet/distributed-mariadb-controller/pkg/mariadb" "github.com/sakura-internet/distributed-mariadb-controller/pkg/nftables" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/process" "github.com/sakura-internet/distributed-mariadb-controller/pkg/systemd" - "golang.org/x/exp/slog" ) -func _newFakeSAKURAController() *SAKURAController { - logger := slog.New(slog.NewJSONHandler(os.Stderr)) - c := NewSAKURAController( +func _newFakeController() *Controller { + logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{})) + c := NewController( logger, - SystemdConnector(systemd.NewFakeSystemdConnector()), - MariaDBConnector(mariadb.NewFakeMariaDBConnector()), - NftablesConnector(nftables.NewFakeNftablesConnector()), - BGPdConnector(bgpd.NewFakeBGPdConnector()), - ProcessControlConnector(process.NewFakeProcessControlConnector()), + WithGlobalInterfaceName("dummy-global-interface-name"), + WithHostAddress("10.0.0.1"), + WithDBServingPort(3306), + WithDBReplicaUserName("repl"), + WithDBReplicaPassword("dummy-db-replica-password"), + WithDBReplicaSourcePort(0), + WithDBAclChainName("dummy-chain-name"), + WithSystemdConnector(systemd.NewFakeSystemdConnector()), + WithMariaDBConnector(mariadb.NewFakeMariaDBConnector()), + WithNftablesConnector(nftables.NewFakeNftablesConnector()), + WithBGPdConnector(bgpd.NewFakeBGPdConnector()), ) - c.HostAddress = "10.0.0.1" - c.MariaDBReplicaPassword = "dummy-db-replica-password" return c } diff --git a/pkg/frrouting/bgpd/bgp.go b/pkg/frrouting/bgpd/bgp.go index 1bc4244..5958606 100644 --- a/pkg/frrouting/bgpd/bgp.go +++ b/pkg/frrouting/bgpd/bgp.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/frrouting/bgpd/connector.go b/pkg/frrouting/bgpd/connector.go index 8e90d7a..4c739eb 100644 --- a/pkg/frrouting/bgpd/connector.go +++ b/pkg/frrouting/bgpd/connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/frrouting/bgpd/fake_bgpd_connector.go b/pkg/frrouting/bgpd/fake_bgpd_connector.go index 510eec8..36c3ebb 100644 --- a/pkg/frrouting/bgpd/fake_bgpd_connector.go +++ b/pkg/frrouting/bgpd/fake_bgpd_connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/frrouting/vtysh/bgpd_connector.go b/pkg/frrouting/vtysh/bgpd_connector.go index e0bdc1e..c0078e5 100644 --- a/pkg/frrouting/vtysh/bgpd_connector.go +++ b/pkg/frrouting/vtysh/bgpd_connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,55 +17,61 @@ package vtysh import ( "encoding/json" "fmt" + "log/slog" "net" + "time" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/bash" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/command" "github.com/sakura-internet/distributed-mariadb-controller/pkg/frrouting/bgpd" - "golang.org/x/exp/slog" ) -func NewDefaultBGPdConnector(logger *slog.Logger) bgpd.BGPdConnector { - return &VtyshBGPdConnector{Logger: logger} -} +var ( + vtyshTimeout = 5 * time.Second +) -// VtyshBGPdConnector is a default implementation of BGPdConnector. +// vtyshBGPdConnector is a default implementation of BGPdConnector. // this impl uses "vtysh" commands to interact with frrouting bgpd. -type VtyshBGPdConnector struct { - Logger *slog.Logger +type vtyshBGPdConnector struct { + logger *slog.Logger +} + +func NewDefaultBGPdConnector(logger *slog.Logger) bgpd.BGPdConnector { + return &vtyshBGPdConnector{logger: logger} } // ShowRoutesWithBGPCommunityList implements BGPdConnector -func (c *VtyshBGPdConnector) ShowRoutesWithBGPCommunityList( +func (c *vtyshBGPdConnector) ShowRoutesWithBGPCommunityList( communityList string, ) (bgpd.BGP, error) { bgp := bgpd.BGP{} - showCmd := fmt.Sprintf("show ip bgp community-list %s json", communityList) - cmd := fmt.Sprintf("vtysh -H /dev/null -c '%s'", showCmd) + name := "vtysh" + args := []string{"-H", "/dev/null", "-c", fmt.Sprintf("show ip bgp community-list %s json", communityList)} - c.Logger.Debug("execute command", "command", cmd, "callerFn", "ShowRoutesWithBGPCommunityList") - out, err := bash.RunCommand(cmd) + c.logger.Debug("execute command", "name", name, "args", args, "callerFn", "ShowRoutesWithBGPCommunityList") + out, err := command.RunWithTimeout(vtyshTimeout, name, args...) if err != nil { return bgp, fmt.Errorf("failed to show ip bgp community-list: %w", err) } if err := json.Unmarshal(out, &bgp); err != nil { - return bgp, fmt.Errorf("failed to unmarchal to bgpd.BGP: %w", err) + return bgp, fmt.Errorf("failed to unmarshal to bgpd.BGP: %w", err) } return bgp, nil } // ConfigureRouteWithRouteMap implements BGPdConnector -func (c *VtyshBGPdConnector) ConfigureRouteWithRouteMap( +func (c *vtyshBGPdConnector) ConfigureRouteWithRouteMap( prefix net.IPNet, routeMap string, ) error { + name := "vtysh" configCommand := fmt.Sprintf("network %s route-map %s", prefix.String(), routeMap) - cmd := fmt.Sprintf("vtysh -H /dev/null -c 'conf t' -c 'router bgp' -c '%s'", configCommand) + args := []string{"-H", "/dev/null", "-c", "conf t", "-c", "router bgp", "-c", configCommand} - c.Logger.Info("execute command", "command", cmd, "callerFn", "ConfigureUnicastRouteWithRouteMap") - if _, err := bash.RunCommand(cmd); err != nil { + c.logger.Info("execute command", "name", name, "args", args, "callerFn", "ConfigureUnicastRouteWithRouteMap") + if _, err := command.RunWithTimeout(vtyshTimeout, name, args...); err != nil { return fmt.Errorf("failed to advertise %s route: %w", prefix.String(), err) } diff --git a/pkg/mariadb/connector.go b/pkg/mariadb/connector.go index a648860..2e3fc37 100644 --- a/pkg/mariadb/connector.go +++ b/pkg/mariadb/connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,65 +16,59 @@ package mariadb import ( "fmt" + "log/slog" + "os" "strings" + "time" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/bash" - "golang.org/x/exp/slog" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/command" ) -// Connector is an interface that communicates with mariadb. -type Connector interface { - /* - * about variable mechanism - */ - - // CheckBoolVariableIsON checks whether the is ON. - CheckBoolVariableIsON(variableName string) bool +const ( + readOnlyVariableName = "read_only" +) - TurnOnBoolVariable(variableName string) error +var ( + mysqlCommandTimeout = 5 * time.Second +) - TurnOffBoolVariable(variableName string) error +// Connector is an interface that communicates with mariadb. +type Connector interface { + // about readonly variable mechanism + IsReadOnly() bool + TurnOnReadOnly() error + TurnOffReadOnly() error - /* - * about replication - */ + // about replication ChangeMasterTo(master MasterInstance) error - StartReplica() error - StopReplica() error - ResetAllReplicas() error - ShowReplicationStatus() (ReplicationStatus, error) - /* - * - */ - + // about operation for DB health check CreateDatabase(dbName string) error - CreateIDTable(dbName string, tableName string) error - InsertIDRecord(dbName string, tableName string, id int) error - DeleteRecords(dbName string, tableName string) error + + // remove master info or relay info + RemoveMasterInfo() error + RemoveRelayInfo() error } func NewDefaultConnector(logger *slog.Logger) Connector { - return &MySQLCommandConnector{Logger: logger} + return &mySQLCommandConnector{logger: logger} } -type MySQLCommandConnector struct { - Logger *slog.Logger +type mySQLCommandConnector struct { + logger *slog.Logger } // CreateIDTable implements Connector -func (c *MySQLCommandConnector) CreateIDTable(dbName string, tableName string) error { +func (c *mySQLCommandConnector) CreateIDTable(dbName string, tableName string) error { createCmd := fmt.Sprintf("create table if not exists %s.%s(id int)", dbName, tableName) - cmd := fmt.Sprintf("timeout -s 9 5 mysql -e '%s'", createCmd) - c.Logger.Debug("execute command", "command", cmd, "callerFn", "CreateIDTable") - if _, err := bash.RunCommand(cmd); err != err { + if _, err := c.runMysqlCommand(createCmd); err != nil { return fmt.Errorf("failed to create %s table on %s table: %w", dbName, tableName, err) } @@ -82,11 +76,9 @@ func (c *MySQLCommandConnector) CreateIDTable(dbName string, tableName string) e } // CreateDatabase implements Connector -func (c *MySQLCommandConnector) CreateDatabase(dbName string) error { +func (c *mySQLCommandConnector) CreateDatabase(dbName string) error { createCmd := fmt.Sprintf("create database if not exists %s", dbName) - cmd := fmt.Sprintf("timeout -s 9 5 mysql -e '%s'", createCmd) - c.Logger.Debug("execute command", "command", cmd, "callerFn", "CreateDatabase") - if _, err := bash.RunCommand(cmd); err != err { + if _, err := c.runMysqlCommand(createCmd); err != nil { return fmt.Errorf("failed to create %s database: %w", dbName, err) } @@ -94,11 +86,9 @@ func (c *MySQLCommandConnector) CreateDatabase(dbName string) error { } // DeleteRecords implements Connector -func (c *MySQLCommandConnector) DeleteRecords(dbName string, tableName string) error { +func (c *mySQLCommandConnector) DeleteRecords(dbName string, tableName string) error { deleteCmd := fmt.Sprintf("delete from %s.%s", dbName, tableName) - cmd := fmt.Sprintf("timeout -s 9 5 mysql -e '%s'", deleteCmd) - c.Logger.Debug("execute command", "command", cmd, "callerFn", "DeleteRecords") - if _, err := bash.RunCommand(cmd); err != nil { + if _, err := c.runMysqlCommand(deleteCmd); err != nil { return fmt.Errorf("failed to delete records from %s.%s: %w", dbName, tableName, err) } @@ -106,58 +96,54 @@ func (c *MySQLCommandConnector) DeleteRecords(dbName string, tableName string) e } // InsertIDRecord implements Connector -func (c *MySQLCommandConnector) InsertIDRecord(dbName string, tableName string, id int) error { +func (c *mySQLCommandConnector) InsertIDRecord(dbName string, tableName string, id int) error { insertCmd := fmt.Sprintf("insert into %s.%s values(%d)", dbName, tableName, id) - cmd := fmt.Sprintf("timeout -s 9 5 mysql -e '%s'", insertCmd) - c.Logger.Debug("execute command", "command", cmd, "callerFn", "InsertIDRecord") - if _, err := bash.RunCommand(cmd); err != nil { + if _, err := c.runMysqlCommand(insertCmd); err != nil { return fmt.Errorf("failed to insert id record to %s.%s: %w", dbName, tableName, err) } return nil } -// CheckBoolVariableIsON implements Connector -func (c *MySQLCommandConnector) CheckBoolVariableIsON(variableName string) bool { - cmd := fmt.Sprintf("mysql -s -N -e 'show variables like \"%s\"'", variableName) - c.Logger.Debug("execute command", "command", cmd, "callerFn", "MariaDBReadOnlyVariableIsON") - out, err := bash.RunCommand(cmd) +// IsReadOnly implements Connector +func (c *mySQLCommandConnector) IsReadOnly() bool { + name := "mysql" + args := []string{"-s", "-N", "-e", fmt.Sprintf("show variables like \"%s\"", readOnlyVariableName)} + c.logger.Debug("execute command", "name", name, "args", args, "callerFn", "CheckBoolVariableIsON") + + out, err := command.RunWithTimeout(mysqlCommandTimeout, name, args...) if err != nil { - c.Logger.Debug("failed to show variable", "name", variableName, "error", err) + c.logger.Debug("failed to show variable", "name", readOnlyVariableName, "error", err) return false } s := string(out) - return strings.Contains(s, "read_only") && strings.Contains(s, "ON") + return strings.Contains(s, readOnlyVariableName) && strings.Contains(s, "ON") } -// TurnOffBoolVariable implements Connector -func (c *MySQLCommandConnector) TurnOffBoolVariable(variableName string) error { - cmd := fmt.Sprintf("mysql -e 'set global %s=0'", variableName) - c.Logger.Info("execute command", "command", cmd, "callerFn", "TurnOffBoolVariable") - if _, err := bash.RunCommand(cmd); err != nil { - return fmt.Errorf("failed to set '%s' variable to 0: %w", variableName, err) +// TurnOffReadOnly implements Connector +func (c *mySQLCommandConnector) TurnOffReadOnly() error { + setCmd := fmt.Sprintf("set global %s=0", readOnlyVariableName) + if _, err := c.runMysqlCommand(setCmd); err != nil { + return fmt.Errorf("failed to set '%s' variable to 0: %w", readOnlyVariableName, err) } return nil } -// TurnOnBoolVariable implements Connector -func (c *MySQLCommandConnector) TurnOnBoolVariable(variableName string) error { - cmd := fmt.Sprintf("mysql -e 'set global %s=1'", variableName) - c.Logger.Info("execute command", "command", cmd, "callerFn", "TurnOnBoolVariable") - if _, err := bash.RunCommand(cmd); err != nil { - return fmt.Errorf("failed to set '%s' variable to 1: %w", variableName, err) +// TurnOnReadOnly implements Connector +func (c *mySQLCommandConnector) TurnOnReadOnly() error { + setCmd := fmt.Sprintf("set global %s=1", readOnlyVariableName) + if _, err := c.runMysqlCommand(setCmd); err != nil { + return fmt.Errorf("failed to set '%s' variable to 1: %w", readOnlyVariableName, err) } return nil } // StopReplica implements Connector -func (c *MySQLCommandConnector) StartReplica() error { - cmd := "mysql -e 'start replica'" - c.Logger.Info("execute command", "command", cmd, "callerFn", "StartReplica") - if _, err := bash.RunCommand(cmd); err != nil { +func (c *mySQLCommandConnector) StartReplica() error { + if _, err := c.runMysqlCommand("start replica"); err != nil { return fmt.Errorf("failed to start replica: %w", err) } @@ -165,10 +151,8 @@ func (c *MySQLCommandConnector) StartReplica() error { } // StopReplica implements Connector -func (c *MySQLCommandConnector) StopReplica() error { - cmd := "mysql -e 'stop replica'" - c.Logger.Info("execute command", "command", cmd, "callerFn", "StopReplica") - if _, err := bash.RunCommand(cmd); err != nil { +func (c *mySQLCommandConnector) StopReplica() error { + if _, err := c.runMysqlCommand("stop replica"); err != nil { return fmt.Errorf("failed to stop replica: %w", err) } @@ -176,10 +160,8 @@ func (c *MySQLCommandConnector) StopReplica() error { } // ResetAllReplicas implements Connector -func (c *MySQLCommandConnector) ResetAllReplicas() error { - cmd := "mysql -e 'reset replica all'" - c.Logger.Info("execute command", "command", cmd, "callerFn", "ResetAllReplicas") - if _, err := bash.RunCommand(cmd); err != nil { +func (c *mySQLCommandConnector) ResetAllReplicas() error { + if _, err := c.runMysqlCommand("reset replica all"); err != nil { return fmt.Errorf("failed to reset all replicas: %w", err) } @@ -187,7 +169,7 @@ func (c *MySQLCommandConnector) ResetAllReplicas() error { } // ChangeMasterTo implements Connector -func (c *MySQLCommandConnector) ChangeMasterTo( +func (c *mySQLCommandConnector) ChangeMasterTo( master MasterInstance, ) error { changeMasterOpts := []string{} @@ -197,21 +179,18 @@ func (c *MySQLCommandConnector) ChangeMasterTo( changeMasterOpts = append(changeMasterOpts, fmt.Sprintf("master_password = \"%s\"", master.Password)) changeMasterOpts = append(changeMasterOpts, fmt.Sprintf("master_use_gtid = %s", master.UseGTID)) - cmd := fmt.Sprintf("mysql -e 'change master to %s'", strings.Join(changeMasterOpts, ", ")) - c.Logger.Info("execute command", "command", cmd, "callerFn", "ChangeMasterTo") - if out, err := bash.RunCommand(cmd); err != nil { - c.Logger.Debug("changeMasterConnectConfig() output", "output", string(out)) - return fmt.Errorf("failed to change master connection config: %w", err) + cmd := fmt.Sprintf("change master to %s", strings.Join(changeMasterOpts, ", ")) + if out, err := c.runMysqlCommand(cmd); err != nil { + c.logger.Debug("changeMasterTo", "output", string(out)) + return fmt.Errorf("failed to change master to: %w", err) } return nil } // ShowReplicationStatus implements Connector -func (c *MySQLCommandConnector) ShowReplicationStatus() (ReplicationStatus, error) { - cmd := "mysql -e 'show replica status\\G'" - c.Logger.Debug("execute command", "command", cmd, "callerFn", "showReplicaStatusCommand") - out, err := bash.RunCommand(cmd) +func (c *mySQLCommandConnector) ShowReplicationStatus() (ReplicationStatus, error) { + out, err := c.runMysqlCommand("show replica status\\G") if err != nil { return ReplicationStatus{}, fmt.Errorf("failed to show replica status: %w", err) } @@ -219,6 +198,39 @@ func (c *MySQLCommandConnector) ShowReplicationStatus() (ReplicationStatus, erro return parseShowReplicaStatusOutput(string(out)), nil } +// runMysqlCommand executes specified mysql command with timeout and logging +func (c *mySQLCommandConnector) runMysqlCommand(mysqlcmd string) ([]byte, error) { + name := "mysql" + args := []string{"-e", mysqlcmd} + + c.logger.Debug("execute command", "name", name, "args", args) + return command.RunWithTimeout(mysqlCommandTimeout, name, args...) +} + +func (c *mySQLCommandConnector) RemoveMasterInfo() error { + _, err := os.Stat(MasterInfoFilePath) + + // do nothing if file is not found + if err != nil { + return nil + } + + // delete if file exists + return os.Remove(MasterInfoFilePath) +} + +func (c *mySQLCommandConnector) RemoveRelayInfo() error { + _, err := os.Stat(RelayInfoFilePath) + + // do nothing if file is not found + if err != nil { + return nil + } + + // delete if file exists + return os.Remove(RelayInfoFilePath) +} + // parseShowReplicaStatusOutput parses the output of the "mysql -e 'show replica status \G'". func parseShowReplicaStatusOutput(out string) ReplicationStatus { m := ReplicationStatus{} diff --git a/pkg/mariadb/connector_whitebox_test.go b/pkg/mariadb/connector_whitebox_test.go index 19ebfe4..4c3ce4f 100644 --- a/pkg/mariadb/connector_whitebox_test.go +++ b/pkg/mariadb/connector_whitebox_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/mariadb/variable.go b/pkg/mariadb/daemon.go similarity index 74% rename from pkg/mariadb/variable.go rename to pkg/mariadb/daemon.go index 6575ac8..a081587 100644 --- a/pkg/mariadb/variable.go +++ b/pkg/mariadb/daemon.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,5 +15,7 @@ package mariadb const ( - ReadOnlyVariableName = "read_only" + SystemdSerivceName = "mariadb" + MasterInfoFilePath = "/var/lib/mysql/master.info" + RelayInfoFilePath = "/var/lib/mysql/relay-log.info" ) diff --git a/pkg/mariadb/fake_connector.go b/pkg/mariadb/fake_connector.go index 1db4052..93408d4 100644 --- a/pkg/mariadb/fake_connector.go +++ b/pkg/mariadb/fake_connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -40,12 +40,6 @@ func (c *FakeMariaDBConnector) ChangeMasterTo(master MasterInstance) error { return nil } -// CheckBoolVariableIsON implements mariadb.Connector -func (c *FakeMariaDBConnector) CheckBoolVariableIsON(variableName string) bool { - c.Timestamp["CheckBoolVariableIsON"] = time.Now() - return c.ReadOnlyVariable -} - // ResetAllReplicas implements mariadb.Connector func (c *FakeMariaDBConnector) ResetAllReplicas() error { c.Timestamp["ResetAllRelicas"] = time.Now() @@ -74,22 +68,23 @@ func (c *FakeMariaDBConnector) StopReplica() error { return nil } -// TurnOffBoolVariable implements mariadb.Connector -func (c *FakeMariaDBConnector) TurnOffBoolVariable(variableName string) error { - c.Timestamp[fmt.Sprintf("TurnOffBoolVariable(%s)", variableName)] = time.Now() - if variableName == ReadOnlyVariableName { - c.ReadOnlyVariable = false - } +// IsReadOnly implements mariadb.Connector +func (c *FakeMariaDBConnector) IsReadOnly() bool { + c.Timestamp["IsReadOnly"] = time.Now() + return c.ReadOnlyVariable +} +// TurnOffReadOnly implements mariadb.Connector +func (c *FakeMariaDBConnector) TurnOffReadOnly() error { + c.Timestamp["TurnOffReadOnly"] = time.Now() + c.ReadOnlyVariable = false return nil } -// TurnOnBoolVariable implements mariadb.Connector -func (c *FakeMariaDBConnector) TurnOnBoolVariable(variableName string) error { - c.Timestamp[fmt.Sprintf("TurnOnBoolVariable(%s)", variableName)] = time.Now() - if variableName == ReadOnlyVariableName { - c.ReadOnlyVariable = true - } +// TurnOnReadOnly implements mariadb.Connector +func (c *FakeMariaDBConnector) TurnOnReadOnly() error { + c.Timestamp["TurnOnReadOnly"] = time.Now() + c.ReadOnlyVariable = true return nil } @@ -117,6 +112,14 @@ func (c *FakeMariaDBConnector) InsertIDRecord(dbName string, tableName string, i return nil } +func (c *FakeMariaDBConnector) RemoveMasterInfo() error { + return nil +} + +func (c *FakeMariaDBConnector) RemoveRelayInfo() error { + return nil +} + // FakeMariaDBFailWriteTestDataConnector is the mariadb connector that fails to write testdata. type FakeMariaDBFailWriteTestDataConnector struct { } @@ -130,8 +133,8 @@ func (c *FakeMariaDBFailWriteTestDataConnector) ChangeMasterTo(master MasterInst return nil } -// CheckBoolVariableIsON implements mariadb.Connector -func (c *FakeMariaDBFailWriteTestDataConnector) CheckBoolVariableIsON(variableName string) bool { +// IsReadOnly implements mariadb.Connector +func (c *FakeMariaDBFailWriteTestDataConnector) IsReadOnly() bool { return true } @@ -160,13 +163,13 @@ func (c *FakeMariaDBFailWriteTestDataConnector) StopReplica() error { return nil } -// TurnOffBoolVariable implements mariadb.Connector -func (c *FakeMariaDBFailWriteTestDataConnector) TurnOffBoolVariable(variableName string) error { +// TurnOffReadOnly implements mariadb.Connector +func (c *FakeMariaDBFailWriteTestDataConnector) TurnOffReadOnly() error { return nil } -// TurnOnBoolVariable implements mariadb.Connector -func (c *FakeMariaDBFailWriteTestDataConnector) TurnOnBoolVariable(variableName string) error { +// TurnOnReadOnly implements mariadb.Connector +func (c *FakeMariaDBFailWriteTestDataConnector) TurnOnReadOnly() error { return nil } @@ -189,3 +192,11 @@ func (*FakeMariaDBFailWriteTestDataConnector) DeleteRecords(dbName string, table func (*FakeMariaDBFailWriteTestDataConnector) InsertIDRecord(dbName string, tableName string, id int) error { return nil } + +func (c *FakeMariaDBFailWriteTestDataConnector) RemoveMasterInfo() error { + return nil +} + +func (c *FakeMariaDBFailWriteTestDataConnector) RemoveRelayInfo() error { + return nil +} diff --git a/pkg/mariadb/fake_mariadb_failed_replication_connector.go b/pkg/mariadb/fake_mariadb_failed_replication_connector.go index b5cac73..7f53052 100644 --- a/pkg/mariadb/fake_mariadb_failed_replication_connector.go +++ b/pkg/mariadb/fake_mariadb_failed_replication_connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -26,8 +26,8 @@ func (c *FakeMariaDBFailedReplicationConnector) ChangeMasterTo(master MasterInst return nil } -// CheckBoolVariableIsON implements mariadb.Connector -func (c *FakeMariaDBFailedReplicationConnector) CheckBoolVariableIsON(variableName string) bool { +// IsReadOnly implements mariadb.Connector +func (c *FakeMariaDBFailedReplicationConnector) IsReadOnly() bool { return true } @@ -56,13 +56,13 @@ func (c *FakeMariaDBFailedReplicationConnector) StopReplica() error { return nil } -// TurnOffBoolVariable implements mariadb.Connector -func (c *FakeMariaDBFailedReplicationConnector) TurnOffBoolVariable(variableName string) error { +// TurnOffReadOnly implements mariadb.Connector +func (c *FakeMariaDBFailedReplicationConnector) TurnOffReadOnly() error { return nil } -// TurnOnBoolVariable implements mariadb.Connector -func (c *FakeMariaDBFailedReplicationConnector) TurnOnBoolVariable(variableName string) error { +// TurnOnReadOnly implements mariadb.Connector +func (c *FakeMariaDBFailedReplicationConnector) TurnOnReadOnly() error { return nil } @@ -85,3 +85,13 @@ func (*FakeMariaDBFailedReplicationConnector) DeleteRecords(dbName string, table func (*FakeMariaDBFailedReplicationConnector) InsertIDRecord(dbName string, tableName string, id int) error { return nil } + +// RemoveMasterInfo implements mariadb.Connector +func (*FakeMariaDBFailedReplicationConnector) RemoveMasterInfo() error { + return nil +} + +// RemoveRelayInfo implements mariadb.Connector +func (*FakeMariaDBFailedReplicationConnector) RemoveRelayInfo() error { + return nil +} diff --git a/pkg/mariadb/replication.go b/pkg/mariadb/replication.go index ee445d2..ca133a1 100644 --- a/pkg/mariadb/replication.go +++ b/pkg/mariadb/replication.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ package mariadb +type MasterUseGTIDValue string + type MasterInstance struct { Host string - Port int + Port uint16 User string Password string UseGTID MasterUseGTIDValue } -type MasterUseGTIDValue string - const ( MasterUseGTIDValueCurrentPos MasterUseGTIDValue = "current_pos" MasterUseGTIDValueSlavePos MasterUseGTIDValue = "slave_pos" diff --git a/pkg/nftables/connector.go b/pkg/nftables/connector.go index 60596bf..d5897ed 100644 --- a/pkg/nftables/connector.go +++ b/pkg/nftables/connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,62 +16,79 @@ package nftables import ( "fmt" - "strings" + "log/slog" + "time" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/bash" - "golang.org/x/exp/slog" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/command" +) + +const ( + builtinTableFilter = "filter" +) + +var ( + nftCommandTimeout = 5 * time.Second ) // Connector is an interface that communicates with nftables. type Connector interface { - FlushChain( - table string, - chain string, - ) error - AddRule( - table string, - chain string, - matches []Match, - statement Statement, - ) error + FlushChain(chain string) error + CreateChain(chain string) error + AddRule(chain string, matches []Match, statement statement) error } -func NewDefaultConnector(logger *slog.Logger) Connector { - return &NftCommandConnector{Logger: logger} +// nftCommandConnector is a default implementation of Connector. +// this impl uses "nft" commands to interact with nftables. +type nftCommandConnector struct { + logger *slog.Logger } -// NftComandConnector is a default implementation of Connector. -// this impl uses "nft" commands to interact with nftables. -type NftCommandConnector struct { - Logger *slog.Logger +func NewDefaultConnector(logger *slog.Logger) Connector { + return &nftCommandConnector{logger: logger} } // AddRule implements Connector -func (c *NftCommandConnector) AddRule( - table string, chain string, matches []Match, statement Statement) error { - matchesStr := make([]string, len(matches)) - for i := range matches { - matchesStr[i] = string(matches[i]) +func (c *nftCommandConnector) AddRule( + chain string, matches []Match, stmt statement, +) error { + name := "nft" + args := []string{"add", "rule", builtinTableFilter, chain} + for _, match := range matches { + args = append(args, match...) } - - cmd := fmt.Sprintf("nft add rule %s %s %s %s", table, chain, strings.Join(matchesStr, " "), statement) - c.Logger.Info("execute command", "command", cmd, "callerFn", "AddRule") - if _, err := bash.RunCommand(cmd); err != nil { - return fmt.Errorf("failed to add rule to chain %s on table %s: %w", chain, table, err) + args = append(args, stmt...) + c.logger.Info("execute command", "name", name, "args", args) + if _, err := command.RunWithTimeout(nftCommandTimeout, name, args...); err != nil { + return fmt.Errorf("failed to add rule to chain %s on table %s: %w", chain, builtinTableFilter, err) } return nil } // FlushChain implements Connector -func (c *NftCommandConnector) FlushChain( - table string, +func (c *nftCommandConnector) FlushChain( + chain string, +) error { + name := "nft" + args := []string{"flush", "chain", builtinTableFilter, chain} + c.logger.Info("execute command", "name", name, "args", args) + if _, err := command.RunWithTimeout(nftCommandTimeout, name, args...); err != nil { + return fmt.Errorf("failed to flush chain %s on table %s: %w", chain, builtinTableFilter, err) + } + + return nil +} + +// CreateChain implements Connector +func (c *nftCommandConnector) CreateChain( chain string, ) error { - cmd := fmt.Sprintf("nft flush chain %s %s", table, chain) - c.Logger.Info("execute command", "command", cmd, "callerFn", "FlushChain") - if _, err := bash.RunCommand(cmd); err != nil { - return fmt.Errorf("failed to flush chain %s on table %s: %w", chain, table, err) + // nft add chain comand returns ok if the chain is already exist. + name := "nft" + args := []string{"add", "chain", builtinTableFilter, chain, "{ type filter hook input priority 0; }"} + c.logger.Info("execute command", "name", name, "args", args) + if _, err := command.RunWithTimeout(nftCommandTimeout, name, args...); err != nil { + return fmt.Errorf("failed to add nft chain: %w", err) } return nil diff --git a/pkg/nftables/fake_connector.go b/pkg/nftables/fake_connector.go index 42c4662..5d310aa 100644 --- a/pkg/nftables/fake_connector.go +++ b/pkg/nftables/fake_connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -25,17 +25,23 @@ type FakeNftablesConnector struct { } // AddRule implements nftables.Connector -func (c *FakeNftablesConnector) AddRule(table string, chain string, matches []Match, statement Statement) error { +func (c *FakeNftablesConnector) AddRule(chain string, matches []Match, statement statement) error { c.Timestamp["AddRule"] = time.Now() return nil } // FlushChain implements nftables.Connector -func (c *FakeNftablesConnector) FlushChain(table string, chain string) error { +func (c *FakeNftablesConnector) FlushChain(chain string) error { c.Timestamp["FlushChain"] = time.Now() return nil } +// CreateChain implements nftables.Connector +func (c *FakeNftablesConnector) CreateChain(chain string) error { + c.Timestamp["CreateChain"] = time.Now() + return nil +} + func NewFakeNftablesConnector() Connector { return &FakeNftablesConnector{ Timestamp: make(map[string]time.Time), diff --git a/pkg/nftables/filter.go b/pkg/nftables/filter.go deleted file mode 100644 index dad1aea..0000000 --- a/pkg/nftables/filter.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package nftables - -const ( - BuiltinTableFilter = "filter" -) diff --git a/pkg/nftables/match.go b/pkg/nftables/match.go index 41dc826..bc0e95c 100644 --- a/pkg/nftables/match.go +++ b/pkg/nftables/match.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,18 +14,20 @@ package nftables -import "fmt" +import ( + "strconv" +) -type Match string +type Match []string func IPSrcAddrMatch(srcAddr string) Match { - return Match(fmt.Sprintf("ip saddr %s", srcAddr)) + return []string{"ip", "saddr", srcAddr} } func TCPDstPortMatch(dport uint16) Match { - return Match(fmt.Sprintf("tcp dport %d", dport)) + return []string{"tcp", "dport", strconv.Itoa(int(dport))} } func IFNameMatch(ifname string) Match { - return Match(fmt.Sprintf("iifname \"%s\"", ifname)) + return []string{"iifname", ifname} } diff --git a/pkg/nftables/statement.go b/pkg/nftables/statement.go index 4deeb0a..c99d0ab 100644 --- a/pkg/nftables/statement.go +++ b/pkg/nftables/statement.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,21 +14,19 @@ package nftables -import "fmt" +type statement []string -type Statement string - -func AcceptStatement() Statement { - return Statement("accept") +func AcceptStatement() statement { + return []string{"accept"} } -func RejectStatement() Statement { - return Statement("reject") +func RejectStatement() statement { + return []string{"reject"} } func RejectStatementWithProto( proto string, protoType string, -) Statement { - return Statement(fmt.Sprintf("reject with %s type %s", proto, protoType)) +) statement { + return []string{"reject", "with", proto, "type", protoType} } diff --git a/pkg/process/connector.go b/pkg/process/connector.go deleted file mode 100644 index 80168d7..0000000 --- a/pkg/process/connector.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package process - -import ( - "fmt" - - "github.com/sakura-internet/distributed-mariadb-controller/pkg/bash" - "golang.org/x/exp/slog" -) - -// ProcessControlConnector is an interface that communicates with Linux processes. -type ProcessControlConnector interface { - KillProcessWithFullName(processName string) error -} - -func NewDefaultConnector(logger *slog.Logger) ProcessControlConnector { - return &ShellCommandConnector{Logger: logger} -} - -// ShellCommandConnector is a default implementation of ProcessControlConnector. -// this impl uses "pkill/etc" commands to interact with Linux processes. -type ShellCommandConnector struct { - Logger *slog.Logger -} - -// KillProcessWithFullName implements Connector -func (c *ShellCommandConnector) KillProcessWithFullName( - processName string, -) error { - cmd := fmt.Sprintf("pkill -9 -f %s", processName) - - c.Logger.Info("execute command", "command", cmd, "callerFn", "KillProcessWithFullName") - if _, err := bash.RunCommand(cmd); err != nil { - return fmt.Errorf("failed to kill %s process: %w", processName, err) - } - - return nil -} diff --git a/pkg/process/fake_connector.go b/pkg/process/fake_connector.go deleted file mode 100644 index 99f4f32..0000000 --- a/pkg/process/fake_connector.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2023 The distributed-mariadb-controller Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package process - -import ( - "time" -) - -// FakeProcessControlConnector is for testing the controller. -type FakeProcessControlConnector struct { - // Timestamp holds the method execution's timestamp. - Timestamp map[string]time.Time - // ProcessLived checks whether the (fake) process is lived. - ProcessLived map[string]bool -} - -// KillProcessWithFullName implements process.ProcessControlConnector -func (c *FakeProcessControlConnector) KillProcessWithFullName(processName string) error { - c.ProcessLived[processName] = false - c.Timestamp["KillProcessWithFullName"] = time.Now() - - return nil -} - -func NewFakeProcessControlConnector() ProcessControlConnector { - return &FakeProcessControlConnector{ - Timestamp: make(map[string]time.Time), - ProcessLived: make(map[string]bool), - } -} diff --git a/pkg/systemd/connector.go b/pkg/systemd/connector.go index ec43216..abedfc9 100644 --- a/pkg/systemd/connector.go +++ b/pkg/systemd/connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,75 +16,63 @@ package systemd import ( "fmt" + "log/slog" + "time" - "github.com/sakura-internet/distributed-mariadb-controller/pkg/bash" - "golang.org/x/exp/slog" + "github.com/sakura-internet/distributed-mariadb-controller/pkg/command" +) + +const ( + systemctlCommandTimeout = 60 * time.Second ) // Connector is an interface that communicates with systemd. type Connector interface { // StartSerivce starts a systemd service. - // preHook are triggered before starting a service. - // postHook are triggered after starting a service. - StartService( - serviceName string, - preHook func() error, - postHook func() error, - ) error - - StopService( - serviceName string, - ) error + StartService(serviceName string) error + + // StopService stops a systemd service. + StopService(serviceName string) error + + // KillService kills a systemd service. + KillService(serviceName string) error + // CheckServiceStatus checks the status of a systemd service. - CheckServiceStatus( - serviceName string, - ) error + CheckServiceStatus(serviceName string) error } -func NewDefaultConnector(logger *slog.Logger) Connector { - return &SystemCtlConnector{Logger: logger} +// systemCtlConnector is a default implementation of Connector. +// this impl uses "systemctl" commands to interact with systemd. +type systemCtlConnector struct { + logger *slog.Logger } -// SystemCtlConnector is a default implementation of Connector. -// this impl uses "systemctl" commands to interact with systemd. -type SystemCtlConnector struct { - Logger *slog.Logger +func NewDefaultConnector(logger *slog.Logger) Connector { + return &systemCtlConnector{logger: logger} } // StartSerivce implements Connector -func (c *SystemCtlConnector) StartService( +func (c *systemCtlConnector) StartService( serviceName string, - preHook func() error, - postHook func() error, ) error { - if preHook != nil { - if err := preHook(); err != nil { - return err - } - } - - cmd := fmt.Sprintf("systemctl start %s", serviceName) - c.Logger.Info("execute command", "command", cmd, "callerFn", "CheckServiceStatus") - if _, err := bash.RunCommand(cmd); err != nil { + name := "systemctl" + args := []string{"start", serviceName} + c.logger.Info("execute command", "name", name, "args", args) + if _, err := command.RunWithTimeout(systemctlCommandTimeout, name, args...); err != nil { return fmt.Errorf("failed to start %s service: %w", serviceName, err) } - if postHook != nil { - if err := postHook(); err != nil { - return err - } - } - return nil } // CheckServiceStatus implements Connector -func (c *SystemCtlConnector) CheckServiceStatus( +func (c *systemCtlConnector) CheckServiceStatus( serviceName string, ) error { - cmd := fmt.Sprintf("systemctl status %s", serviceName) - c.Logger.Debug("execute command", "command", cmd, "callerFn", "CheckServiceStatus") - if _, err := bash.RunCommand(cmd); err != nil { + name := "systemctl" + args := []string{"status", serviceName} + c.logger.Debug("execute command", "name", name, "args", args) + if _, err := command.RunWithTimeout(systemctlCommandTimeout, name, args...); err != nil { return fmt.Errorf("failed to check %s service: %w", serviceName, err) } @@ -92,13 +80,27 @@ func (c *SystemCtlConnector) CheckServiceStatus( } // StopService implements Connector -func (c *SystemCtlConnector) StopService( +func (c *systemCtlConnector) StopService( serviceName string, ) error { - cmd := fmt.Sprintf("systemctl stop %s", serviceName) + name := "systemctl" + args := []string{"stop", serviceName} + c.logger.Info("execute command", "name", name, "args", args) + if _, err := command.RunWithTimeout(systemctlCommandTimeout, name, args...); err != nil { + return fmt.Errorf("failed to stop service %s : %w", serviceName, err) + } - c.Logger.Info("execute command", "command", cmd, "callerFn", "StopService") - if _, err := bash.RunCommand(cmd); err != nil { + return nil +} + +// KillService implements Connector +func (c *systemCtlConnector) KillService( + serviceName string, +) error { + name := "systemctl" + args := []string{"kill", "-s", "SIGKILL", serviceName} + c.logger.Info("execute command", "name", name, "args", args) + if _, err := command.RunWithTimeout(systemctlCommandTimeout, name, args...); err != nil { return fmt.Errorf("failed to stop service %s : %w", serviceName, err) } diff --git a/pkg/systemd/fake_connector.go b/pkg/systemd/fake_connector.go index aa07835..bfd4ccf 100644 --- a/pkg/systemd/fake_connector.go +++ b/pkg/systemd/fake_connector.go @@ -1,4 +1,4 @@ -// Copyright 2023 The distributed-mariadb-controller Authors +// Copyright 2025 The distributed-mariadb-controller Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ func (c *FakeSystemdConnector) CheckServiceStatus(serviceName string) error { } // StartService implements systemd.Connector -func (c *FakeSystemdConnector) StartService(serviceName string, preHook func() error, postHook func() error) error { +func (c *FakeSystemdConnector) StartService(serviceName string) error { c.ServiceStarted[serviceName] = true c.Timestamp["StartService"] = time.Now() return nil @@ -46,6 +46,13 @@ func (c *FakeSystemdConnector) StopService(serviceName string) error { return nil } +// KillService implements systemd.Connector +func (c *FakeSystemdConnector) KillService(serviceName string) error { + c.ServiceStarted[serviceName] = false + c.Timestamp["KillService"] = time.Now() + return nil +} + func NewFakeSystemdConnector() Connector { return &FakeSystemdConnector{ Timestamp: make(map[string]time.Time),