From b45947e9749653b168a43bb9308c10c96ac23679 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 01:40:53 +0100 Subject: [PATCH 01/22] Move localtests to golang+docker --- .github/workflows/replica-tests.yml | 33 +- .gitignore | 1 + go.mod | 4 +- go.sum | 8 +- go/cmd/gh-ost-localtests/main.go | 104 ++++++ go/localtests/test.go | 196 +++++++++++ go/localtests/tester.go | 198 +++++++++++ go/localtests/utils.go | 135 ++++++++ go/sql/parser.go | 6 + go/sql/parser_test.go | 42 ++- localtests/Dockerfile | 22 ++ localtests/docker-compose.yml | 25 ++ localtests/init-replica.sql | 9 + localtests/init.sql | 5 + localtests/keyword-column/extra_args | 2 +- localtests/mysql-env.sh | 15 + localtests/trivial/extra_args | 2 +- vendor/github.com/google/shlex/COPYING | 202 ++++++++++++ vendor/github.com/google/shlex/README | 2 + vendor/github.com/google/shlex/shlex.go | 416 ++++++++++++++++++++++++ vendor/modules.txt | 5 +- 21 files changed, 1403 insertions(+), 29 deletions(-) create mode 100644 go/cmd/gh-ost-localtests/main.go create mode 100644 go/localtests/test.go create mode 100644 go/localtests/tester.go create mode 100644 go/localtests/utils.go create mode 100644 localtests/Dockerfile create mode 100644 localtests/docker-compose.yml create mode 100644 localtests/init-replica.sql create mode 100644 localtests/init.sql create mode 100755 localtests/mysql-env.sh create mode 100644 vendor/github.com/google/shlex/COPYING create mode 100644 vendor/github.com/google/shlex/README create mode 100644 vendor/github.com/google/shlex/shlex.go diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index f2a52ece8..9195c545c 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -1,6 +1,6 @@ name: migration tests - -on: [pull_request] +on: + - pull_request jobs: build: @@ -8,17 +8,30 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - version: [mysql-5.7.25,mysql-8.0.16,PerconaServer-8.0.21] + tests: + - image: mysql:5.7 + engine: innodb + - image: mysql:8.0 + engine: innodb + - image: percona:5.7 + engine: innodb + - image: percona:8.0 + engine: innodb + - image: percona:5.7 + engine: rocksdb + - image: percona:8.0 + engine: rocksdb steps: - uses: actions/checkout@v2 - - name: Set up Go - uses: actions/setup-go@v1 - with: - go-version: 1.17 + - name: generate mysql environment file + env: + TEST_STORAGE_ENGINE: "${{ matrix.tests.engine }}" + run: localtests/mysql-env.sh - - name: migration tests + - name: run localtests env: - TEST_MYSQL_VERSION: ${{ matrix.version }} - run: script/cibuild-gh-ost-replica-tests + TEST_DOCKER_IMAGE: "${{ matrix.tests.image }}" + run: docker-compose -f localtests/docker-compose.yml up --abort-on-container-exit --no-log-prefix tests + diff --git a/.gitignore b/.gitignore index 605546d20..9c711e9e8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /.gopath/ /bin/ /libexec/ +/localtests/mysql.env /.vendor/ .idea/ diff --git a/go.mod b/go.mod index 885d97eeb..d25669c57 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,11 @@ require ( github.com/go-ini/ini v1.62.0 github.com/go-mysql-org/go-mysql v1.3.0 github.com/go-sql-driver/mysql v1.6.0 + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/openark/golib v0.0.0-20210531070646-355f37940af8 github.com/satori/go.uuid v1.2.0 - golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 + golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 golang.org/x/text v0.3.6 ) @@ -21,7 +22,6 @@ require ( github.com/smartystreets/goconvey v1.6.4 // indirect go.uber.org/atomic v1.7.0 // indirect golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect - golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.62.0 // indirect ) diff --git a/go.sum b/go.sum index 6b44084a0..a2cf26214 100644 --- a/go.sum +++ b/go.sum @@ -17,6 +17,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= @@ -81,8 +83,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -96,14 +96,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/go/cmd/gh-ost-localtests/main.go b/go/cmd/gh-ost-localtests/main.go new file mode 100644 index 000000000..df7ab947c --- /dev/null +++ b/go/cmd/gh-ost-localtests/main.go @@ -0,0 +1,104 @@ +package main + +import ( + "database/sql" + "flag" + "fmt" + "log" + "os" + + _ "github.com/go-sql-driver/mysql" + + "github.com/github/gh-ost/go/localtests" +) + +var AppVersion string + +func envStringVarOrDefault(envVar, defaultVal string) string { + if val := os.Getenv(envVar); val != "" { + return val + } + return defaultVal +} + +func main() { + // flags + var printVersion, testNoop bool + var testName string + var cnf localtests.Config + flag.StringVar(&cnf.Host, "host", localtests.DefaultHost, "mysql host") + flag.Int64Var(&cnf.Port, "port", localtests.DefaultPort, "mysql port") + flag.StringVar(&cnf.Username, "username", localtests.DefaultUsername, "mysql username") + flag.StringVar(&cnf.Password, "password", localtests.DefaultPassword, "mysql password") + flag.StringVar(&cnf.TestsDir, "tests-dir", "/etc/localtests", "path to localtests directory") + flag.StringVar(&testName, "test", "", "run a single test by name (default: run all tests)") + flag.BoolVar(&testNoop, "test-noop", false, "run a single noop migration, eg: --alter='ENGINE=InnoDB'") + flag.StringVar(&cnf.StorageEngine, "storage-engine", envStringVarOrDefault("TEST_STORAGE_ENGINE", "innodb"), "mysql storage engine") + flag.StringVar(&cnf.GhostBinary, "binary", "gh-ost", "path to gh-ost binary") + flag.StringVar(&cnf.MysqlBinary, "mysql-binary", "mysql", "path to mysql binary") + flag.BoolVar(&printVersion, "version", false, "print version and exit") + flag.Parse() + + // print version + if printVersion { + fmt.Println(AppVersion) + os.Exit(0) + } + + // connect to replica + replica, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/?interpolateParams=true", + cnf.Username, + cnf.Password, + cnf.Host, + cnf.Port, + )) + if err != nil { + log.Fatal(err) + } + defer replica.Close() + + // connect to primary + primary, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/?interpolateParams=true", + cnf.Username, + cnf.Password, + "primary", // TODO: fix me + cnf.Port, + )) + if err != nil { + log.Fatal(err) + } + defer primary.Close() + + // start tester + tester := localtests.NewTester(cnf, primary, replica) + if err = tester.WaitForMySQLAvailable(); err != nil { + log.Fatalf("Failed to setup MySQL database servers: %+v", err) + } + + // find tests + var tests []localtests.Test + if testNoop { + tests = []localtests.Test{ + { + Name: "noop", + ExtraArgs: []string{ + fmt.Sprintf("--alter='ENGINE=%s'", cnf.StorageEngine), + }, + }, + } + } else { + tests, err = tester.ReadTests(testName) + if err != nil { + log.Fatalf("Failed to read tests: %+v", err) + } + } + + // run tests + for _, test := range tests { + log.Println("------------------------------------------------------------------------------------------------------------") + log.Printf("Loading test %q at %s/%s", test.Name, cnf.TestsDir, test.Name) + if err = tester.RunTest(test); err != nil { + log.Fatalf("Failed to run test %s: %+v", test.Name, err) + } + } +} diff --git a/go/localtests/test.go b/go/localtests/test.go new file mode 100644 index 000000000..1ed659f10 --- /dev/null +++ b/go/localtests/test.go @@ -0,0 +1,196 @@ +package localtests + +import ( + "bytes" + "database/sql" + "fmt" + "log" + "os/exec" +) + +// Test represents a single test. +type Test struct { + Name string + Path string + CreateSQLFile string + DestroySQLFile string + ExtraArgs []string + ExpectedFailure string + SQLMode *string + IgnoreVersions []string + ValidateOrderBy string + ValidateColumns []string + ValidateOrigColumns []string + // + origPrimaryInfo *MysqlInfo +} + +func (test *Test) prepareDBPrimary(primary *sql.DB) (err error) { + test.origPrimaryInfo, err = getMysqlHostInfo(primary) + if err != nil { + return err + } + + if test.SQLMode != nil { + if err = setDBGlobalSqlMode(primary, *test.SQLMode); err != nil { + return err + } + log.Printf("[%s] sql_mode set to %q on primary", test.Name, *test.SQLMode) + } + return err +} + +func (test *Test) resetDBPrimary(config Config, primary *sql.DB) { + if test.SQLMode != nil && test.origPrimaryInfo != nil { + log.Printf("[%s] resetting primary to sql_mode: %s", test.Name, test.origPrimaryInfo.SQLMode) + if err := setDBGlobalSqlMode(primary, test.origPrimaryInfo.SQLMode); err != nil { + log.Printf("[%s] failed to reset primary to sql_mode: %+v", test.Name, err) + } + } + + if test.DestroySQLFile != "" { + log.Printf("[%s] running destroy.sql file", test.Name) + stdin, stderr, err := execSQLFile(config, test.DestroySQLFile) + if err != nil { + log.Printf("[%s] failed to destroy test schema %s%s%s: %+v", test.Name, failedEmoji, failedEmoji, failedEmoji, stderr.String()) + log.Printf("[%s] destroy.sql: %s", test.Name, stdin.String()) + } + } +} + +// Prepare inits a test and runs a 'mysql' client/shell command to populate +// the test schema. The create.sql file is read by golang and passed to +// 'mysql' over stdin. +func (test *Test) Prepare(config Config, primary *sql.DB) error { + if test.CreateSQLFile == "" { + return nil + } + + if err := test.prepareDBPrimary(primary); err != nil { + return err + } + + stdin, stderr, err := execSQLFile(config, test.CreateSQLFile) + if err != nil { + log.Printf("[%s] failed to prepare test schema %s%s%s: %+v", test.Name, failedEmoji, failedEmoji, failedEmoji, stderr.String()) + log.Printf("[%s] create.sql: %s", test.Name, stdin.String()) + } + return err +} + +// Migrate runs the migration test. +func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { + defer test.resetDBPrimary(config, primary) + + replicaInfo, err := getMysqlHostInfo(replica) + if err != nil { + return err + } + + flags := []string{ + fmt.Sprintf("--user=%s", config.Username), + fmt.Sprintf("--password=%s", config.Password), + fmt.Sprintf("--host=%s", config.Host), + fmt.Sprintf("--port=%d", config.Port), + fmt.Sprintf("--assume-master-host=%s:%d", PrimaryHost, replicaInfo.Port), // TODO: fix this + fmt.Sprintf("--database=%s", testDatabase), + fmt.Sprintf("--table=%s", testTable), + fmt.Sprintf("--chunk-size=%d", testChunkSize), + fmt.Sprintf("--default-retries=%d", testDefaultRetries), + fmt.Sprintf("--throttle-query=%s", throttleQuery), + fmt.Sprintf("--throttle-flag-file=%s", throttleFlagFile), + fmt.Sprintf("--serve-socket-file=%s", testSocketFile), + fmt.Sprintf("--storage-engine=%s", config.StorageEngine), + "--allow-on-master", + "--assume-rbr", + "--debug", + "--exact-rowcount", + "--execute", + "--initially-drop-old-table", + "--initially-drop-ghost-table", + "--initially-drop-socket-file", + "--stack", + "--verbose", + } + if !flagsSliceContainsAlter(test.ExtraArgs) { + test.ExtraArgs = append(test.ExtraArgs, fmt.Sprintf(`--alter=ENGINE=%s`, config.StorageEngine)) + } + if len(test.ExtraArgs) > 0 { + flags = append(flags, test.ExtraArgs...) + } + + log.Printf("[%s] running gh-ost command with extra args: %+v", test.Name, test.ExtraArgs) + + var output, stderr bytes.Buffer + cmd := exec.Command(config.GhostBinary, flags...) + cmd.Stderr = &stderr + cmd.Stdout = &output + + if err = cmd.Run(); err != nil { + if isExpectedFailureOutput(&stderr, test.ExpectedFailure) { + return nil + } + output.Write(stderr.Bytes()) + log.Printf("[%s] test failed: %+v", test.Name, output.String()) + } + return err +} + +/* +func getPrimaryOrUniqueKey(db *sql.DB, database, table string) (string, error) { + return "id", nil // TODO: fix this +} +*/ + +// Validate performs a validation of the migration test results. +func (test *Test) Validate(config Config, primary, replica *sql.DB) error { + if len(test.ValidateColumns) == 0 || len(test.ValidateOrigColumns) == 0 { + return nil + } + + /* + primaryKey, err := getPrimaryOrUniqueKey(replica, testDatabase, testTable) + if err != nil { + return err + } + + var query string + var maxPrimaryKeyVal interface{} + if maxPrimaryKeyVal == nil { + query = fmt.Sprintf("select * from %s.%s limit 10", testDatabase, testTable) + } else { + query = fmt.Sprintf("select * from %s.%s where %s > %+v limit 10", + testDatabase, testTable, primaryKey, maxPrimaryKeyVal, + ) + } + var rowMap sqlutils.RowMap + err = sqlutils.QueryRowsMap(replica, query, func(m sqlutils.RowMap) error { + for _, col := range test.ValidateColumns { + if val, found := m[col]; found { + rowMap[col] = val + } + } + }) + + values := make([]interface{}, 0) + for range test.ValidateOrigColumns { + var val interface{} + values = append(values, &val) + } + maxPrimaryKeyVal = values[0] + + for rows.Next() { + if err = rows.Scan(values...); err != nil { + return err + } + for i, value := range values { + if value == nil { + continue + } + log.Printf("[%s] row value for %q col: %d", test.Name, test.ValidateOrigColumns[i], value) + } + } + */ + + return nil +} diff --git a/go/localtests/tester.go b/go/localtests/tester.go new file mode 100644 index 000000000..fc9359c74 --- /dev/null +++ b/go/localtests/tester.go @@ -0,0 +1,198 @@ +package localtests + +import ( + "database/sql" + "errors" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/google/shlex" +) + +const ( + PrimaryHost = "primary" + DefaultHost = "replica" + DefaultPort int64 = 3306 + DefaultUsername = "gh-ost" + DefaultPassword = "gh-ost" + testDatabase = "test" + testTable = "gh_ost_test" + testSocketFile = "/tmp/gh-ost.test.sock" + testChunkSize int64 = 10 + testDefaultRetries int64 = 3 + throttleFlagFile = "/tmp/gh-ost-test.ghost.throttle.flag" + throttleQuery = "select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc" + // + failedEmoji = "\u274C" + successEmoji = "\u2705" +) + +// Config represents the configuration. +type Config struct { + Host string + Port int64 + Username string + Password string + GhostBinary string + MysqlBinary string + StorageEngine string + TestsDir string +} + +type Tester struct { + config Config + primary *sql.DB + replica *sql.DB +} + +func NewTester(config Config, primary, replica *sql.DB) *Tester { + return &Tester{ + config: config, + primary: primary, + replica: replica, + } +} + +// WaitForMySQLAvailable waits for MySQL to become ready for +// testing on both the primary and replica. +func (t *Tester) WaitForMySQLAvailable() error { + interval := 2 * time.Second + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-time.After(5 * time.Minute): + return errors.New("timed out waiting for mysql") + case <-ticker.C: + if err := func() error { + primaryGTIDExec, err := pingAndGetGTIDExecuted(t.primary, interval) + if err != nil { + return err + } + replicaGTIDExec, err := pingAndGetGTIDExecuted(t.replica, interval) + if err != nil { + return err + } + + if !replicaGTIDExec.Contain(primaryGTIDExec) { + return fmt.Errorf("Replica/primary GTID not equal: %s != %s", + replicaGTIDExec.String(), primaryGTIDExec.String(), + ) + } + return nil + }(); err != nil { + log.Printf("Waiting for MySQL primary/replica to init: %+v", err) + continue + } + + log.Printf("MySQL primary/replica is available %s", successEmoji) + return nil + } + } +} + +// ReadTests reads test configurations from a directory. As single test isee +// returned when specificTestName is specified. +func (t *Tester) ReadTests(specificTestName string) (tests []Test, err error) { + subdirs, err := ioutil.ReadDir(t.config.TestsDir) + if err != nil { + return tests, err + } + + for _, subdir := range subdirs { + test := Test{ + Name: subdir.Name(), + Path: filepath.Join(t.config.TestsDir, subdir.Name()), + } + + stat, err := os.Stat(test.Path) + if err != nil || !stat.IsDir() { + continue + } + + if specificTestName != "" && !strings.EqualFold(test.Name, specificTestName) { + continue + } + + test.CreateSQLFile = filepath.Join(test.Path, "create.sql") + if _, err = os.Stat(test.CreateSQLFile); err != nil { + log.Printf("Failed to find create.sql file %q: %+v", test.CreateSQLFile, err) + return tests, err + } + + destroySQLFile := filepath.Join(test.Path, "destroy.sql") + if _, err = os.Stat(destroySQLFile); err == nil { + test.DestroySQLFile = destroySQLFile + } + + expectFailureFile := filepath.Join(test.Path, "expect_failure") + test.ExpectedFailure, _ = readTestFile(expectFailureFile) + + sqlModeFile := filepath.Join(test.Path, "sql_mode") + if sqlMode, err := readTestFile(sqlModeFile); err == nil { + test.SQLMode = &sqlMode + } + + orderByFile := filepath.Join(test.Path, "order_by") + test.ValidateOrderBy, _ = readTestFile(orderByFile) + + origColumnsFile := filepath.Join(test.Path, "orig_columns") + if origColumns, err := readTestFile(origColumnsFile); err == nil { + origColumns = strings.Replace(origColumns, " ", "", -1) + test.ValidateOrigColumns = strings.Split(origColumns, ",") + } + + ghostColumnsFile := filepath.Join(test.Path, "ghost_columns") + if ghostColumns, err := readTestFile(ghostColumnsFile); err == nil { + ghostColumns = strings.Replace(ghostColumns, " ", "", -1) + test.ValidateColumns = strings.Split(ghostColumns, ",") + } + + extraArgsFile := filepath.Join(test.Path, "extra_args") + if _, err = os.Stat(extraArgsFile); err == nil { + extraArgsStr, err := readTestFile(extraArgsFile) + if err != nil { + log.Printf("Failed to read extra_args file %q: %+v", extraArgsFile, err) + return tests, err + } + if test.ExtraArgs, err = shlex.Split(extraArgsStr); err != nil { + log.Printf("Failed to read extra_args file %q: %+v", extraArgsFile, err) + return tests, err + } + } + + tests = append(tests, test) + } + + return tests, err +} + +// RunTest prepares and runs a single test. +func (t *Tester) RunTest(test Test) (err error) { + if err = test.Prepare(t.config, t.primary); err != nil { + return err + } + log.Printf("[%s] prepared test %s", test.Name, successEmoji) + + if err = test.Migrate(t.config, t.primary, t.replica); err != nil { + log.Printf("[%s] failed to migrate test %s%s%s", test.Name, failedEmoji, + failedEmoji, failedEmoji) + return err + } + log.Printf("[%s] successfully migrated test %s", test.Name, successEmoji) + + if err = test.Validate(t.config, t.primary, t.replica); err != nil { + log.Printf("[%s] failed to validate test %s%s%s", test.Name, failedEmoji, + failedEmoji, failedEmoji) + return err + } + log.Printf("[%s] successfully validated test %s", test.Name, successEmoji) + + return err +} diff --git a/go/localtests/utils.go b/go/localtests/utils.go new file mode 100644 index 000000000..ff227c4a5 --- /dev/null +++ b/go/localtests/utils.go @@ -0,0 +1,135 @@ +package localtests + +import ( + "bufio" + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "strings" + "time" + + "github.com/go-mysql-org/go-mysql/mysql" +) + +func execSQLFile(config Config, sqlFile string) (stdin, stderr bytes.Buffer, err error) { + defaultsFile, err := writeMysqlClientDefaultsFile(config) + if err != nil { + return stderr, stdin, err + } + defer os.Remove(defaultsFile) + + flags := []string{ + fmt.Sprintf("--defaults-file=%s", defaultsFile), + fmt.Sprintf("--host=%s", PrimaryHost), // TODO: fix this + fmt.Sprintf("--port=%d", config.Port), + "--default-character-set=utf8mb4", + testDatabase, + } + + f, err := os.Open(sqlFile) + if err != nil { + return stderr, stdin, err + } + defer f.Close() + + cmd := exec.Command(config.MysqlBinary, flags...) + cmd.Stdin = io.TeeReader(f, &stdin) + cmd.Stderr = &stderr + cmd.Stdout = os.Stdout + + return stdin, stderr, cmd.Run() +} + +func flagsSliceContainsAlter(flags []string) bool { + for _, flag := range flags { + if strings.HasPrefix(flag, "--alter") || strings.HasPrefix(flag, "-alter") { + return true + } + } + return false +} + +type MysqlInfo struct { + Host string + Port int64 + Version string + VersionComment string + SQLMode string +} + +func getMysqlHostInfo(db *sql.DB) (*MysqlInfo, error) { + var info MysqlInfo + res := db.QueryRow("select @@hostname, @@port, @@version, @@version_comment, @@global.sql_mode") + if res.Err() != nil { + return nil, res.Err() + } + err := res.Scan(&info.Host, &info.Port, &info.Version, &info.VersionComment, &info.SQLMode) + return &info, err +} + +func isExpectedFailureOutput(output io.Reader, expectedFailure string) bool { + scanner := bufio.NewScanner(output) + for scanner.Scan() { + if !strings.Contains(scanner.Text(), "FATAL") { + continue + } else if strings.Contains(scanner.Text(), expectedFailure) { + return true + } + } + return false +} + +func pingAndGetGTIDExecuted(db *sql.DB, timeout time.Duration) (*mysql.UUIDSet, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + if err := db.PingContext(ctx); err != nil { + return nil, err + } + + row := db.QueryRowContext(ctx, "select @@global.gtid_executed") + if err := row.Err(); err != nil { + return nil, err + } + + var uuidSet string + if err := row.Scan(&uuidSet); err != nil { + return nil, err + } else if uuidSet == "" { + return nil, errors.New("gtid_executed is undefined") + } + return mysql.ParseUUIDSet(uuidSet) +} + +func readTestFile(file string) (string, error) { + bytes, err := ioutil.ReadFile(file) + if err != nil { + return "", err + } + return strings.TrimSpace(string(bytes)), nil +} + +func setDBGlobalSqlMode(db *sql.DB, sqlMode string) (err error) { + _, err = db.Exec("set @@global.sql_mode=?", sqlMode) + return err +} + +func writeMysqlClientDefaultsFile(config Config) (string, error) { + defaultsFile, err := os.CreateTemp("", "gh-ost-localtests.my.cnf") + if err != nil { + return "", err + } + defer defaultsFile.Close() + + _, err = defaultsFile.Write([]byte(fmt.Sprintf( + "[client]\nuser=%s\npassword=%s\n", + config.Username, config.Password, + ))) + return defaultsFile.Name(), err +} diff --git a/go/sql/parser.go b/go/sql/parser.go index 2ddc60f50..7a0c7a67d 100644 --- a/go/sql/parser.go +++ b/go/sql/parser.go @@ -135,6 +135,12 @@ func (this *AlterTableParser) parseAlterToken(alterToken string) { func (this *AlterTableParser) ParseAlterStatement(alterStatement string) (err error) { this.alterStatementOptions = alterStatement + for _, trimQuote := range []string{`'`, `"`} { + if strings.HasPrefix(this.alterStatementOptions, trimQuote) && strings.HasSuffix(this.alterStatementOptions, trimQuote) { + this.alterStatementOptions = strings.TrimPrefix(this.alterStatementOptions, trimQuote) + this.alterStatementOptions = strings.TrimSuffix(this.alterStatementOptions, trimQuote) + } + } for _, alterTableRegexp := range alterTableExplicitSchemaTableRegexps { if submatch := alterTableRegexp.FindStringSubmatch(this.alterStatementOptions); len(submatch) > 0 { this.explicitSchema = submatch[1] diff --git a/go/sql/parser_test.go b/go/sql/parser_test.go index df9284280..18d8da2bd 100644 --- a/go/sql/parser_test.go +++ b/go/sql/parser_test.go @@ -6,6 +6,7 @@ package sql import ( + "fmt" "reflect" "testing" @@ -18,13 +19,40 @@ func init() { } func TestParseAlterStatement(t *testing.T) { - statement := "add column t int, engine=innodb" - parser := NewAlterTableParser() - err := parser.ParseAlterStatement(statement) - test.S(t).ExpectNil(err) - test.S(t).ExpectEquals(parser.alterStatementOptions, statement) - test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) - test.S(t).ExpectFalse(parser.IsAutoIncrementDefined()) + // plain alter + { + statement := "add column t int, engine=innodb" + parser := NewAlterTableParser() + err := parser.ParseAlterStatement(statement) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) + test.S(t).ExpectFalse(parser.HasNonTrivialRenames()) + test.S(t).ExpectFalse(parser.IsAutoIncrementDefined()) + } + // single-quoted alter + { + statement := "add column t int, engine=innodb" + parser := NewAlterTableParser() + err := parser.ParseAlterStatement(fmt.Sprintf(`'%s'`, statement)) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) + } + // single-quoted w/comment alter + { + statement := "add column t int 'single-quoted comment'" + parser := NewAlterTableParser() + err := parser.ParseAlterStatement(fmt.Sprintf(`'%s'`, statement)) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) + } + // double-quoted alter + { + statement := "add column t int, engine=innodb" + parser := NewAlterTableParser() + err := parser.ParseAlterStatement(fmt.Sprintf(`"%s"`, statement)) + test.S(t).ExpectNil(err) + test.S(t).ExpectEquals(parser.alterStatementOptions, statement) + } } func TestParseAlterStatementTrivialRename(t *testing.T) { diff --git a/localtests/Dockerfile b/localtests/Dockerfile new file mode 100644 index 000000000..1246e3a93 --- /dev/null +++ b/localtests/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.17 AS build +LABEL maintainer="github@github.com" + +COPY . /go/src/github.com/github/gh-ost +WORKDIR /go/src/github.com/github/gh-ost + +RUN go build -o gh-ost go/cmd/gh-ost/main.go +RUN go build -o gh-ost-localtests go/cmd/gh-ost-localtests/main.go + + + +FROM debian:buster-slim AS image + +RUN apt-get update +RUN apt-get install -y default-mysql-client +RUN rm -rf /var/lib/apt/lists/* + +COPY --from=build /go/src/github.com/github/gh-ost/gh-ost /usr/local/bin/gh-ost +COPY --from=build /go/src/github.com/github/gh-ost/gh-ost-localtests /usr/local/bin/gh-ost-localtests +COPY --from=build /go/src/github.com/github/gh-ost/localtests /etc/localtests + +ENTRYPOINT ["gh-ost-localtests"] diff --git a/localtests/docker-compose.yml b/localtests/docker-compose.yml new file mode 100644 index 000000000..6bf5383e7 --- /dev/null +++ b/localtests/docker-compose.yml @@ -0,0 +1,25 @@ +version: "3.6" +services: + tests: + build: + context: "../" + dockerfile: "localtests/Dockerfile" + env_file: "mysql.env" + depends_on: + - "primary" + - "replica" + primary: + image: ${TEST_DOCKER_IMAGE} + command: "--bind-address=0.0.0.0 --enforce-gtid-consistency --gtid-mode=ON --event-scheduler=ON --log-bin --log-slave-updates --server-id=1" + env_file: "mysql.env" + volumes: + - "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro" + replica: + image: ${TEST_DOCKER_IMAGE} + command: "--bind-address=0.0.0.0 --enforce-gtid-consistency --gtid-mode=ON --log-bin --log-slave-updates --read-only=ON --server-id=2" + env_file: "mysql.env" + depends_on: + - "primary" + volumes: + - "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro" + - "./init-replica.sql:/docker-entrypoint-initdb.d/02-init-replica.sql:ro" diff --git a/localtests/init-replica.sql b/localtests/init-replica.sql new file mode 100644 index 000000000..b48282ada --- /dev/null +++ b/localtests/init-replica.sql @@ -0,0 +1,9 @@ +STOP SLAVE; +RESET SLAVE; +RESET MASTER; + +CHANGE MASTER TO MASTER_HOST='primary', MASTER_USER='gh-ost', MASTER_PASSWORD='gh-ost', MASTER_PORT=3306, + MASTER_AUTO_POSITION=1, MASTER_CONNECT_RETRY=1; + +START SLAVE; +SET @@GLOBAL.read_only=ON; diff --git a/localtests/init.sql b/localtests/init.sql new file mode 100644 index 000000000..daec43903 --- /dev/null +++ b/localtests/init.sql @@ -0,0 +1,5 @@ +CREATE DATABASE IF NOT EXISTS `test`; + +CREATE USER IF NOT EXISTS `gh-ost`@`%`; +SET PASSWORD FOR `gh-ost`@`%` = PASSWORD('gh-ost'); +GRANT ALL ON *.* TO `gh-ost`@`%`; diff --git a/localtests/keyword-column/extra_args b/localtests/keyword-column/extra_args index 5d73843b0..4b091d601 100644 --- a/localtests/keyword-column/extra_args +++ b/localtests/keyword-column/extra_args @@ -1 +1 @@ ---alter='add column `index` int unsigned' \ +--alter='add column `index` int unsigned' diff --git a/localtests/mysql-env.sh b/localtests/mysql-env.sh new file mode 100755 index 000000000..eb0752120 --- /dev/null +++ b/localtests/mysql-env.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +DIR=$(readlink -f $(dirname $0)) +FILE=$DIR/mysql.env + +( + echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true' + + echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}" + if [ "$TEST_STORAGE_ENGINE" == "rocksdb" ]; then + echo 'INIT_ROCKSDB=true' + fi +) | tee $FILE + +echo "Wrote env file to $FILE" diff --git a/localtests/trivial/extra_args b/localtests/trivial/extra_args index 8b6320aa1..75bbe43a4 100644 --- a/localtests/trivial/extra_args +++ b/localtests/trivial/extra_args @@ -1 +1 @@ ---throttle-query='select false' \ +--throttle-query='select false' diff --git a/vendor/github.com/google/shlex/COPYING b/vendor/github.com/google/shlex/COPYING new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/github.com/google/shlex/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/google/shlex/README b/vendor/github.com/google/shlex/README new file mode 100644 index 000000000..c86bcc066 --- /dev/null +++ b/vendor/github.com/google/shlex/README @@ -0,0 +1,2 @@ +go-shlex is a simple lexer for go that supports shell-style quoting, +commenting, and escaping. diff --git a/vendor/github.com/google/shlex/shlex.go b/vendor/github.com/google/shlex/shlex.go new file mode 100644 index 000000000..d98308bce --- /dev/null +++ b/vendor/github.com/google/shlex/shlex.go @@ -0,0 +1,416 @@ +/* +Copyright 2012 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package shlex implements a simple lexer which splits input in to tokens using +shell-style rules for quoting and commenting. + +The basic use case uses the default ASCII lexer to split a string into sub-strings: + + shlex.Split("one \"two three\" four") -> []string{"one", "two three", "four"} + +To process a stream of strings: + + l := NewLexer(os.Stdin) + for ; token, err := l.Next(); err != nil { + // process token + } + +To access the raw token stream (which includes tokens for comments): + + t := NewTokenizer(os.Stdin) + for ; token, err := t.Next(); err != nil { + // process token + } + +*/ +package shlex + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +// TokenType is a top-level token classification: A word, space, comment, unknown. +type TokenType int + +// runeTokenClass is the type of a UTF-8 character classification: A quote, space, escape. +type runeTokenClass int + +// the internal state used by the lexer state machine +type lexerState int + +// Token is a (type, value) pair representing a lexographical token. +type Token struct { + tokenType TokenType + value string +} + +// Equal reports whether tokens a, and b, are equal. +// Two tokens are equal if both their types and values are equal. A nil token can +// never be equal to another token. +func (a *Token) Equal(b *Token) bool { + if a == nil || b == nil { + return false + } + if a.tokenType != b.tokenType { + return false + } + return a.value == b.value +} + +// Named classes of UTF-8 runes +const ( + spaceRunes = " \t\r\n" + escapingQuoteRunes = `"` + nonEscapingQuoteRunes = "'" + escapeRunes = `\` + commentRunes = "#" +) + +// Classes of rune token +const ( + unknownRuneClass runeTokenClass = iota + spaceRuneClass + escapingQuoteRuneClass + nonEscapingQuoteRuneClass + escapeRuneClass + commentRuneClass + eofRuneClass +) + +// Classes of lexographic token +const ( + UnknownToken TokenType = iota + WordToken + SpaceToken + CommentToken +) + +// Lexer state machine states +const ( + startState lexerState = iota // no runes have been seen + inWordState // processing regular runes in a word + escapingState // we have just consumed an escape rune; the next rune is literal + escapingQuotedState // we have just consumed an escape rune within a quoted string + quotingEscapingState // we are within a quoted string that supports escaping ("...") + quotingState // we are within a string that does not support escaping ('...') + commentState // we are within a comment (everything following an unquoted or unescaped # +) + +// tokenClassifier is used for classifying rune characters. +type tokenClassifier map[rune]runeTokenClass + +func (typeMap tokenClassifier) addRuneClass(runes string, tokenType runeTokenClass) { + for _, runeChar := range runes { + typeMap[runeChar] = tokenType + } +} + +// newDefaultClassifier creates a new classifier for ASCII characters. +func newDefaultClassifier() tokenClassifier { + t := tokenClassifier{} + t.addRuneClass(spaceRunes, spaceRuneClass) + t.addRuneClass(escapingQuoteRunes, escapingQuoteRuneClass) + t.addRuneClass(nonEscapingQuoteRunes, nonEscapingQuoteRuneClass) + t.addRuneClass(escapeRunes, escapeRuneClass) + t.addRuneClass(commentRunes, commentRuneClass) + return t +} + +// ClassifyRune classifiees a rune +func (t tokenClassifier) ClassifyRune(runeVal rune) runeTokenClass { + return t[runeVal] +} + +// Lexer turns an input stream into a sequence of tokens. Whitespace and comments are skipped. +type Lexer Tokenizer + +// NewLexer creates a new lexer from an input stream. +func NewLexer(r io.Reader) *Lexer { + + return (*Lexer)(NewTokenizer(r)) +} + +// Next returns the next word, or an error. If there are no more words, +// the error will be io.EOF. +func (l *Lexer) Next() (string, error) { + for { + token, err := (*Tokenizer)(l).Next() + if err != nil { + return "", err + } + switch token.tokenType { + case WordToken: + return token.value, nil + case CommentToken: + // skip comments + default: + return "", fmt.Errorf("Unknown token type: %v", token.tokenType) + } + } +} + +// Tokenizer turns an input stream into a sequence of typed tokens +type Tokenizer struct { + input bufio.Reader + classifier tokenClassifier +} + +// NewTokenizer creates a new tokenizer from an input stream. +func NewTokenizer(r io.Reader) *Tokenizer { + input := bufio.NewReader(r) + classifier := newDefaultClassifier() + return &Tokenizer{ + input: *input, + classifier: classifier} +} + +// scanStream scans the stream for the next token using the internal state machine. +// It will panic if it encounters a rune which it does not know how to handle. +func (t *Tokenizer) scanStream() (*Token, error) { + state := startState + var tokenType TokenType + var value []rune + var nextRune rune + var nextRuneType runeTokenClass + var err error + + for { + nextRune, _, err = t.input.ReadRune() + nextRuneType = t.classifier.ClassifyRune(nextRune) + + if err == io.EOF { + nextRuneType = eofRuneClass + err = nil + } else if err != nil { + return nil, err + } + + switch state { + case startState: // no runes read yet + { + switch nextRuneType { + case eofRuneClass: + { + return nil, io.EOF + } + case spaceRuneClass: + { + } + case escapingQuoteRuneClass: + { + tokenType = WordToken + state = quotingEscapingState + } + case nonEscapingQuoteRuneClass: + { + tokenType = WordToken + state = quotingState + } + case escapeRuneClass: + { + tokenType = WordToken + state = escapingState + } + case commentRuneClass: + { + tokenType = CommentToken + state = commentState + } + default: + { + tokenType = WordToken + value = append(value, nextRune) + state = inWordState + } + } + } + case inWordState: // in a regular word + { + switch nextRuneType { + case eofRuneClass: + { + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case spaceRuneClass: + { + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case escapingQuoteRuneClass: + { + state = quotingEscapingState + } + case nonEscapingQuoteRuneClass: + { + state = quotingState + } + case escapeRuneClass: + { + state = escapingState + } + default: + { + value = append(value, nextRune) + } + } + } + case escapingState: // the rune after an escape character + { + switch nextRuneType { + case eofRuneClass: + { + err = fmt.Errorf("EOF found after escape character") + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + default: + { + state = inWordState + value = append(value, nextRune) + } + } + } + case escapingQuotedState: // the next rune after an escape character, in double quotes + { + switch nextRuneType { + case eofRuneClass: + { + err = fmt.Errorf("EOF found after escape character") + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + default: + { + state = quotingEscapingState + value = append(value, nextRune) + } + } + } + case quotingEscapingState: // in escaping double quotes + { + switch nextRuneType { + case eofRuneClass: + { + err = fmt.Errorf("EOF found when expecting closing quote") + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case escapingQuoteRuneClass: + { + state = inWordState + } + case escapeRuneClass: + { + state = escapingQuotedState + } + default: + { + value = append(value, nextRune) + } + } + } + case quotingState: // in non-escaping single quotes + { + switch nextRuneType { + case eofRuneClass: + { + err = fmt.Errorf("EOF found when expecting closing quote") + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case nonEscapingQuoteRuneClass: + { + state = inWordState + } + default: + { + value = append(value, nextRune) + } + } + } + case commentState: // in a comment + { + switch nextRuneType { + case eofRuneClass: + { + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } + case spaceRuneClass: + { + if nextRune == '\n' { + state = startState + token := &Token{ + tokenType: tokenType, + value: string(value)} + return token, err + } else { + value = append(value, nextRune) + } + } + default: + { + value = append(value, nextRune) + } + } + } + default: + { + return nil, fmt.Errorf("Unexpected state: %v", state) + } + } + } +} + +// Next returns the next token in the stream. +func (t *Tokenizer) Next() (*Token, error) { + return t.scanStream() +} + +// Split partitions a string into a slice of strings. +func Split(s string) ([]string, error) { + l := NewLexer(strings.NewReader(s)) + subStrings := make([]string, 0) + for { + word, err := l.Next() + if err != nil { + if err == io.EOF { + return subStrings, nil + } + return subStrings, err + } + subStrings = append(subStrings, word) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6f7639b77..2cfe1d69f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -11,6 +11,9 @@ github.com/go-mysql-org/go-mysql/utils # github.com/go-sql-driver/mysql v1.6.0 ## explicit; go 1.10 github.com/go-sql-driver/mysql +# github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 +## explicit; go 1.13 +github.com/google/shlex # github.com/openark/golib v0.0.0-20210531070646-355f37940af8 ## explicit; go 1.16 github.com/openark/golib/log @@ -37,8 +40,6 @@ github.com/siddontang/go-log/loggers # go.uber.org/atomic v1.7.0 ## explicit; go 1.13 go.uber.org/atomic -# golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 -## explicit; go 1.11 # golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 ## explicit; go 1.11 golang.org/x/net/context From 34646f553d8f248d6885ed082cc74822e164f680 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 01:56:07 +0100 Subject: [PATCH 02/22] Add collapsible stdout --- go/localtests/test.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index 1ed659f10..ae9b13ed1 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -1,11 +1,15 @@ package localtests import ( + "bufio" "bytes" "database/sql" "fmt" + "io" "log" + "os" "os/exec" + "strings" ) // Test represents a single test. @@ -126,12 +130,22 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { cmd.Stderr = &stderr cmd.Stdout = &output + if strings.TrimSpace(os.Getenv("GITHUB_ACTION")) == "true" { + go func(reader io.Reader) { + scanner := bufio.NewScanner(output) + fmt.Printf("::group::%s stdout\n", test.Name) + for scanner.Scan() { + fmt.Println(scanner.Text()) + } + fmt.Println("::endgroup::") + }(&output) + } + if err = cmd.Run(); err != nil { if isExpectedFailureOutput(&stderr, test.ExpectedFailure) { return nil } - output.Write(stderr.Bytes()) - log.Printf("[%s] test failed: %+v", test.Name, output.String()) + log.Printf("[%s] test failed: %+v", test.Name, stderr.String()) } return err } From 94c27becc98d38c3a976639885ef5facbc517cfc Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 01:58:50 +0100 Subject: [PATCH 03/22] Lint --- go/localtests/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index ae9b13ed1..c31800d19 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -132,7 +132,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { if strings.TrimSpace(os.Getenv("GITHUB_ACTION")) == "true" { go func(reader io.Reader) { - scanner := bufio.NewScanner(output) + scanner := bufio.NewScanner(&output) fmt.Printf("::group::%s stdout\n", test.Name) for scanner.Scan() { fmt.Println(scanner.Text()) From 9c253cbdda60cea0f1f54bec557e4f1fd1ac2e15 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 02:00:12 +0100 Subject: [PATCH 04/22] Fix cond --- go/localtests/test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index c31800d19..9f2e14410 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -130,7 +130,9 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { cmd.Stderr = &stderr cmd.Stdout = &output - if strings.TrimSpace(os.Getenv("GITHUB_ACTION")) == "true" { + // group stdout log lines if running in GitHub Actions + // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines + if strings.TrimSpace(os.Getenv("GITHUB_ACTION")) != "" { go func(reader io.Reader) { scanner := bufio.NewScanner(&output) fmt.Printf("::group::%s stdout\n", test.Name) From 807c052e039f482fbcccbefc0f59c92ebae69560 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 02:00:56 +0100 Subject: [PATCH 05/22] Lint 2 --- go/localtests/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index 9f2e14410..20ef6f23f 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -134,7 +134,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines if strings.TrimSpace(os.Getenv("GITHUB_ACTION")) != "" { go func(reader io.Reader) { - scanner := bufio.NewScanner(&output) + scanner := bufio.NewScanner(reader) fmt.Printf("::group::%s stdout\n", test.Name) for scanner.Scan() { fmt.Println(scanner.Text()) From e1ce2340c1e5fe90223c9db52975555a7ac2099e Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 02:04:10 +0100 Subject: [PATCH 06/22] Add GITHUB_ACTION env --- localtests/mysql-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localtests/mysql-env.sh b/localtests/mysql-env.sh index eb0752120..7fad95e60 100755 --- a/localtests/mysql-env.sh +++ b/localtests/mysql-env.sh @@ -4,8 +4,8 @@ DIR=$(readlink -f $(dirname $0)) FILE=$DIR/mysql.env ( + echo "GITHUB_ACTION=${GITHUB_ACTION}" echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true' - echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}" if [ "$TEST_STORAGE_ENGINE" == "rocksdb" ]; then echo 'INIT_ROCKSDB=true' From 4a59304fde73599514ce77d26cc287982dcecf39 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 02:10:53 +0100 Subject: [PATCH 07/22] Fix output --- go/localtests/test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index 20ef6f23f..537abfaf7 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -125,9 +125,9 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { log.Printf("[%s] running gh-ost command with extra args: %+v", test.Name, test.ExtraArgs) - var output, stderr bytes.Buffer + var output bytes.Buffer cmd := exec.Command(config.GhostBinary, flags...) - cmd.Stderr = &stderr + cmd.Stderr = &output cmd.Stdout = &output // group stdout log lines if running in GitHub Actions @@ -147,7 +147,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { if isExpectedFailureOutput(&stderr, test.ExpectedFailure) { return nil } - log.Printf("[%s] test failed: %+v", test.Name, stderr.String()) + log.Printf("[%s] test failed", test.Name) } return err } From bb2b2549659e34d64870b1d8a5aeb041d4f166a6 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 02:12:26 +0100 Subject: [PATCH 08/22] Fix output 2 --- go/localtests/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index 537abfaf7..0ddb70b1f 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -144,7 +144,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { } if err = cmd.Run(); err != nil { - if isExpectedFailureOutput(&stderr, test.ExpectedFailure) { + if isExpectedFailureOutput(&output, test.ExpectedFailure) { return nil } log.Printf("[%s] test failed", test.Name) From 9ae65f4dcb5ace8db1133a1f70be93a00fef5011 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 02:17:17 +0100 Subject: [PATCH 09/22] Fix output 3 --- go/localtests/test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index 0ddb70b1f..91ab34d66 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -130,6 +130,11 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { cmd.Stderr = &output cmd.Stdout = &output + errChan := make(chan error) + go func() { + errChan <- cmd.Run() + }() + // group stdout log lines if running in GitHub Actions // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines if strings.TrimSpace(os.Getenv("GITHUB_ACTION")) != "" { @@ -143,7 +148,8 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { }(&output) } - if err = cmd.Run(); err != nil { + err <- errChan + if err != nil { if isExpectedFailureOutput(&output, test.ExpectedFailure) { return nil } From 7b63b82029948bf043c7f77ccbf9043a8b974472 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 02:18:44 +0100 Subject: [PATCH 10/22] Fix output 4 --- go/localtests/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index 91ab34d66..58d67cda9 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -148,7 +148,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { }(&output) } - err <- errChan + err = <-errChan if err != nil { if isExpectedFailureOutput(&output, test.ExpectedFailure) { return nil From bae5a55db1735fe2b053263282a99a249dbcd409 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 03:06:07 +0100 Subject: [PATCH 11/22] Fix output 5 --- go/localtests/test.go | 60 +++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index 58d67cda9..8d02a68b5 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -5,13 +5,16 @@ import ( "bytes" "database/sql" "fmt" - "io" "log" "os" "os/exec" - "strings" + "sync" ) +func isGitHubActions() bool { + return os.Getenv("GITHUB_ACTION") != "" +} + // Test represents a single test. type Test struct { Name string @@ -125,35 +128,48 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { log.Printf("[%s] running gh-ost command with extra args: %+v", test.Name, test.ExtraArgs) - var output bytes.Buffer + var stderr, stdout bytes.Buffer cmd := exec.Command(config.GhostBinary, flags...) - cmd.Stderr = &output - cmd.Stdout = &output + cmd.Stderr = &stderr + cmd.Stdout = &stdout + + if err = cmd.Start(); err != nil { + return err + } - errChan := make(chan error) + var wg sync.WaitGroup + stop := make(chan bool) go func() { - errChan <- cmd.Run() + defer wg.Done() + //if isGitHubActions() { + fmt.Printf("::group::%s stdout\n", test.Name) + //} + for { + select { + case <-stop: + //if isGitHubActions() { + fmt.Println("::endgroup::") + //} + return + default: + scanner := bufio.NewScanner(&stdout) + for scanner.Scan() { + fmt.Println(scanner.Text()) + } + } + } }() + wg.Add(1) - // group stdout log lines if running in GitHub Actions - // https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines - if strings.TrimSpace(os.Getenv("GITHUB_ACTION")) != "" { - go func(reader io.Reader) { - scanner := bufio.NewScanner(reader) - fmt.Printf("::group::%s stdout\n", test.Name) - for scanner.Scan() { - fmt.Println(scanner.Text()) - } - fmt.Println("::endgroup::") - }(&output) - } + err = cmd.Wait() + stop <- true + wg.Wait() - err = <-errChan if err != nil { - if isExpectedFailureOutput(&output, test.ExpectedFailure) { + if isExpectedFailureOutput(&stderr, test.ExpectedFailure) { return nil } - log.Printf("[%s] test failed", test.Name) + log.Printf("[%s] test failed: %+v", test.Name, stderr.String()) } return err } From c11b1939c244e42e997c2f97e41c509d487ea323 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 03:14:09 +0100 Subject: [PATCH 12/22] Fix output 6 --- go/localtests/test.go | 23 +++++++++-------------- go/localtests/utils.go | 4 ++++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index 8d02a68b5..a9d1070ad 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -6,15 +6,10 @@ import ( "database/sql" "fmt" "log" - "os" "os/exec" "sync" ) -func isGitHubActions() bool { - return os.Getenv("GITHUB_ACTION") != "" -} - // Test represents a single test. type Test struct { Name string @@ -138,18 +133,18 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { } var wg sync.WaitGroup - stop := make(chan bool) + stopStdout := make(chan bool) go func() { defer wg.Done() - //if isGitHubActions() { - fmt.Printf("::group::%s stdout\n", test.Name) - //} + if isGitHubActions() { + fmt.Printf("::group::%s stdout\n", test.Name) + } for { select { - case <-stop: - //if isGitHubActions() { - fmt.Println("::endgroup::") - //} + case <-stopStdout: + if isGitHubActions() { + fmt.Println("::endgroup::") + } return default: scanner := bufio.NewScanner(&stdout) @@ -162,7 +157,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { wg.Add(1) err = cmd.Wait() - stop <- true + stopStdout <- true wg.Wait() if err != nil { diff --git a/go/localtests/utils.go b/go/localtests/utils.go index ff227c4a5..0fd1c6e95 100644 --- a/go/localtests/utils.go +++ b/go/localtests/utils.go @@ -85,6 +85,10 @@ func isExpectedFailureOutput(output io.Reader, expectedFailure string) bool { return false } +func isGitHubActions() bool { + return os.Getenv("GITHUB_ACTION") != "" +} + func pingAndGetGTIDExecuted(db *sql.DB, timeout time.Duration) (*mysql.UUIDSet, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() From 333342d1e352fe143dc6e7818bcaad094d437b8e Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 03:26:22 +0100 Subject: [PATCH 13/22] Tweak --- go/localtests/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index a9d1070ad..add169fad 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -148,7 +148,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { return default: scanner := bufio.NewScanner(&stdout) - for scanner.Scan() { + if err := scanner.Scan(); err == nil { fmt.Println(scanner.Text()) } } From 4ccef404c3de921bdf74de4c7410daffaaa9ddd5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 03:27:02 +0100 Subject: [PATCH 14/22] Tweak 2 --- go/localtests/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index add169fad..c974adb86 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -148,7 +148,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { return default: scanner := bufio.NewScanner(&stdout) - if err := scanner.Scan(); err == nil { + if found := scanner.Scan(); found { fmt.Println(scanner.Text()) } } From 48349e947fcaa6f035cbbe9139dc3ec0d31b38d5 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sun, 4 Dec 2022 03:29:34 +0100 Subject: [PATCH 15/22] Tweak undo --- go/localtests/test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index c974adb86..a9d1070ad 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -148,7 +148,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { return default: scanner := bufio.NewScanner(&stdout) - if found := scanner.Scan(); found { + for scanner.Scan() { fmt.Println(scanner.Text()) } } From 5e981a7283a826c04c025366567ddda0ecd742dd Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 8 Dec 2022 23:06:42 +0100 Subject: [PATCH 16/22] Add wrapper script --- .github/workflows/replica-tests.yml | 10 ++-------- go/localtests/test.go | 2 ++ localtests/docker-compose.yml | 1 - localtests/tests.sh | 31 +++++++++++++++++++++++++++++ 4 files changed, 35 insertions(+), 9 deletions(-) create mode 100755 localtests/tests.sh diff --git a/.github/workflows/replica-tests.yml b/.github/workflows/replica-tests.yml index 9195c545c..67de2f167 100644 --- a/.github/workflows/replica-tests.yml +++ b/.github/workflows/replica-tests.yml @@ -4,7 +4,6 @@ on: jobs: build: - runs-on: ubuntu-20.04 strategy: matrix: @@ -25,13 +24,8 @@ jobs: steps: - uses: actions/checkout@v2 - - name: generate mysql environment file - env: - TEST_STORAGE_ENGINE: "${{ matrix.tests.engine }}" - run: localtests/mysql-env.sh - - name: run localtests env: TEST_DOCKER_IMAGE: "${{ matrix.tests.image }}" - run: docker-compose -f localtests/docker-compose.yml up --abort-on-container-exit --no-log-prefix tests - + TEST_STORAGE_ENGINE: "${{ matrix.tests.engine }}" + run: localtests/tests.sh diff --git a/go/localtests/test.go b/go/localtests/test.go index a9d1070ad..7cc1837e7 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -136,9 +136,11 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { stopStdout := make(chan bool) go func() { defer wg.Done() + if isGitHubActions() { fmt.Printf("::group::%s stdout\n", test.Name) } + for { select { case <-stopStdout: diff --git a/localtests/docker-compose.yml b/localtests/docker-compose.yml index 6bf5383e7..8c75793a6 100644 --- a/localtests/docker-compose.yml +++ b/localtests/docker-compose.yml @@ -3,7 +3,6 @@ services: tests: build: context: "../" - dockerfile: "localtests/Dockerfile" env_file: "mysql.env" depends_on: - "primary" diff --git a/localtests/tests.sh b/localtests/tests.sh new file mode 100755 index 000000000..f53e8dd6f --- /dev/null +++ b/localtests/tests.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e + +export TEST_DOCKER_IMAGE=${1:-mysql:5.7} + +LOCALTESTS_DIR="$(readlink -f $(dirname $0))" +cd $LOCALTESTS_DIR + +# generate mysql.env file for containers +[ ! -z "$GITHUB_ACTION" ] && echo "::group::generate mysql env" +./mysql-env.sh +[ ! -z "$GITHUB_ACTION" ] && echo "::endgroup::" + +# conditional pre-build +EXTRA_UP_FLAGS= +if [ -z "$GITHUB_ACTION" ]; then + EXTRA_UP_FLAGS="--build" +else + echo "::group::docker image build" + docker-compose build + echo "::endgroup::" +fi + +# this will start the test container and the +# mysql primary/replica w/docker-compose +docker-compose up \ + --abort-on-container-exit \ + --no-log-prefix \ + $EXTRA_UP_FLAGS \ + tests From 8c0677dffe2a8b7ff90661d3cef1823add701718 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Thu, 8 Dec 2022 23:31:36 +0100 Subject: [PATCH 17/22] Fix docker env --- localtests/docker-compose.yml | 1 + localtests/mysql-env.sh | 15 --------------- localtests/tests.sh | 15 +++++++++++---- 3 files changed, 12 insertions(+), 19 deletions(-) delete mode 100755 localtests/mysql-env.sh diff --git a/localtests/docker-compose.yml b/localtests/docker-compose.yml index 8c75793a6..6bf5383e7 100644 --- a/localtests/docker-compose.yml +++ b/localtests/docker-compose.yml @@ -3,6 +3,7 @@ services: tests: build: context: "../" + dockerfile: "localtests/Dockerfile" env_file: "mysql.env" depends_on: - "primary" diff --git a/localtests/mysql-env.sh b/localtests/mysql-env.sh deleted file mode 100755 index 7fad95e60..000000000 --- a/localtests/mysql-env.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -DIR=$(readlink -f $(dirname $0)) -FILE=$DIR/mysql.env - -( - echo "GITHUB_ACTION=${GITHUB_ACTION}" - echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true' - echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}" - if [ "$TEST_STORAGE_ENGINE" == "rocksdb" ]; then - echo 'INIT_ROCKSDB=true' - fi -) | tee $FILE - -echo "Wrote env file to $FILE" diff --git a/localtests/tests.sh b/localtests/tests.sh index f53e8dd6f..eb82c5718 100755 --- a/localtests/tests.sh +++ b/localtests/tests.sh @@ -5,11 +5,18 @@ set -e export TEST_DOCKER_IMAGE=${1:-mysql:5.7} LOCALTESTS_DIR="$(readlink -f $(dirname $0))" -cd $LOCALTESTS_DIR # generate mysql.env file for containers [ ! -z "$GITHUB_ACTION" ] && echo "::group::generate mysql env" -./mysql-env.sh +( + echo "GITHUB_ACTION=${GITHUB_ACTION}" + echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true' + echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}" + if [ "$TEST_STORAGE_ENGINE" == "rocksdb" ]; then + echo 'INIT_ROCKSDB=true' + fi +) | tee $LOCALTESTS_DIR/mysql.env +echo "Wrote env file to $LOCALTESTS_DIR/mysql.env" [ ! -z "$GITHUB_ACTION" ] && echo "::endgroup::" # conditional pre-build @@ -18,13 +25,13 @@ if [ -z "$GITHUB_ACTION" ]; then EXTRA_UP_FLAGS="--build" else echo "::group::docker image build" - docker-compose build + docker-compose -f localtests/docker-compose.yml build echo "::endgroup::" fi # this will start the test container and the # mysql primary/replica w/docker-compose -docker-compose up \ +TEST_DOCKER_IMAGE=mysql:5.7 docker-compose -f localtests/docker-compose.yml up \ --abort-on-container-exit \ --no-log-prefix \ $EXTRA_UP_FLAGS \ From 00b8b48518e62a609b08aa4947ae149be8ac8361 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 10 Dec 2022 00:01:55 +0100 Subject: [PATCH 18/22] Fix logging --- go/localtests/test.go | 47 ++++++++++++----------------------------- go/localtests/tester.go | 16 +++++++++----- 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/go/localtests/test.go b/go/localtests/test.go index 7cc1837e7..25c32ff3d 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -1,13 +1,13 @@ package localtests import ( - "bufio" "bytes" "database/sql" "fmt" + "io" "log" + "os" "os/exec" - "sync" ) // Test represents a single test. @@ -39,6 +39,7 @@ func (test *Test) prepareDBPrimary(primary *sql.DB) (err error) { } log.Printf("[%s] sql_mode set to %q on primary", test.Name, *test.SQLMode) } + return err } @@ -123,44 +124,24 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { log.Printf("[%s] running gh-ost command with extra args: %+v", test.Name, test.ExtraArgs) - var stderr, stdout bytes.Buffer + var stderr bytes.Buffer cmd := exec.Command(config.GhostBinary, flags...) - cmd.Stderr = &stderr - cmd.Stdout = &stdout + cmd.Stderr = io.MultiWriter(os.Stderr, &stderr) + cmd.Stdout = os.Stdout + + if isGitHubActions() { + fmt.Printf("::group::%s gh-ost output\n", test.Name) + } if err = cmd.Start(); err != nil { return err } - var wg sync.WaitGroup - stopStdout := make(chan bool) - go func() { - defer wg.Done() - - if isGitHubActions() { - fmt.Printf("::group::%s stdout\n", test.Name) - } - - for { - select { - case <-stopStdout: - if isGitHubActions() { - fmt.Println("::endgroup::") - } - return - default: - scanner := bufio.NewScanner(&stdout) - for scanner.Scan() { - fmt.Println(scanner.Text()) - } - } - } - }() - wg.Add(1) - err = cmd.Wait() - stopStdout <- true - wg.Wait() + + if isGitHubActions() { + fmt.Println("::endgroup::") + } if err != nil { if isExpectedFailureOutput(&stderr, test.ExpectedFailure) { diff --git a/go/localtests/tester.go b/go/localtests/tester.go index fc9359c74..fe302bfbb 100644 --- a/go/localtests/tester.go +++ b/go/localtests/tester.go @@ -3,7 +3,6 @@ package localtests import ( "database/sql" "errors" - "fmt" "io/ioutil" "log" "os" @@ -81,9 +80,7 @@ func (t *Tester) WaitForMySQLAvailable() error { } if !replicaGTIDExec.Contain(primaryGTIDExec) { - return fmt.Errorf("Replica/primary GTID not equal: %s != %s", - replicaGTIDExec.String(), primaryGTIDExec.String(), - ) + return errors.New("replica/primary GTID not equal") } return nil }(); err != nil { @@ -91,7 +88,16 @@ func (t *Tester) WaitForMySQLAvailable() error { continue } - log.Printf("MySQL primary/replica is available %s", successEmoji) + if info, err := getMysqlHostInfo(t.primary); err == nil { + log.Printf("MySQL primary %s:%d, version %s (%s) available %s", info.Host, info.Port, info.Version, + info.VersionComment, successEmoji) + } + + if info, err := getMysqlHostInfo(t.replica); err == nil { + log.Printf("MySQL replica %s:%d, version %s (%s) available %s", info.Host, info.Port, info.Version, + info.VersionComment, successEmoji) + } + return nil } } From fd12c49f4464bfdfb454cd87cc40b1a0a8b4eafa Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 10 Dec 2022 00:10:51 +0100 Subject: [PATCH 19/22] Fix output again --- go/localtests/tester.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/go/localtests/tester.go b/go/localtests/tester.go index fe302bfbb..2dc897ed1 100644 --- a/go/localtests/tester.go +++ b/go/localtests/tester.go @@ -69,7 +69,7 @@ func (t *Tester) WaitForMySQLAvailable() error { case <-time.After(5 * time.Minute): return errors.New("timed out waiting for mysql") case <-ticker.C: - if err := func() error { + err := func() error { primaryGTIDExec, err := pingAndGetGTIDExecuted(t.primary, interval) if err != nil { return err @@ -83,22 +83,23 @@ func (t *Tester) WaitForMySQLAvailable() error { return errors.New("replica/primary GTID not equal") } return nil - }(); err != nil { + }() + if err != nil { log.Printf("Waiting for MySQL primary/replica to init: %+v", err) continue } - if info, err := getMysqlHostInfo(t.primary); err == nil { + var info *MysqlInfo + if info, err = getMysqlHostInfo(t.primary); err == nil { log.Printf("MySQL primary %s:%d, version %s (%s) available %s", info.Host, info.Port, info.Version, info.VersionComment, successEmoji) } - - if info, err := getMysqlHostInfo(t.replica); err == nil { + if info, err = getMysqlHostInfo(t.replica); err == nil { log.Printf("MySQL replica %s:%d, version %s (%s) available %s", info.Host, info.Port, info.Version, info.VersionComment, successEmoji) } - return nil + return err } } } From c45bd5dbef7fa6e295f73d4db46f934c6545fbd0 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 10 Dec 2022 00:11:40 +0100 Subject: [PATCH 20/22] Delete localtests/test.sh --- localtests/test.sh | 289 --------------------------------------------- 1 file changed, 289 deletions(-) delete mode 100755 localtests/test.sh diff --git a/localtests/test.sh b/localtests/test.sh deleted file mode 100755 index 14ecd83ea..000000000 --- a/localtests/test.sh +++ /dev/null @@ -1,289 +0,0 @@ -#!/bin/bash - -# Local integration tests. To be used by CI. -# See https://github.com/github/gh-ost/tree/doc/local-tests.md -# - -# Usage: localtests/test/sh [filter] -# By default, runs all tests. Given filter, will only run tests matching given regep - -tests_path=$(dirname $0) -test_logfile=/tmp/gh-ost-test.log -default_ghost_binary=/tmp/gh-ost-test -ghost_binary="" -storage_engine=innodb -exec_command_file=/tmp/gh-ost-test.bash -ghost_structure_output_file=/tmp/gh-ost-test.ghost.structure.sql -orig_content_output_file=/tmp/gh-ost-test.orig.content.csv -ghost_content_output_file=/tmp/gh-ost-test.ghost.content.csv -throttle_flag_file=/tmp/gh-ost-test.ghost.throttle.flag - -master_host= -master_port= -replica_host= -replica_port= -original_sql_mode= - -OPTIND=1 -while getopts "b:s:" OPTION -do - case $OPTION in - b) - ghost_binary="$OPTARG";; - s) - storage_engine="$OPTARG";; - esac -done -shift $((OPTIND-1)) - -test_pattern="${1:-.}" - -verify_master_and_replica() { - if [ "$(gh-ost-test-mysql-master -e "select 1" -ss)" != "1" ] ; then - echo "Cannot verify gh-ost-test-mysql-master" - exit 1 - fi - read master_host master_port <<< $(gh-ost-test-mysql-master -e "select @@hostname, @@port" -ss) - [ "$master_host" == "$(hostname)" ] && master_host="127.0.0.1" - echo "# master verified at $master_host:$master_port" - if ! gh-ost-test-mysql-master -e "set global event_scheduler := 1" ; then - echo "Cannot enable event_scheduler on master" - exit 1 - fi - original_sql_mode="$(gh-ost-test-mysql-master -e "select @@global.sql_mode" -s -s)" - echo "sql_mode on master is ${original_sql_mode}" - - echo "Gracefully sleeping for 3 seconds while replica is setting up..." - sleep 3 - - if [ "$(gh-ost-test-mysql-replica -e "select 1" -ss)" != "1" ] ; then - echo "Cannot verify gh-ost-test-mysql-replica" - exit 1 - fi - if [ "$(gh-ost-test-mysql-replica -e "select @@global.binlog_format" -ss)" != "ROW" ] ; then - echo "Expecting test replica to have binlog_format=ROW" - exit 1 - fi - read replica_host replica_port <<< $(gh-ost-test-mysql-replica -e "select @@hostname, @@port" -ss) - [ "$replica_host" == "$(hostname)" ] && replica_host="127.0.0.1" - echo "# replica verified at $replica_host:$replica_port" -} - -exec_cmd() { - echo "$@" - command "$@" 1> $test_logfile 2>&1 - return $? -} - -echo_dot() { - echo -n "." -} - -start_replication() { - gh-ost-test-mysql-replica -e "stop slave; start slave;" - num_attempts=0 - while gh-ost-test-mysql-replica -e "show slave status\G" | grep Seconds_Behind_Master | grep -q NULL ; do - ((num_attempts=num_attempts+1)) - if [ $num_attempts -gt 10 ] ; then - echo - echo "ERROR replication failure" - exit 1 - fi - echo_dot - sleep 1 - done -} - -test_single() { - local test_name - test_name="$1" - - if [ -f $tests_path/$test_name/ignore_versions ] ; then - ignore_versions=$(cat $tests_path/$test_name/ignore_versions) - mysql_version=$(gh-ost-test-mysql-master -s -s -e "select @@version") - mysql_version_comment=$(gh-ost-test-mysql-master -s -s -e "select @@version_comment") - if echo "$mysql_version" | egrep -q "^${ignore_versions}" ; then - echo -n "Skipping: $test_name" - return 0 - elif echo "$mysql_version_comment" | egrep -i -q "^${ignore_versions}" ; then - echo -n "Skipping: $test_name" - return 0 - fi - fi - - echo -n "Testing: $test_name" - - echo_dot - start_replication - echo_dot - - if [ -f $tests_path/$test_name/sql_mode ] ; then - gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='$(cat $tests_path/$test_name/sql_mode)'" - fi - - gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/create.sql - test_create_result=$? - - if [ $test_create_result -ne 0 ] ; then - echo - echo "ERROR $test_name create failure. cat $tests_path/$test_name/create.sql:" - cat $tests_path/$test_name/create.sql - return 1 - fi - - extra_args="" - if [ -f $tests_path/$test_name/extra_args ] ; then - extra_args=$(cat $tests_path/$test_name/extra_args) - fi - orig_columns="*" - ghost_columns="*" - order_by="" - if [ -f $tests_path/$test_name/orig_columns ] ; then - orig_columns=$(cat $tests_path/$test_name/orig_columns) - fi - if [ -f $tests_path/$test_name/ghost_columns ] ; then - ghost_columns=$(cat $tests_path/$test_name/ghost_columns) - fi - if [ -f $tests_path/$test_name/order_by ] ; then - order_by="order by $(cat $tests_path/$test_name/order_by)" - fi - # graceful sleep for replica to catch up - echo_dot - sleep 1 - # - cmd="$ghost_binary \ - --user=gh-ost \ - --password=gh-ost \ - --host=$replica_host \ - --port=$replica_port \ - --assume-master-host=${master_host}:${master_port} - --database=test \ - --table=gh_ost_test \ - --storage-engine=${storage_engine} \ - --alter='engine=${storage_engine}' \ - --exact-rowcount \ - --assume-rbr \ - --initially-drop-old-table \ - --initially-drop-ghost-table \ - --throttle-query='select timestampdiff(second, min(last_update), now()) < 5 from _gh_ost_test_ghc' \ - --throttle-flag-file=$throttle_flag_file \ - --serve-socket-file=/tmp/gh-ost.test.sock \ - --initially-drop-socket-file \ - --test-on-replica \ - --default-retries=3 \ - --chunk-size=10 \ - --verbose \ - --debug \ - --stack \ - --execute ${extra_args[@]}" - echo_dot - echo $cmd > $exec_command_file - echo_dot - bash $exec_command_file 1> $test_logfile 2>&1 - - execution_result=$? - - if [ -f $tests_path/$test_name/sql_mode ] ; then - gh-ost-test-mysql-master --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'" - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "set @@global.sql_mode='${original_sql_mode}'" - fi - - if [ -f $tests_path/$test_name/destroy.sql ] ; then - gh-ost-test-mysql-master --default-character-set=utf8mb4 test < $tests_path/$test_name/destroy.sql - fi - - if [ -f $tests_path/$test_name/expect_failure ] ; then - if [ $execution_result -eq 0 ] ; then - echo - echo "ERROR $test_name execution was expected to exit on error but did not. cat $test_logfile" - return 1 - fi - if [ -s $tests_path/$test_name/expect_failure ] ; then - # 'expect_failure' file has content. We expect to find this content in the log. - expected_error_message="$(cat $tests_path/$test_name/expect_failure)" - if grep -q "$expected_error_message" $test_logfile ; then - return 0 - fi - echo - echo "ERROR $test_name execution was expected to exit with error message '${expected_error_message}' but did not. cat $test_logfile" - return 1 - fi - # 'expect_failure' file has no content. We generally agree that the failure is correct - return 0 - fi - - if [ $execution_result -ne 0 ] ; then - echo - echo "ERROR $test_name execution failure. cat $test_logfile:" - cat $test_logfile - return 1 - fi - - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "show create table _gh_ost_test_gho\G" -ss > $ghost_structure_output_file - - if [ -f $tests_path/$test_name/expect_table_structure ] ; then - expected_table_structure="$(cat $tests_path/$test_name/expect_table_structure)" - if ! grep -q "$expected_table_structure" $ghost_structure_output_file ; then - echo - echo "ERROR $test_name: table structure was expected to include ${expected_table_structure} but did not. cat $ghost_structure_output_file:" - cat $ghost_structure_output_file - return 1 - fi - fi - - echo_dot - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test ${order_by}" -ss > $orig_content_output_file - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho ${order_by}" -ss > $ghost_content_output_file - orig_checksum=$(cat $orig_content_output_file | md5sum) - ghost_checksum=$(cat $ghost_content_output_file | md5sum) - - if [ "$orig_checksum" != "$ghost_checksum" ] ; then - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${orig_columns} from gh_ost_test" -ss > $orig_content_output_file - gh-ost-test-mysql-replica --default-character-set=utf8mb4 test -e "select ${ghost_columns} from _gh_ost_test_gho" -ss > $ghost_content_output_file - echo "ERROR $test_name: checksum mismatch" - echo "---" - diff $orig_content_output_file $ghost_content_output_file - - echo "diff $orig_content_output_file $ghost_content_output_file" - - return 1 - fi -} - -build_binary() { - echo "Building" - rm -f $default_ghost_binary - [ "$ghost_binary" == "" ] && ghost_binary="$default_ghost_binary" - if [ -f "$ghost_binary" ] ; then - echo "Using binary: $ghost_binary" - return 0 - fi - - go build -o $ghost_binary go/cmd/gh-ost/main.go - - if [ $? -ne 0 ] ; then - echo "Build failure" - exit 1 - fi -} - -test_all() { - build_binary - find $tests_path ! -path . -type d -mindepth 1 -maxdepth 1 | cut -d "/" -f 3 | egrep "$test_pattern" | sort | while read test_name ; do - test_single "$test_name" - if [ $? -ne 0 ] ; then - create_statement=$(gh-ost-test-mysql-replica test -t -e "show create table _gh_ost_test_gho \G") - echo "$create_statement" >> $test_logfile - echo "+ FAIL" - return 1 - else - echo - echo "+ pass" - fi - gh-ost-test-mysql-replica -e "start slave" - done -} - -verify_master_and_replica -test_all From ad19d1e9c3d57d04c8db9c74b07798b228eced6d Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 10 Dec 2022 00:14:20 +0100 Subject: [PATCH 21/22] Fix tests.sh --- localtests/tests.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/localtests/tests.sh b/localtests/tests.sh index eb82c5718..aeef21406 100755 --- a/localtests/tests.sh +++ b/localtests/tests.sh @@ -2,19 +2,20 @@ set -e -export TEST_DOCKER_IMAGE=${1:-mysql:5.7} - LOCALTESTS_DIR="$(readlink -f $(dirname $0))" +if [ -z "$TEST_DOCKER_IMAGE" ]; then + echo "TEST_DOCKER_IMAGE env var must be set" + exit 1 +fi + # generate mysql.env file for containers [ ! -z "$GITHUB_ACTION" ] && echo "::group::generate mysql env" ( echo "GITHUB_ACTION=${GITHUB_ACTION}" echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true' echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}" - if [ "$TEST_STORAGE_ENGINE" == "rocksdb" ]; then - echo 'INIT_ROCKSDB=true' - fi + [ "$TEST_STORAGE_ENGINE" == "rocksdb" ] && echo 'INIT_ROCKSDB=true' ) | tee $LOCALTESTS_DIR/mysql.env echo "Wrote env file to $LOCALTESTS_DIR/mysql.env" [ ! -z "$GITHUB_ACTION" ] && echo "::endgroup::" @@ -31,7 +32,7 @@ fi # this will start the test container and the # mysql primary/replica w/docker-compose -TEST_DOCKER_IMAGE=mysql:5.7 docker-compose -f localtests/docker-compose.yml up \ +docker-compose -f localtests/docker-compose.yml up \ --abort-on-container-exit \ --no-log-prefix \ $EXTRA_UP_FLAGS \ From 115fcfb9e84de1a69d43154e8628b014f3746020 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Sat, 10 Dec 2022 20:14:41 +0100 Subject: [PATCH 22/22] WIP --- .gitignore | 2 +- .../main.go | 0 go/localtests/test.go | 95 ++++++++++--------- go/localtests/tester.go | 12 ++- localtests/Dockerfile | 6 +- localtests/docker-compose.yml | 8 +- localtests/tests.sh | 48 +++++++--- 7 files changed, 98 insertions(+), 73 deletions(-) rename go/cmd/{gh-ost-localtests => gh-ost-tester}/main.go (100%) diff --git a/.gitignore b/.gitignore index 9c711e9e8..7598cc25f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ /.gopath/ /bin/ /libexec/ -/localtests/mysql.env +/localtests/tests.env /.vendor/ .idea/ diff --git a/go/cmd/gh-ost-localtests/main.go b/go/cmd/gh-ost-tester/main.go similarity index 100% rename from go/cmd/gh-ost-localtests/main.go rename to go/cmd/gh-ost-tester/main.go diff --git a/go/localtests/test.go b/go/localtests/test.go index 25c32ff3d..96fcab164 100644 --- a/go/localtests/test.go +++ b/go/localtests/test.go @@ -8,6 +8,10 @@ import ( "log" "os" "os/exec" + "strings" + "sync" + + "github.com/openark/golib/sqlutils" ) // Test represents a single test. @@ -133,11 +137,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { fmt.Printf("::group::%s gh-ost output\n", test.Name) } - if err = cmd.Start(); err != nil { - return err - } - - err = cmd.Wait() + err = cmd.Run() if isGitHubActions() { fmt.Println("::endgroup::") @@ -145,6 +145,7 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { if err != nil { if isExpectedFailureOutput(&stderr, test.ExpectedFailure) { + log.Printf("[%s] test got expected failure: %s", test.ExpectedFailure) return nil } log.Printf("[%s] test failed: %+v", test.Name, stderr.String()) @@ -152,61 +153,61 @@ func (test *Test) Migrate(config Config, primary, replica *sql.DB) (err error) { return err } -/* -func getPrimaryOrUniqueKey(db *sql.DB, database, table string) (string, error) { +func getTablePrimaryKey(db *sql.DB, database, table string) (string, error) { return "id", nil // TODO: fix this } -*/ + +type validationResult struct { + Source string + Rows sqlutils.RowMap +} // Validate performs a validation of the migration test results. -func (test *Test) Validate(config Config, primary, replica *sql.DB) error { +func (test *Test) Validate(config Config, db *sql.DB) error { if len(test.ValidateColumns) == 0 || len(test.ValidateOrigColumns) == 0 { return nil } - /* - primaryKey, err := getPrimaryOrUniqueKey(replica, testDatabase, testTable) - if err != nil { - return err - } + primaryKey, err := getTablePrimaryKey(db, testDatabase, testTable) + if err != nil { + return err + } - var query string - var maxPrimaryKeyVal interface{} - if maxPrimaryKeyVal == nil { - query = fmt.Sprintf("select * from %s.%s limit 10", testDatabase, testTable) - } else { - query = fmt.Sprintf("select * from %s.%s where %s > %+v limit 10", - testDatabase, testTable, primaryKey, maxPrimaryKeyVal, - ) - } - var rowMap sqlutils.RowMap - err = sqlutils.QueryRowsMap(replica, query, func(m sqlutils.RowMap) error { - for _, col := range test.ValidateColumns { - if val, found := m[col]; found { - rowMap[col] = val - } - } - }) + limit := 10 + orderBy := primaryKey + if test.ValidateOrderBy != "" { + orderBy = test.ValidateOrderBy + } - values := make([]interface{}, 0) - for range test.ValidateOrigColumns { - var val interface{} - values = append(values, &val) - } - maxPrimaryKeyVal = values[0] + outChan := make(chan validationResult, 2) + getTableMap := func(wg *sync.WaitGroup, database, table string, columns []string, outChan chan validationResult) { + defer wg.Done() - for rows.Next() { - if err = rows.Scan(values...); err != nil { - return err - } - for i, value := range values { - if value == nil { - continue - } - log.Printf("[%s] row value for %q col: %d", test.Name, test.ValidateOrigColumns[i], value) + query := fmt.Sprintf("select %s from %s.%s order by %s limit %d", + strings.Join(columns, ", "), + database, table, + orderBy, limit, + ) + err := sqlutils.QueryRowsMap(db, query, func(m sqlutils.RowMap) error { + outChan <- validationResult{ + Source: table, + Rows: m, } + return nil + }) + if err != nil { + log.Printf("[%s] failed to validate table %s: %+v", test.Name, table, err) } - */ + } + + var wg sync.WaitGroup + go getTableMap(&wg, testDatabase, testTable, test.ValidateColumns, outChan) + go getTableMap(&wg, testDatabase, fmt.Sprintf("_%s_del", testTable), test.ValidateOrigColumns, outChan) + wg.Add(2) + wg.Wait() + for result := range outChan { + log.Printf("[%s] result for %s: %+v", test.Name, result.Source, result.Rows) + } return nil } diff --git a/go/localtests/tester.go b/go/localtests/tester.go index 2dc897ed1..a0d173005 100644 --- a/go/localtests/tester.go +++ b/go/localtests/tester.go @@ -114,8 +114,10 @@ func (t *Tester) ReadTests(specificTestName string) (tests []Test, err error) { for _, subdir := range subdirs { test := Test{ - Name: subdir.Name(), - Path: filepath.Join(t.config.TestsDir, subdir.Name()), + Name: subdir.Name(), + Path: filepath.Join(t.config.TestsDir, subdir.Name()), + ValidateColumns: []string{"*"}, + ValidateOrigColumns: []string{"*"}, } stat, err := os.Stat(test.Path) @@ -147,7 +149,9 @@ func (t *Tester) ReadTests(specificTestName string) (tests []Test, err error) { } orderByFile := filepath.Join(test.Path, "order_by") - test.ValidateOrderBy, _ = readTestFile(orderByFile) + if validateOrderBy, err := readTestFile(orderByFile); err == nil { + test.ValidateOrderBy = validateOrderBy + } origColumnsFile := filepath.Join(test.Path, "orig_columns") if origColumns, err := readTestFile(origColumnsFile); err == nil { @@ -194,7 +198,7 @@ func (t *Tester) RunTest(test Test) (err error) { } log.Printf("[%s] successfully migrated test %s", test.Name, successEmoji) - if err = test.Validate(t.config, t.primary, t.replica); err != nil { + if err = test.Validate(t.config, t.replica); err != nil { log.Printf("[%s] failed to validate test %s%s%s", test.Name, failedEmoji, failedEmoji, failedEmoji) return err diff --git a/localtests/Dockerfile b/localtests/Dockerfile index 1246e3a93..540420144 100644 --- a/localtests/Dockerfile +++ b/localtests/Dockerfile @@ -5,7 +5,7 @@ COPY . /go/src/github.com/github/gh-ost WORKDIR /go/src/github.com/github/gh-ost RUN go build -o gh-ost go/cmd/gh-ost/main.go -RUN go build -o gh-ost-localtests go/cmd/gh-ost-localtests/main.go +RUN go build -o gh-ost-tester go/cmd/gh-ost-tester/main.go @@ -16,7 +16,7 @@ RUN apt-get install -y default-mysql-client RUN rm -rf /var/lib/apt/lists/* COPY --from=build /go/src/github.com/github/gh-ost/gh-ost /usr/local/bin/gh-ost -COPY --from=build /go/src/github.com/github/gh-ost/gh-ost-localtests /usr/local/bin/gh-ost-localtests +COPY --from=build /go/src/github.com/github/gh-ost/gh-ost-tester /usr/local/bin/gh-ost-tester COPY --from=build /go/src/github.com/github/gh-ost/localtests /etc/localtests -ENTRYPOINT ["gh-ost-localtests"] +ENTRYPOINT ["gh-ost-tester"] diff --git a/localtests/docker-compose.yml b/localtests/docker-compose.yml index 6bf5383e7..f11efa6b5 100644 --- a/localtests/docker-compose.yml +++ b/localtests/docker-compose.yml @@ -4,22 +4,24 @@ services: build: context: "../" dockerfile: "localtests/Dockerfile" - env_file: "mysql.env" + env_file: "tests.env" depends_on: - "primary" - "replica" primary: image: ${TEST_DOCKER_IMAGE} command: "--bind-address=0.0.0.0 --enforce-gtid-consistency --gtid-mode=ON --event-scheduler=ON --log-bin --log-slave-updates --server-id=1" - env_file: "mysql.env" + env_file: "tests.env" volumes: - "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro" replica: image: ${TEST_DOCKER_IMAGE} command: "--bind-address=0.0.0.0 --enforce-gtid-consistency --gtid-mode=ON --log-bin --log-slave-updates --read-only=ON --server-id=2" - env_file: "mysql.env" + env_file: "tests.env" depends_on: - "primary" + ports: + - "3306:3306" volumes: - "./init.sql:/docker-entrypoint-initdb.d/01-init.sql:ro" - "./init-replica.sql:/docker-entrypoint-initdb.d/02-init-replica.sql:ro" diff --git a/localtests/tests.sh b/localtests/tests.sh index aeef21406..86c6f56b1 100755 --- a/localtests/tests.sh +++ b/localtests/tests.sh @@ -9,29 +9,47 @@ if [ -z "$TEST_DOCKER_IMAGE" ]; then exit 1 fi -# generate mysql.env file for containers -[ ! -z "$GITHUB_ACTION" ] && echo "::group::generate mysql env" -( - echo "GITHUB_ACTION=${GITHUB_ACTION}" - echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true' - echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}" - [ "$TEST_STORAGE_ENGINE" == "rocksdb" ] && echo 'INIT_ROCKSDB=true' -) | tee $LOCALTESTS_DIR/mysql.env -echo "Wrote env file to $LOCALTESTS_DIR/mysql.env" -[ ! -z "$GITHUB_ACTION" ] && echo "::endgroup::" +if [ -z "$TEST_STORAGE_ENGINE" ]; then + TEST_STORAGE_ENGINE=innodb +fi + +# generate env file for containers +generate_test_env() { + [ ! -z "$GITHUB_ACTION" ] && echo "::group::generate mysql env" + + ( + echo "GITHUB_ACTION=${GITHUB_ACTION}" + echo 'MYSQL_ALLOW_EMPTY_PASSWORD=true' + echo "TEST_STORAGE_ENGINE=${TEST_STORAGE_ENGINE}" + [ "$TEST_STORAGE_ENGINE" == "rocksdb" ] && echo 'INIT_ROCKSDB=true' + ) | tee $LOCALTESTS_DIR/tests.env + echo "Wrote env file to $LOCALTESTS_DIR/tests.env" + + [ ! -z "$GITHUB_ACTION" ] && echo "::endgroup::" + return 0 +} + +# prebuild the test docker image +prebuild_docker_image() { + echo "::group::docker image build" + docker-compose -f localtests/docker-compose.yml build + echo "::endgroup::" +} + +### + +generate_test_env # conditional pre-build EXTRA_UP_FLAGS= if [ -z "$GITHUB_ACTION" ]; then EXTRA_UP_FLAGS="--build" else - echo "::group::docker image build" - docker-compose -f localtests/docker-compose.yml build - echo "::endgroup::" + prebuild_docker_image fi -# this will start the test container and the -# mysql primary/replica w/docker-compose +# start test container and mysql primary/replica +# with docker-compose docker-compose -f localtests/docker-compose.yml up \ --abort-on-container-exit \ --no-log-prefix \