Golang localtests

.github/workflows/replica-tests.yml

@@ -1,24 +1,31 @@
 name: migration tests
-
-on: [pull_request]
+on:
+  - pull_request
 
 jobs:
   build:
-
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        version: [mysql-5.7.25,mysql-8.0.16,PerconaServer-8.0.21]
+        tests:
+          - image: mysql:5.7
+            engine: innodb
+          - image: mysql:8.0
+            engine: innodb
+          - image: percona:5.7
+            engine: innodb
+          - image: percona:8.0
+            engine: innodb
+          - image: percona:5.7
+            engine: rocksdb
+          - image: percona:8.0
+            engine: rocksdb
 
     steps:
     - uses: actions/checkout@v2
 
-    - name: Set up Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.17
-
-    - name: migration tests
+    - name: run localtests
       env:
-        TEST_MYSQL_VERSION: ${{ matrix.version }}
-      run:  script/cibuild-gh-ost-replica-tests
+        TEST_DOCKER_IMAGE: "${{ matrix.tests.image }}"
+        TEST_STORAGE_ENGINE: "${{ matrix.tests.engine }}"
+      run: localtests/tests.sh

.gitignore

@@ -1,5 +1,6 @@
 /.gopath/
 /bin/
 /libexec/
+/localtests/tests.env
 /.vendor/
 .idea/

go.mod

@@ -6,10 +6,11 @@ require (
 	github.com/go-ini/ini v1.62.0
 	github.com/go-mysql-org/go-mysql v1.3.0
 	github.com/go-sql-driver/mysql v1.6.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/openark/golib v0.0.0-20210531070646-355f37940af8
 	github.com/satori/go.uuid v1.2.0
-	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
 	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
+	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467
 	golang.org/x/text v0.3.6
 )
 
@@ -21,7 +22,6 @@ require (
 	github.com/smartystreets/goconvey v1.6.4 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 // indirect
-	golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
 )

go.sum

@@ -17,6 +17,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
@@ -81,8 +83,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g=
-golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -96,14 +96,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM=
 golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

go/cmd/gh-ost-tester/main.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"database/sql"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	_ "github.com/go-sql-driver/mysql"
+
+	"github.com/github/gh-ost/go/localtests"
+)
+
+var AppVersion string
+
+func envStringVarOrDefault(envVar, defaultVal string) string {
+	if val := os.Getenv(envVar); val != "" {
+		return val
+	}
+	return defaultVal
+}
+
+func main() {
+	// flags
+	var printVersion, testNoop bool
+	var testName string
+	var cnf localtests.Config
+	flag.StringVar(&cnf.Host, "host", localtests.DefaultHost, "mysql host")
+	flag.Int64Var(&cnf.Port, "port", localtests.DefaultPort, "mysql port")
+	flag.StringVar(&cnf.Username, "username", localtests.DefaultUsername, "mysql username")
+	flag.StringVar(&cnf.Password, "password", localtests.DefaultPassword, "mysql password")
+	flag.StringVar(&cnf.TestsDir, "tests-dir", "/etc/localtests", "path to localtests directory")
+	flag.StringVar(&testName, "test", "", "run a single test by name (default: run all tests)")
+	flag.BoolVar(&testNoop, "test-noop", false, "run a single noop migration, eg: --alter='ENGINE=InnoDB'")
+	flag.StringVar(&cnf.StorageEngine, "storage-engine", envStringVarOrDefault("TEST_STORAGE_ENGINE", "innodb"), "mysql storage engine")
+	flag.StringVar(&cnf.GhostBinary, "binary", "gh-ost", "path to gh-ost binary")
+	flag.StringVar(&cnf.MysqlBinary, "mysql-binary", "mysql", "path to mysql binary")
+	flag.BoolVar(&printVersion, "version", false, "print version and exit")
+	flag.Parse()
+
+	// print version
+	if printVersion {
+		fmt.Println(AppVersion)
+		os.Exit(0)
+	}
+
+	// connect to replica
+	replica, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/?interpolateParams=true",
+		cnf.Username,
+		cnf.Password,
+		cnf.Host,
+		cnf.Port,
+	))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer replica.Close()
+
+	// connect to primary
+	primary, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/?interpolateParams=true",
+		cnf.Username,
+		cnf.Password,
+		"primary", // TODO: fix me
+		cnf.Port,
+	))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer primary.Close()
+
+	// start tester
+	tester := localtests.NewTester(cnf, primary, replica)
+	if err = tester.WaitForMySQLAvailable(); err != nil {
+		log.Fatalf("Failed to setup MySQL database servers: %+v", err)
+	}
+
+	// find tests
+	var tests []localtests.Test
+	if testNoop {
+		tests = []localtests.Test{
+			{
+				Name: "noop",
+				ExtraArgs: []string{
+					fmt.Sprintf("--alter='ENGINE=%s'", cnf.StorageEngine),
+				},
+			},
+		}
+	} else {
+		tests, err = tester.ReadTests(testName)
+		if err != nil {
+			log.Fatalf("Failed to read tests: %+v", err)
+		}
+	}
+
+	// run tests
+	for _, test := range tests {
+		log.Println("------------------------------------------------------------------------------------------------------------")
+		log.Printf("Loading test %q at %s/%s", test.Name, cnf.TestsDir, test.Name)
+		if err = tester.RunTest(test); err != nil {
+			log.Fatalf("Failed to run test %s: %+v", test.Name, err)
+		}
+	}
+}