From 80503bbe945c5c891c8fe9f7b0711bd794b578d3 Mon Sep 17 00:00:00 2001 From: Kartikay Date: Wed, 5 Feb 2025 00:59:16 +0530 Subject: [PATCH 1/5] added test coverage Signed-off-by: Kartikay --- magefiles/test.go | 11 ++--------- magefiles/util.go | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/magefiles/test.go b/magefiles/test.go index 2a3d3377ae..231fbca463 100644 --- a/magefiles/test.go +++ b/magefiles/test.go @@ -15,7 +15,7 @@ type Test mg.Namespace var emptyEnv map[string]string -// All Runs all test suites +// All Runs all test suites and generates a combined coverage report func (t Test) All() error { ds := Testds{} c := Testcons{} @@ -27,7 +27,7 @@ func (t Test) All() error { // UnitCover Runs the unit tests and generates a coverage report func (t Test) UnitCover() error { - if err := t.unit(true); err != nil { + if err := t.Unit(); err != nil { return err } fmt.Println("Running coverage...") @@ -36,15 +36,8 @@ func (t Test) UnitCover() error { // Unit Runs the unit tests func (t Test) Unit() error { - return t.unit(false) -} - -func (Test) unit(coverage bool) error { fmt.Println("running unit tests") args := []string{"-tags", "ci,skipintegrationtests", "-race", "-timeout", "10m", "-count=1"} - if coverage { - args = append(args, "-covermode=atomic", "-coverprofile=coverage.txt") - } return goTest("./...", args...) } diff --git a/magefiles/util.go b/magefiles/util.go index 1834decb58..6c46d0af0b 100644 --- a/magefiles/util.go +++ b/magefiles/util.go @@ -21,12 +21,24 @@ func goTest(path string, args ...string) error { // run go test in a directory func goDirTest(dir string, path string, args ...string) error { - testArgs := append([]string{"test", "-failfast", "-count=1"}, args...) + testArgs := append([]string{ + "test", + "-covermode=atomic", + "-coverprofile=coverage.txt", + "-failfast", + "-count=1", + }, args...) return RunSh(goCmdForTests(), WithV(), WithDir(dir), WithArgs(testArgs...))(path) } func goDirTestWithEnv(dir string, path string, env map[string]string, args ...string) error { - testArgs := append([]string{"test", "-failfast", "-count=1"}, args...) + testArgs := append([]string{ + "test", + "-covermode=atomic", + "-coverprofile=coverage.txt", + "-failfast", + "-count=1", + }, args...) return RunSh(goCmdForTests(), WithV(), WithDir(dir), WithEnv(env), WithArgs(testArgs...))(path) } From 64f1377a1758efd4bbaac6a01e10eafa1976d054 Mon Sep 17 00:00:00 2001 From: Kartikay Date: Thu, 6 Feb 2025 00:35:09 +0530 Subject: [PATCH 2/5] different profiles for tests Signed-off-by: Kartikay --- magefiles/test.go | 4 ++-- magefiles/util.go | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/magefiles/test.go b/magefiles/test.go index 231fbca463..330b726753 100644 --- a/magefiles/test.go +++ b/magefiles/test.go @@ -22,7 +22,7 @@ func (t Test) All() error { mg.Deps(t.Unit, t.Integration, t.Steelthread, t.Image, t.Analyzers, ds.Crdb, ds.Postgres, ds.Spanner, ds.Mysql, c.Crdb, c.Spanner, c.Postgres, c.Mysql) - return nil + return combineCoverage() } // UnitCover Runs the unit tests and generates a coverage report @@ -31,7 +31,7 @@ func (t Test) UnitCover() error { return err } fmt.Println("Running coverage...") - return sh.RunV("go", "tool", "cover", "-html=coverage.txt") + return sh.RunV("go", "tool", "cover", "-html=coverage-unit.txt") } // Unit Runs the unit tests diff --git a/magefiles/util.go b/magefiles/util.go index 6c46d0af0b..325a2f4c92 100644 --- a/magefiles/util.go +++ b/magefiles/util.go @@ -8,6 +8,7 @@ import ( "log" "os" "os/exec" + "path/filepath" "strings" "github.com/magefile/mage/mg" @@ -24,7 +25,7 @@ func goDirTest(dir string, path string, args ...string) error { testArgs := append([]string{ "test", "-covermode=atomic", - "-coverprofile=coverage.txt", + fmt.Sprintf("-coverprofile=coverage-%s.txt", sanitizePath(path)), "-failfast", "-count=1", }, args...) @@ -35,7 +36,7 @@ func goDirTestWithEnv(dir string, path string, env map[string]string, args ...st testArgs := append([]string{ "test", "-covermode=atomic", - "-coverprofile=coverage.txt", + fmt.Sprintf("-coverprofile=coverage-%s.txt", sanitizePath(path)), "-failfast", "-count=1", }, args...) @@ -222,3 +223,45 @@ func run(dir string, env map[string]string, stdout, stderr io.Writer, cmd string err = c.Run() return sh.CmdRan(err), sh.ExitStatus(err), err } + +func sanitizePath(path string) string { + parts := strings.Split(strings.TrimPrefix(path, "./"), "/") + for i := len(parts) - 1; i >= 0; i-- { + if parts[i] != "..." && parts[i] != "" { + return strings.ReplaceAll(parts[i], "/", "-") + } + } + return "all" +} + +func combineCoverage() error { + files, err := filepath.Glob("coverage-*.txt") + if err != nil { + return err + } + if len(files) == 0 { + return fmt.Errorf("no coverage files found") + } + + f, err := os.Create("coverage.txt") + if err != nil { + return err + } + defer f.Close() + + args := []string{"run", "github.com/wadey/gocovmerge@latest"} + args = append(args, files...) + + err = RunSh(goCmdForTests(), WithV(), WithStdout(f))(args...) + if err != nil { + return err + } + + for _, file := range files { + if err := os.Remove(file); err != nil { + return err + } + } + + return nil +} From 59fd80d6920dea391e076e2a36eea557e9fd5a68 Mon Sep 17 00:00:00 2001 From: Kartikay Date: Sat, 8 Feb 2025 19:07:28 +0530 Subject: [PATCH 3/5] use path hash instead Signed-off-by: Kartikay --- magefiles/util.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/magefiles/util.go b/magefiles/util.go index 325a2f4c92..f018c1abf3 100644 --- a/magefiles/util.go +++ b/magefiles/util.go @@ -3,6 +3,8 @@ package main import ( + "crypto/sha256" + "encoding/hex" "fmt" "io" "log" @@ -25,7 +27,7 @@ func goDirTest(dir string, path string, args ...string) error { testArgs := append([]string{ "test", "-covermode=atomic", - fmt.Sprintf("-coverprofile=coverage-%s.txt", sanitizePath(path)), + fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath(path)), "-failfast", "-count=1", }, args...) @@ -36,7 +38,7 @@ func goDirTestWithEnv(dir string, path string, env map[string]string, args ...st testArgs := append([]string{ "test", "-covermode=atomic", - fmt.Sprintf("-coverprofile=coverage-%s.txt", sanitizePath(path)), + fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath(path)), "-failfast", "-count=1", }, args...) @@ -224,14 +226,9 @@ func run(dir string, env map[string]string, stdout, stderr io.Writer, cmd string return sh.CmdRan(err), sh.ExitStatus(err), err } -func sanitizePath(path string) string { - parts := strings.Split(strings.TrimPrefix(path, "./"), "/") - for i := len(parts) - 1; i >= 0; i-- { - if parts[i] != "..." && parts[i] != "" { - return strings.ReplaceAll(parts[i], "/", "-") - } - } - return "all" +func hashPath(path string) string { + h := sha256.Sum256([]byte(path)) + return hex.EncodeToString(h[:]) } func combineCoverage() error { From 3f62b006fa98505ff9c9ff1a0e4eb3190a838c6f Mon Sep 17 00:00:00 2001 From: Kartikay <120778728+kartikaysaxena@users.noreply.github.com> Date: Mon, 10 Feb 2025 21:51:27 +0000 Subject: [PATCH 4/5] separate targets for cover Signed-off-by: Kartikay <120778728+kartikaysaxena@users.noreply.github.com> --- magefiles/test.go | 296 ++++++++++++++++++++++++++++++++++++---------- magefiles/util.go | 4 - 2 files changed, 235 insertions(+), 65 deletions(-) diff --git a/magefiles/test.go b/magefiles/test.go index 330b726753..038b2e4507 100644 --- a/magefiles/test.go +++ b/magefiles/test.go @@ -15,48 +15,113 @@ type Test mg.Namespace var emptyEnv map[string]string -// All Runs all test suites and generates a combined coverage report +// All Runs all test suites func (t Test) All() error { ds := Testds{} c := Testcons{} mg.Deps(t.Unit, t.Integration, t.Steelthread, t.Image, t.Analyzers, ds.Crdb, ds.Postgres, ds.Spanner, ds.Mysql, c.Crdb, c.Spanner, c.Postgres, c.Mysql) + return nil +} + +// All Runs all test suites and generates a combined coverage report +func (t Test) AllCover() error { + ds := Testds{} + c := Testcons{} + mg.Deps(t.UnitCover, t.IntegrationCover, t.SteelthreadCover, t.ImageCover, t.AnalyzersCover, + ds.CrdbCover, ds.PostgresCover, ds.SpannerCover, ds.MysqlCover, + c.CrdbCover, c.SpannerCover, c.PostgresCover, c.MysqlCover) return combineCoverage() } // UnitCover Runs the unit tests and generates a coverage report func (t Test) UnitCover() error { - if err := t.Unit(); err != nil { + if err := t.unit(true); err != nil { return err } fmt.Println("Running coverage...") - return sh.RunV("go", "tool", "cover", "-html=coverage-unit.txt") + return nil } // Unit Runs the unit tests func (t Test) Unit() error { + return t.unit(false) +} + +func (Test) unit(coverage bool) error { fmt.Println("running unit tests") args := []string{"-tags", "ci,skipintegrationtests", "-race", "-timeout", "10m", "-count=1"} + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./..."))) + } return goTest("./...", args...) } +// Image Run tests that run the built image and generates a coverage report +func (t Test) ImageCover() error { + if err := t.image(true); err != nil { + return err + } + return nil +} + // Image Run tests that run the built image func (Test) Image() error { mg.Deps(Build{}.Testimage) return goDirTest("./cmd/spicedb", "./...", "-tags", "docker,image") } -// Integration Run integration tests -func (Test) Integration() error { - mg.Deps(checkDocker) - return goTest("./internal/services/integrationtesting/...", "-tags", "ci,docker", "-timeout", "15m") +func (Test) image(coverage bool) error { + mg.Deps(Build{}.Testimage) + args := []string{"-tags", "docker,image"} + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./..."))) + } + return goDirTest("./cmd/spicedb", "./...", args...) } -// Steelthread Run steelthread tests -func (Test) Steelthread() error { +// IntegrationCover Runs the integration tests and generates a coverage report +func (t Test) IntegrationCover() error { + if err := t.integration(true); err != nil { + return err + } + return nil +} + +// Integration Runs the integration tests +func (t Test) Integration() error { + return t.integration(false) +} + +func (Test) integration(coverage bool) error { + args := []string{"-tags", "ci,docker", "-timeout", "15m"} + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/services/integrationtesting/..."))) + } + return goTest("./internal/services/integrationtesting/...", args...) +} + +// SteelthreadCover Runs the steelthread tests and generates a coverage report +func (t Test) SteelthreadCover() error { + if err := t.steelthread(true); err != nil { + return err + } + return nil +} + +// Steelthread Runs the steelthread tests +func (t Test) Steelthread() error { + return t.steelthread(false) +} + +func (Test) steelthread(coverage bool) error { fmt.Println("running steel thread tests") - return goTest("./internal/services/steelthreadtesting/...", "-tags", "steelthread,docker,image", "-timeout", "15m", "-v") + args := []string{"-tags", "steelthread,docker,image", "-timeout", "15m", "-v"} + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/services/steelthreadtesting/..."))) + } + return goTest("./internal/services/steelthreadtesting/...", args...) } // RegenSteelthread Regenerate the steelthread tests @@ -67,9 +132,25 @@ func (Test) RegenSteelthread() error { }), WithArgs("test", "./internal/services/steelthreadtesting/...", "-tags", "steelthread,docker,image", "-timeout", "15m", "-v"))("go") } +// AnalyzersCover Runs the analyzer tests and generates a coverage report +func (t Test) AnalyzersCover() error { + if err := t.analyzers(true); err != nil { + return err + } + return nil +} + // Analyzers Run the analyzer unit tests -func (Test) Analyzers() error { - return goDirTest("./tools/analyzers", "./...") +func (t Test) Analyzers() error { + return t.analyzers(false) +} + +func (Test) analyzers(coverage bool) error { + args := []string{} + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./..."))) + } + return goDirTest("./tools/analyzers", "./...", args...) } // Wasm Run wasm browser tests @@ -87,39 +168,96 @@ func (Test) Wasm() error { type Testds mg.Namespace -// Crdb Run datastore tests for crdb -func (tds Testds) Crdb() error { - return tds.crdb("") +// CrdbCover Runs the CRDB datastore tests and generates a coverage report +func (tds Testds) CrdbCover() error { + if err := tds.crdb(true, ""); err != nil { + return err + } + return nil } -func (tds Testds) CrdbVer(version string) error { - return tds.crdb(version) +// Crdb Runs the CRDB datastore tests +func (tds Testds) Crdb() error { + return tds.crdb(false, "") } -func (Testds) crdb(version string) error { +func (Testds) crdb(coverage bool, version string) error { + args := []string{"-tags", "ci,docker", "-timeout", "10m"} + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/datastore/crdb/..."))) + } return datastoreTest("crdb", map[string]string{ "CRDB_TEST_VERSION": version, - }) + }, args...) } -// Spanner Run datastore tests for spanner -func (Testds) Spanner() error { - return datastoreTest("spanner", emptyEnv) +// PostgresCover Runs the Postgres datastore tests and generates a coverage report +func (tds Testds) PostgresCover() error { + if err := tds.postgres(true, ""); err != nil { + return err + } + return nil } -// Postgres Run datastore tests for postgres +// Postgres Runs the Postgres datastore tests func (tds Testds) Postgres() error { - return tds.postgres("") + return tds.postgres(false, "") } -func (tds Testds) PostgresVer(version string) error { - return tds.postgres(version) -} - -func (Testds) postgres(version string) error { +func (Testds) postgres(coverage bool, version string) error { + args := []string{"-tags", "ci,docker", "-timeout", "10m"} + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/datastore/postgres/..."))) + } return datastoreTest("postgres", map[string]string{ "POSTGRES_TEST_VERSION": version, - }, "postgres") + }, args...) +} + +// SpannerCover Runs the Spanner datastore tests and generates a coverage report +func (tds Testds) SpannerCover() error { + if err := tds.spanner(true); err != nil { + return err + } + return nil +} + +// Spanner Runs the Spanner datastore tests +func (tds Testds) Spanner() error { + return tds.spanner(false) +} + +func (Testds) spanner(coverage bool) error { + args := []string{"-tags", "ci,docker", "-timeout", "10m"} + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/datastore/spanner/..."))) + } + return datastoreTest("spanner", emptyEnv, args...) +} + +// MysqlCover Runs the MySQL datastore tests and generates a coverage report +func (tds Testds) MysqlCover() error { + if err := tds.mysql(true); err != nil { + return err + } + return nil +} + +// Mysql Runs the MySQL datastore tests +func (tds Testds) Mysql() error { + return tds.mysql(false) +} + +func (Testds) mysql(coverage bool) error { + args := []string{"-tags", "ci,docker", "-timeout", "10m"} + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/datastore/mysql/..."))) + } + return datastoreTest("mysql", emptyEnv, args...) +} + +func (tds Testds) PostgresVer(version string) error { + return tds.postgres(false, version) } // Pgbouncer Run datastore tests for postgres with Pgbouncer @@ -137,11 +275,6 @@ func (Testds) pgbouncer(version string) error { }, "pgbouncer") } -// Mysql Run datastore tests for mysql -func (Testds) Mysql() error { - return datastoreTest("mysql", emptyEnv) -} - func datastoreTest(datastore string, env map[string]string, tags ...string) error { mergedTags := append([]string{"ci", "docker"}, tags...) tagString := strings.Join(mergedTags, ",") @@ -151,38 +284,84 @@ func datastoreTest(datastore string, env map[string]string, tags ...string) erro type Testcons mg.Namespace -// Crdb Run consistency tests for crdb +// CrdbCover Runs the CRDB consistency tests and generates a coverage report +func (tc Testcons) CrdbCover() error { + if err := tc.crdb(true, ""); err != nil { + return err + } + return nil +} + +// Crdb Runs the CRDB consistency tests func (tc Testcons) Crdb() error { - return tc.crdb("") + return tc.crdb(false, "") } func (tc Testcons) CrdbVer(version string) error { - return tc.crdb(version) + return tc.crdb(false, version) } -func (Testcons) crdb(version string) error { +func (Testcons) crdb(coverage bool, version string) error { return consistencyTest("crdb", map[string]string{ "CRDB_TEST_VERSION": version, - }) + }, coverage) } -// Spanner Run consistency tests for spanner -func (Testcons) Spanner() error { - return consistencyTest("spanner", emptyEnv) +// PostgresCover Runs the Postgres consistency tests and generates a coverage report +func (tc Testcons) PostgresCover() error { + if err := tc.postgres(true, ""); err != nil { + return err + } + return nil } +// Postgres Runs the Postgres consistency tests func (tc Testcons) Postgres() error { - return tc.postgres("") -} - -func (tc Testcons) PostgresVer(version string) error { - return tc.postgres(version) + return tc.postgres(false, "") } -func (Testcons) postgres(version string) error { +func (Testcons) postgres(coverage bool, version string) error { return consistencyTest("postgres", map[string]string{ "POSTGRES_TEST_VERSION": version, - }) + }, coverage) +} + +// SpannerCover Runs the Spanner consistency tests and generates a coverage report +func (tc Testcons) SpannerCover() error { + if err := tc.spanner(true); err != nil { + return err + } + return nil +} + +// Spanner Runs the Spanner consistency tests +func (tc Testcons) Spanner() error { + return tc.spanner(false) +} + +func (Testcons) spanner(coverage bool) error { + return consistencyTest("spanner", emptyEnv, coverage) +} + +// MysqlCover Runs the MySQL consistency tests and generates a coverage report +func (tc Testcons) MysqlCover() error { + if err := tc.mysql(true); err != nil { + return err + } + return nil +} + +// Mysql Runs the MySQL consistency tests +func (tc Testcons) Mysql() error { + return tc.mysql(false) +} + +func (Testcons) mysql(coverage bool) error { + return consistencyTest("mysql", emptyEnv, coverage) +} + +func (tc Testcons) PostgresVer(version string) error { + return tc.postgres(false, version) } // Pgbouncer Run consistency tests for postgres with pgbouncer @@ -197,16 +376,11 @@ func (Testcons) PgbouncerVer(version string) error { return nil } -// Mysql Run consistency tests for mysql -func (Testcons) Mysql() error { - return consistencyTest("mysql", emptyEnv) -} - -func consistencyTest(datastore string, env map[string]string) error { +func consistencyTest(datastore string, env map[string]string, coverage bool) error { + args := []string{"-tags", "ci,docker,datastoreconsistency", "-timeout", "10m", "-run", fmt.Sprintf("TestConsistencyPerDatastore/%s", datastore)} mg.Deps(checkDocker) - return goDirTestWithEnv(".", "./internal/services/integrationtesting/...", - env, - "-tags", "ci,docker,datastoreconsistency", - "-timeout", "10m", - "-run", fmt.Sprintf("TestConsistencyPerDatastore/%s", datastore)) + if coverage { + args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/services/integrationtesting/..."))) + } + return goDirTestWithEnv(".", "./internal/services/integrationtesting/...", env, args...) } diff --git a/magefiles/util.go b/magefiles/util.go index f018c1abf3..c664ff14ce 100644 --- a/magefiles/util.go +++ b/magefiles/util.go @@ -26,8 +26,6 @@ func goTest(path string, args ...string) error { func goDirTest(dir string, path string, args ...string) error { testArgs := append([]string{ "test", - "-covermode=atomic", - fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath(path)), "-failfast", "-count=1", }, args...) @@ -37,8 +35,6 @@ func goDirTest(dir string, path string, args ...string) error { func goDirTestWithEnv(dir string, path string, env map[string]string, args ...string) error { testArgs := append([]string{ "test", - "-covermode=atomic", - fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath(path)), "-failfast", "-count=1", }, args...) From 752a5eafe4625f92471a3c5df1f8d14c2ca7f003 Mon Sep 17 00:00:00 2001 From: Kartikay Date: Thu, 13 Feb 2025 05:38:08 +0530 Subject: [PATCH 5/5] context coverage bool Signed-off-by: Kartikay --- magefiles/go.mod | 1 + magefiles/go.sum | 1 + magefiles/test.go | 343 +++++++++++++--------------------------------- magefiles/util.go | 41 +++--- 4 files changed, 120 insertions(+), 266 deletions(-) diff --git a/magefiles/go.mod b/magefiles/go.mod index e7b53c4063..887f3c00fa 100644 --- a/magefiles/go.mod +++ b/magefiles/go.mod @@ -7,6 +7,7 @@ require ( github.com/bufbuild/buf v1.35.1 github.com/ecordell/optgen v0.0.9 github.com/envoyproxy/protoc-gen-validate v1.0.4 + github.com/google/uuid v1.1.2 github.com/magefile/mage v1.15.0 github.com/planetscale/vtprotobuf v0.6.1-0.20240409071808-615f978279ca golang.org/x/tools v0.22.0 diff --git a/magefiles/go.sum b/magefiles/go.sum index a7152f81fe..81e96a7230 100644 --- a/magefiles/go.sum +++ b/magefiles/go.sum @@ -221,6 +221,7 @@ github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0Z github.com/google/pprof v0.0.0-20240622144329-c177fd99eaa9 h1:ouFdLLCOyCfnxGpQTMZKHLyHr/D1GFbQzEsJxumO16E= github.com/google/pprof v0.0.0-20240622144329-c177fd99eaa9/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= diff --git a/magefiles/test.go b/magefiles/test.go index 038b2e4507..242b7186ac 100644 --- a/magefiles/test.go +++ b/magefiles/test.go @@ -3,6 +3,7 @@ package main import ( + "context" "fmt" "os" "strings" @@ -19,109 +20,62 @@ var emptyEnv map[string]string func (t Test) All() error { ds := Testds{} c := Testcons{} - mg.Deps(t.Unit, t.Integration, t.Steelthread, t.Image, t.Analyzers, + ctx := context.Background() + cover := false + for _, arg := range os.Args { + if arg == "-cover=true" { + cover = true + break + } + } + ctx = context.WithValue(ctx, "cover", cover) + mg.CtxDeps(ctx, t.Unit, t.Integration, t.Steelthread, t.Image, t.Analyzers, ds.Crdb, ds.Postgres, ds.Spanner, ds.Mysql, c.Crdb, c.Spanner, c.Postgres, c.Mysql) + return nil } -// All Runs all test suites and generates a combined coverage report -func (t Test) AllCover() error { - ds := Testds{} - c := Testcons{} - mg.Deps(t.UnitCover, t.IntegrationCover, t.SteelthreadCover, t.ImageCover, t.AnalyzersCover, - ds.CrdbCover, ds.PostgresCover, ds.SpannerCover, ds.MysqlCover, - c.CrdbCover, c.SpannerCover, c.PostgresCover, c.MysqlCover) +func (t Test) Combine() error { return combineCoverage() } // UnitCover Runs the unit tests and generates a coverage report -func (t Test) UnitCover() error { - if err := t.unit(true); err != nil { +func (t Test) UnitCover(ctx context.Context) error { + if err := t.unit(ctx, true); err != nil { return err } fmt.Println("Running coverage...") - return nil + return sh.RunV("go", "tool", "cover", "-html=coverage.txt") } // Unit Runs the unit tests -func (t Test) Unit() error { - return t.unit(false) +func (t Test) Unit(ctx context.Context) error { + return t.unit(ctx, false) } -func (Test) unit(coverage bool) error { +func (Test) unit(ctx context.Context, coverage bool) error { fmt.Println("running unit tests") args := []string{"-tags", "ci,skipintegrationtests", "-race", "-timeout", "10m", "-count=1"} - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./..."))) - } - return goTest("./...", args...) -} - -// Image Run tests that run the built image and generates a coverage report -func (t Test) ImageCover() error { - if err := t.image(true); err != nil { - return err - } - return nil + return goTest(ctx, "./...", args...) } // Image Run tests that run the built image -func (Test) Image() error { - mg.Deps(Build{}.Testimage) - return goDirTest("./cmd/spicedb", "./...", "-tags", "docker,image") -} - -func (Test) image(coverage bool) error { +func (Test) Image(ctx context.Context) error { mg.Deps(Build{}.Testimage) - args := []string{"-tags", "docker,image"} - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./..."))) - } - return goDirTest("./cmd/spicedb", "./...", args...) + return goDirTest(ctx, "./cmd/spicedb", "./...", "-tags", "docker,image") } -// IntegrationCover Runs the integration tests and generates a coverage report -func (t Test) IntegrationCover() error { - if err := t.integration(true); err != nil { - return err - } - return nil -} - -// Integration Runs the integration tests -func (t Test) Integration() error { - return t.integration(false) -} - -func (Test) integration(coverage bool) error { - args := []string{"-tags", "ci,docker", "-timeout", "15m"} - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/services/integrationtesting/..."))) - } - return goTest("./internal/services/integrationtesting/...", args...) -} - -// SteelthreadCover Runs the steelthread tests and generates a coverage report -func (t Test) SteelthreadCover() error { - if err := t.steelthread(true); err != nil { - return err - } - return nil -} - -// Steelthread Runs the steelthread tests -func (t Test) Steelthread() error { - return t.steelthread(false) +// Integration Run integration tests +func (Test) Integration(ctx context.Context) error { + mg.Deps(checkDocker) + return goTest(ctx, "./internal/services/integrationtesting/...", "-tags", "ci,docker", "-timeout", "15m") } -func (Test) steelthread(coverage bool) error { +// Steelthread Run steelthread tests +func (Test) Steelthread(ctx context.Context) error { fmt.Println("running steel thread tests") - args := []string{"-tags", "steelthread,docker,image", "-timeout", "15m", "-v"} - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/services/steelthreadtesting/..."))) - } - return goTest("./internal/services/steelthreadtesting/...", args...) + return goTest(ctx, "./internal/services/steelthreadtesting/...", "-tags", "steelthread,docker,image", "-timeout", "15m", "-v") } // RegenSteelthread Regenerate the steelthread tests @@ -132,25 +86,9 @@ func (Test) RegenSteelthread() error { }), WithArgs("test", "./internal/services/steelthreadtesting/...", "-tags", "steelthread,docker,image", "-timeout", "15m", "-v"))("go") } -// AnalyzersCover Runs the analyzer tests and generates a coverage report -func (t Test) AnalyzersCover() error { - if err := t.analyzers(true); err != nil { - return err - } - return nil -} - // Analyzers Run the analyzer unit tests -func (t Test) Analyzers() error { - return t.analyzers(false) -} - -func (Test) analyzers(coverage bool) error { - args := []string{} - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./..."))) - } - return goDirTest("./tools/analyzers", "./...", args...) +func (Test) Analyzers(ctx context.Context) error { + return goDirTest(ctx, "./tools/analyzers", "./...") } // Wasm Run wasm browser tests @@ -168,200 +106,102 @@ func (Test) Wasm() error { type Testds mg.Namespace -// CrdbCover Runs the CRDB datastore tests and generates a coverage report -func (tds Testds) CrdbCover() error { - if err := tds.crdb(true, ""); err != nil { - return err - } - return nil +// Crdb Run datastore tests for crdb +func (tds Testds) Crdb(ctx context.Context) error { + return tds.crdb(ctx, "") } -// Crdb Runs the CRDB datastore tests -func (tds Testds) Crdb() error { - return tds.crdb(false, "") +func (tds Testds) CrdbVer(ctx context.Context, version string) error { + return tds.crdb(ctx, version) } -func (Testds) crdb(coverage bool, version string) error { - args := []string{"-tags", "ci,docker", "-timeout", "10m"} - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/datastore/crdb/..."))) - } - return datastoreTest("crdb", map[string]string{ +func (Testds) crdb(ctx context.Context, version string) error { + return datastoreTest(ctx, "crdb", map[string]string{ "CRDB_TEST_VERSION": version, - }, args...) -} - -// PostgresCover Runs the Postgres datastore tests and generates a coverage report -func (tds Testds) PostgresCover() error { - if err := tds.postgres(true, ""); err != nil { - return err - } - return nil -} - -// Postgres Runs the Postgres datastore tests -func (tds Testds) Postgres() error { - return tds.postgres(false, "") -} - -func (Testds) postgres(coverage bool, version string) error { - args := []string{"-tags", "ci,docker", "-timeout", "10m"} - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/datastore/postgres/..."))) - } - return datastoreTest("postgres", map[string]string{ - "POSTGRES_TEST_VERSION": version, - }, args...) -} - -// SpannerCover Runs the Spanner datastore tests and generates a coverage report -func (tds Testds) SpannerCover() error { - if err := tds.spanner(true); err != nil { - return err - } - return nil -} - -// Spanner Runs the Spanner datastore tests -func (tds Testds) Spanner() error { - return tds.spanner(false) -} - -func (Testds) spanner(coverage bool) error { - args := []string{"-tags", "ci,docker", "-timeout", "10m"} - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/datastore/spanner/..."))) - } - return datastoreTest("spanner", emptyEnv, args...) + }) } -// MysqlCover Runs the MySQL datastore tests and generates a coverage report -func (tds Testds) MysqlCover() error { - if err := tds.mysql(true); err != nil { - return err - } - return nil +// Spanner Run datastore tests for spanner +func (Testds) Spanner(ctx context.Context) error { + return datastoreTest(ctx, "spanner", emptyEnv) } -// Mysql Runs the MySQL datastore tests -func (tds Testds) Mysql() error { - return tds.mysql(false) +// Postgres Run datastore tests for postgres +func (tds Testds) Postgres(ctx context.Context) error { + return tds.postgres(ctx, "") } -func (Testds) mysql(coverage bool) error { - args := []string{"-tags", "ci,docker", "-timeout", "10m"} - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/datastore/mysql/..."))) - } - return datastoreTest("mysql", emptyEnv, args...) +func (tds Testds) PostgresVer(ctx context.Context, version string) error { + return tds.postgres(ctx, version) } -func (tds Testds) PostgresVer(version string) error { - return tds.postgres(false, version) +func (Testds) postgres(ctx context.Context, version string) error { + return datastoreTest(ctx, "postgres", map[string]string{ + "POSTGRES_TEST_VERSION": version, + }, "postgres") } // Pgbouncer Run datastore tests for postgres with Pgbouncer -func (tds Testds) Pgbouncer() error { - return tds.pgbouncer("") +func (tds Testds) Pgbouncer(ctx context.Context) error { + return tds.pgbouncer(ctx, "") } -func (tds Testds) PgbouncerVer(version string) error { - return tds.pgbouncer(version) +func (tds Testds) PgbouncerVer(ctx context.Context, version string) error { + return tds.pgbouncer(ctx, version) } -func (Testds) pgbouncer(version string) error { - return datastoreTest("postgres", map[string]string{ +func (Testds) pgbouncer(ctx context.Context, version string) error { + return datastoreTest(ctx, "postgres", map[string]string{ "POSTGRES_TEST_VERSION": version, }, "pgbouncer") } -func datastoreTest(datastore string, env map[string]string, tags ...string) error { +// Mysql Run datastore tests for mysql +func (Testds) Mysql(ctx context.Context) error { + return datastoreTest(ctx, "mysql", emptyEnv) +} + +func datastoreTest(ctx context.Context, datastore string, env map[string]string, tags ...string) error { mergedTags := append([]string{"ci", "docker"}, tags...) tagString := strings.Join(mergedTags, ",") mg.Deps(checkDocker) - return goDirTestWithEnv(".", fmt.Sprintf("./internal/datastore/%s/...", datastore), env, "-tags", tagString, "-timeout", "10m") + return goDirTestWithEnv(ctx, ".", fmt.Sprintf("./internal/datastore/%s/...", datastore), env, "-tags", tagString, "-timeout", "10m") } type Testcons mg.Namespace -// CrdbCover Runs the CRDB consistency tests and generates a coverage report -func (tc Testcons) CrdbCover() error { - if err := tc.crdb(true, ""); err != nil { - return err - } - return nil +// Crdb Run consistency tests for crdb +func (tc Testcons) Crdb(ctx context.Context) error { + return tc.crdb(ctx, "") } -// Crdb Runs the CRDB consistency tests -func (tc Testcons) Crdb() error { - return tc.crdb(false, "") +func (tc Testcons) CrdbVer(ctx context.Context, version string) error { + return tc.crdb(ctx, version) } -func (tc Testcons) CrdbVer(version string) error { - return tc.crdb(false, version) -} - -func (Testcons) crdb(coverage bool, version string) error { - return consistencyTest("crdb", map[string]string{ +func (Testcons) crdb(ctx context.Context, version string) error { + return consistencyTest(ctx, "crdb", map[string]string{ "CRDB_TEST_VERSION": version, - }, coverage) + }) } -// PostgresCover Runs the Postgres consistency tests and generates a coverage report -func (tc Testcons) PostgresCover() error { - if err := tc.postgres(true, ""); err != nil { - return err - } - return nil +// Spanner Run consistency tests for spanner +func (Testcons) Spanner(ctx context.Context) error { + return consistencyTest(ctx, "spanner", emptyEnv) } -// Postgres Runs the Postgres consistency tests -func (tc Testcons) Postgres() error { - return tc.postgres(false, "") +func (tc Testcons) Postgres(ctx context.Context) error { + return tc.postgres(ctx, "") } -func (Testcons) postgres(coverage bool, version string) error { - return consistencyTest("postgres", map[string]string{ - "POSTGRES_TEST_VERSION": version, - }, coverage) +func (tc Testcons) PostgresVer(ctx context.Context, version string) error { + return tc.postgres(ctx, version) } -// SpannerCover Runs the Spanner consistency tests and generates a coverage report -func (tc Testcons) SpannerCover() error { - if err := tc.spanner(true); err != nil { - return err - } - return nil -} - -// Spanner Runs the Spanner consistency tests -func (tc Testcons) Spanner() error { - return tc.spanner(false) -} - -func (Testcons) spanner(coverage bool) error { - return consistencyTest("spanner", emptyEnv, coverage) -} - -// MysqlCover Runs the MySQL consistency tests and generates a coverage report -func (tc Testcons) MysqlCover() error { - if err := tc.mysql(true); err != nil { - return err - } - return nil -} - -// Mysql Runs the MySQL consistency tests -func (tc Testcons) Mysql() error { - return tc.mysql(false) -} - -func (Testcons) mysql(coverage bool) error { - return consistencyTest("mysql", emptyEnv, coverage) -} - -func (tc Testcons) PostgresVer(version string) error { - return tc.postgres(false, version) +func (Testcons) postgres(ctx context.Context, version string) error { + return consistencyTest(ctx, "postgres", map[string]string{ + "POSTGRES_TEST_VERSION": version, + }) } // Pgbouncer Run consistency tests for postgres with pgbouncer @@ -376,11 +216,16 @@ func (Testcons) PgbouncerVer(version string) error { return nil } -func consistencyTest(datastore string, env map[string]string, coverage bool) error { - args := []string{"-tags", "ci,docker,datastoreconsistency", "-timeout", "10m", "-run", fmt.Sprintf("TestConsistencyPerDatastore/%s", datastore)} +// Mysql Run consistency tests for mysql +func (Testcons) Mysql(ctx context.Context) error { + return consistencyTest(ctx, "mysql", emptyEnv) +} + +func consistencyTest(ctx context.Context, datastore string, env map[string]string) error { mg.Deps(checkDocker) - if coverage { - args = append(args, "-covermode=atomic", fmt.Sprintf("-coverprofile=coverage-%s.txt", hashPath("./internal/services/integrationtesting/..."))) - } - return goDirTestWithEnv(".", "./internal/services/integrationtesting/...", env, args...) + return goDirTestWithEnv(ctx, ".", "./internal/services/integrationtesting/...", + env, + "-tags", "ci,docker,datastoreconsistency", + "-timeout", "10m", + "-run", fmt.Sprintf("TestConsistencyPerDatastore/%s", datastore)) } diff --git a/magefiles/util.go b/magefiles/util.go index c664ff14ce..9dd5f63b1c 100644 --- a/magefiles/util.go +++ b/magefiles/util.go @@ -3,8 +3,7 @@ package main import ( - "crypto/sha256" - "encoding/hex" + "context" "fmt" "io" "log" @@ -13,31 +12,50 @@ import ( "path/filepath" "strings" + "github.com/google/uuid" "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" ) // run go test in the root -func goTest(path string, args ...string) error { - return goDirTest(".", path, args...) +func goTest(ctx context.Context, path string, args ...string) error { + return goDirTest(ctx, ".", path, args...) } // run go test in a directory -func goDirTest(dir string, path string, args ...string) error { +func goDirTest(ctx context.Context, dir string, path string, args ...string) error { testArgs := append([]string{ "test", "-failfast", "-count=1", }, args...) + if cover, _ := ctx.Value("cover").(bool); cover { + if err := os.MkdirAll("coverage", 0o755); err != nil { + return fmt.Errorf("failed to create coverage directory: %w", err) + } + testArgs = append(testArgs, []string{ + "-covermode=atomic", + fmt.Sprintf("-coverprofile=coverage-%s.txt", uuid.New().String()), + }...) + } return RunSh(goCmdForTests(), WithV(), WithDir(dir), WithArgs(testArgs...))(path) } -func goDirTestWithEnv(dir string, path string, env map[string]string, args ...string) error { +func goDirTestWithEnv(ctx context.Context, dir string, path string, env map[string]string, args ...string) error { testArgs := append([]string{ "test", "-failfast", "-count=1", }, args...) + if cover, _ := ctx.Value("cover").(bool); cover { + if err := os.MkdirAll("coverage", 0o755); err != nil { + return fmt.Errorf("failed to create coverage directory: %w", err) + } + testArgs = append(testArgs, []string{ + "-covermode=atomic", + fmt.Sprintf("-coverprofile=coverage-%s.txt", uuid.New().String()), + }...) + } return RunSh(goCmdForTests(), WithV(), WithDir(dir), WithEnv(env), WithArgs(testArgs...))(path) } @@ -222,11 +240,6 @@ func run(dir string, env map[string]string, stdout, stderr io.Writer, cmd string return sh.CmdRan(err), sh.ExitStatus(err), err } -func hashPath(path string) string { - h := sha256.Sum256([]byte(path)) - return hex.EncodeToString(h[:]) -} - func combineCoverage() error { files, err := filepath.Glob("coverage-*.txt") if err != nil { @@ -250,11 +263,5 @@ func combineCoverage() error { return err } - for _, file := range files { - if err := os.Remove(file); err != nil { - return err - } - } - return nil }