Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

magefile: added combined test coverage support for mage test:all #2235

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
+135 −66
Open

magefile: added combined test coverage support for mage test:all #2235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
80503bb
added test coverage
kartikaysaxena Feb 4, 2025
64f1377
different profiles for tests
kartikaysaxena Feb 5, 2025
3c14817
Merge branch 'main' into mage-test-coverage
kartikaysaxena Feb 8, 2025
59fd80d
use path hash instead
kartikaysaxena Feb 8, 2025
3f62b00
separate targets for cover
kartikaysaxena Feb 10, 2025
752a5ea
context coverage bool
kartikaysaxena Feb 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions magefiles/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/bufbuild/buf v1.35.1
github.com/ecordell/optgen v0.0.9
github.com/envoyproxy/protoc-gen-validate v1.0.4
github.com/google/uuid v1.1.2
github.com/magefile/mage v1.15.0
github.com/planetscale/vtprotobuf v0.6.1-0.20240409071808-615f978279ca
golang.org/x/tools v0.22.0
Expand Down
1 change: 1 addition & 0 deletions magefiles/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0Z
github.com/google/pprof v0.0.0-20240622144329-c177fd99eaa9 h1:ouFdLLCOyCfnxGpQTMZKHLyHr/D1GFbQzEsJxumO16E=
github.com/google/pprof v0.0.0-20240622144329-c177fd99eaa9/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
Expand Down
132 changes: 72 additions & 60 deletions magefiles/test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
package main

import (
"context"
"fmt"
"os"
"strings"
Expand All @@ -19,51 +20,62 @@ var emptyEnv map[string]string
func (t Test) All() error {
ds := Testds{}
c := Testcons{}
mg.Deps(t.Unit, t.Integration, t.Steelthread, t.Image, t.Analyzers,
ctx := context.Background()
cover := false
for _, arg := range os.Args {
if arg == "-cover=true" {
cover = true
break
}
}
ctx = context.WithValue(ctx, "cover", cover)
mg.CtxDeps(ctx, t.Unit, t.Integration, t.Steelthread, t.Image, t.Analyzers,
ds.Crdb, ds.Postgres, ds.Spanner, ds.Mysql,
c.Crdb, c.Spanner, c.Postgres, c.Mysql)

return nil
}

func (t Test) Combine() error {
return combineCoverage()
}

// UnitCover Runs the unit tests and generates a coverage report
func (t Test) UnitCover() error {
if err := t.unit(true); err != nil {
func (t Test) UnitCover(ctx context.Context) error {
if err := t.unit(ctx, true); err != nil {
return err
}
fmt.Println("Running coverage...")
return sh.RunV("go", "tool", "cover", "-html=coverage.txt")
}

// Unit Runs the unit tests
func (t Test) Unit() error {
return t.unit(false)
func (t Test) Unit(ctx context.Context) error {
return t.unit(ctx, false)
}

func (Test) unit(coverage bool) error {
func (Test) unit(ctx context.Context, coverage bool) error {
fmt.Println("running unit tests")
args := []string{"-tags", "ci,skipintegrationtests", "-race", "-timeout", "10m", "-count=1"}
if coverage {
args = append(args, "-covermode=atomic", "-coverprofile=coverage.txt")
}
return goTest("./...", args...)
return goTest(ctx, "./...", args...)
}

// Image Run tests that run the built image
func (Test) Image() error {
func (Test) Image(ctx context.Context) error {
mg.Deps(Build{}.Testimage)
return goDirTest("./cmd/spicedb", "./...", "-tags", "docker,image")
return goDirTest(ctx, "./cmd/spicedb", "./...", "-tags", "docker,image")
}

// Integration Run integration tests
func (Test) Integration() error {
func (Test) Integration(ctx context.Context) error {
mg.Deps(checkDocker)
return goTest("./internal/services/integrationtesting/...", "-tags", "ci,docker", "-timeout", "15m")
return goTest(ctx, "./internal/services/integrationtesting/...", "-tags", "ci,docker", "-timeout", "15m")
}

// Steelthread Run steelthread tests
func (Test) Steelthread() error {
func (Test) Steelthread(ctx context.Context) error {
fmt.Println("running steel thread tests")
return goTest("./internal/services/steelthreadtesting/...", "-tags", "steelthread,docker,image", "-timeout", "15m", "-v")
return goTest(ctx, "./internal/services/steelthreadtesting/...", "-tags", "steelthread,docker,image", "-timeout", "15m", "-v")
}

// RegenSteelthread Regenerate the steelthread tests
Expand All @@ -75,8 +87,8 @@ func (Test) RegenSteelthread() error {
}

// Analyzers Run the analyzer unit tests
func (Test) Analyzers() error {
return goDirTest("./tools/analyzers", "./...")
func (Test) Analyzers(ctx context.Context) error {
return goDirTest(ctx, "./tools/analyzers", "./...")
}

// Wasm Run wasm browser tests
Expand All @@ -95,99 +107,99 @@ func (Test) Wasm() error {
type Testds mg.Namespace

// Crdb Run datastore tests for crdb
func (tds Testds) Crdb() error {
return tds.crdb("")
func (tds Testds) Crdb(ctx context.Context) error {
return tds.crdb(ctx, "")
}

func (tds Testds) CrdbVer(version string) error {
return tds.crdb(version)
func (tds Testds) CrdbVer(ctx context.Context, version string) error {
return tds.crdb(ctx, version)
}

func (Testds) crdb(version string) error {
return datastoreTest("crdb", map[string]string{
func (Testds) crdb(ctx context.Context, version string) error {
return datastoreTest(ctx, "crdb", map[string]string{
"CRDB_TEST_VERSION": version,
})
}

// Spanner Run datastore tests for spanner
func (Testds) Spanner() error {
return datastoreTest("spanner", emptyEnv)
func (Testds) Spanner(ctx context.Context) error {
return datastoreTest(ctx, "spanner", emptyEnv)
}

// Postgres Run datastore tests for postgres
func (tds Testds) Postgres() error {
return tds.postgres("")
func (tds Testds) Postgres(ctx context.Context) error {
return tds.postgres(ctx, "")
}

func (tds Testds) PostgresVer(version string) error {
return tds.postgres(version)
func (tds Testds) PostgresVer(ctx context.Context, version string) error {
return tds.postgres(ctx, version)
}

func (Testds) postgres(version string) error {
return datastoreTest("postgres", map[string]string{
func (Testds) postgres(ctx context.Context, version string) error {
return datastoreTest(ctx, "postgres", map[string]string{
"POSTGRES_TEST_VERSION": version,
}, "postgres")
}

// Pgbouncer Run datastore tests for postgres with Pgbouncer
func (tds Testds) Pgbouncer() error {
return tds.pgbouncer("")
func (tds Testds) Pgbouncer(ctx context.Context) error {
return tds.pgbouncer(ctx, "")
}

func (tds Testds) PgbouncerVer(version string) error {
return tds.pgbouncer(version)
func (tds Testds) PgbouncerVer(ctx context.Context, version string) error {
return tds.pgbouncer(ctx, version)
}

func (Testds) pgbouncer(version string) error {
return datastoreTest("postgres", map[string]string{
func (Testds) pgbouncer(ctx context.Context, version string) error {
return datastoreTest(ctx, "postgres", map[string]string{
"POSTGRES_TEST_VERSION": version,
}, "pgbouncer")
}

// Mysql Run datastore tests for mysql
func (Testds) Mysql() error {
return datastoreTest("mysql", emptyEnv)
func (Testds) Mysql(ctx context.Context) error {
return datastoreTest(ctx, "mysql", emptyEnv)
}

func datastoreTest(datastore string, env map[string]string, tags ...string) error {
func datastoreTest(ctx context.Context, datastore string, env map[string]string, tags ...string) error {
mergedTags := append([]string{"ci", "docker"}, tags...)
tagString := strings.Join(mergedTags, ",")
mg.Deps(checkDocker)
return goDirTestWithEnv(".", fmt.Sprintf("./internal/datastore/%s/...", datastore), env, "-tags", tagString, "-timeout", "10m")
return goDirTestWithEnv(ctx, ".", fmt.Sprintf("./internal/datastore/%s/...", datastore), env, "-tags", tagString, "-timeout", "10m")
}

type Testcons mg.Namespace

// Crdb Run consistency tests for crdb
func (tc Testcons) Crdb() error {
return tc.crdb("")
func (tc Testcons) Crdb(ctx context.Context) error {
return tc.crdb(ctx, "")
}

func (tc Testcons) CrdbVer(version string) error {
return tc.crdb(version)
func (tc Testcons) CrdbVer(ctx context.Context, version string) error {
return tc.crdb(ctx, version)
}

func (Testcons) crdb(version string) error {
return consistencyTest("crdb", map[string]string{
func (Testcons) crdb(ctx context.Context, version string) error {
return consistencyTest(ctx, "crdb", map[string]string{
"CRDB_TEST_VERSION": version,
})
}

// Spanner Run consistency tests for spanner
func (Testcons) Spanner() error {
return consistencyTest("spanner", emptyEnv)
func (Testcons) Spanner(ctx context.Context) error {
return consistencyTest(ctx, "spanner", emptyEnv)
}

func (tc Testcons) Postgres() error {
return tc.postgres("")
func (tc Testcons) Postgres(ctx context.Context) error {
return tc.postgres(ctx, "")
}

func (tc Testcons) PostgresVer(version string) error {
return tc.postgres(version)
func (tc Testcons) PostgresVer(ctx context.Context, version string) error {
return tc.postgres(ctx, version)
}

func (Testcons) postgres(version string) error {
return consistencyTest("postgres", map[string]string{
func (Testcons) postgres(ctx context.Context, version string) error {
return consistencyTest(ctx, "postgres", map[string]string{
"POSTGRES_TEST_VERSION": version,
})
}
Expand All @@ -205,13 +217,13 @@ func (Testcons) PgbouncerVer(version string) error {
}

// Mysql Run consistency tests for mysql
func (Testcons) Mysql() error {
return consistencyTest("mysql", emptyEnv)
func (Testcons) Mysql(ctx context.Context) error {
return consistencyTest(ctx, "mysql", emptyEnv)
}

func consistencyTest(datastore string, env map[string]string) error {
func consistencyTest(ctx context.Context, datastore string, env map[string]string) error {
mg.Deps(checkDocker)
return goDirTestWithEnv(".", "./internal/services/integrationtesting/...",
return goDirTestWithEnv(ctx, ".", "./internal/services/integrationtesting/...",
env,
"-tags", "ci,docker,datastoreconsistency",
"-timeout", "10m",
Expand Down
67 changes: 61 additions & 6 deletions magefiles/util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,30 +3,59 @@
package main

import (
"context"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"strings"

"github.com/google/uuid"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
)

// run go test in the root
func goTest(path string, args ...string) error {
return goDirTest(".", path, args...)
func goTest(ctx context.Context, path string, args ...string) error {
return goDirTest(ctx, ".", path, args...)
}

// run go test in a directory
func goDirTest(dir string, path string, args ...string) error {
testArgs := append([]string{"test", "-failfast", "-count=1"}, args...)
func goDirTest(ctx context.Context, dir string, path string, args ...string) error {
testArgs := append([]string{
"test",
"-failfast",
"-count=1",
}, args...)
if cover, _ := ctx.Value("cover").(bool); cover {
if err := os.MkdirAll("coverage", 0o755); err != nil {
return fmt.Errorf("failed to create coverage directory: %w", err)
}
testArgs = append(testArgs, []string{
"-covermode=atomic",
fmt.Sprintf("-coverprofile=coverage-%s.txt", uuid.New().String()),
}...)
}
return RunSh(goCmdForTests(), WithV(), WithDir(dir), WithArgs(testArgs...))(path)
}

func goDirTestWithEnv(dir string, path string, env map[string]string, args ...string) error {
testArgs := append([]string{"test", "-failfast", "-count=1"}, args...)
func goDirTestWithEnv(ctx context.Context, dir string, path string, env map[string]string, args ...string) error {
testArgs := append([]string{
"test",
"-failfast",
"-count=1",
}, args...)
if cover, _ := ctx.Value("cover").(bool); cover {
if err := os.MkdirAll("coverage", 0o755); err != nil {
return fmt.Errorf("failed to create coverage directory: %w", err)
}
testArgs = append(testArgs, []string{
"-covermode=atomic",
fmt.Sprintf("-coverprofile=coverage-%s.txt", uuid.New().String()),
}...)
}
return RunSh(goCmdForTests(), WithV(), WithDir(dir), WithEnv(env), WithArgs(testArgs...))(path)
}

Expand Down Expand Up @@ -210,3 +239,29 @@ func run(dir string, env map[string]string, stdout, stderr io.Writer, cmd string
err = c.Run()
return sh.CmdRan(err), sh.ExitStatus(err), err
}

func combineCoverage() error {
files, err := filepath.Glob("coverage-*.txt")
if err != nil {
return err
}
if len(files) == 0 {
return fmt.Errorf("no coverage files found")
}

f, err := os.Create("coverage.txt")
if err != nil {
return err
}
defer f.Close()

args := []string{"run", "github.com/wadey/gocovmerge@latest"}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use go tool covdata merge, see https://go.dev/doc/build-cover

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it seems we need to build a binary for this for specifying the -cover flag before the tests which would generate the files(tests should be carried on the same binary?)

args = append(args, files...)

err = RunSh(goCmdForTests(), WithV(), WithStdout(f))(args...)
if err != nil {
return err
}

return nil
}