cmd/go, testing: add TB.ArtifactDir and -artifacts flag

Add TB.ArtifactDir, which returns a directory for a test to store
output files in. Add a -artifacts testflag which enables persistent
storage of artifacts in the output directory (-outputdir, or the
current directory by default).

Fixes golang#71287

Change-Id: I5f6515a6cd6c103f88588f4c033d5ea11ffd0c3c
Reviewed-on: https://go-review.googlesource.com/c/go/+/696399
LUCI-TryBot-Result: Go LUCI <[email protected]>
Reviewed-by: Alan Donovan <[email protected]>

Author: neild

api/next/71287.txt

@@ -0,0 +1,4 @@
+pkg testing, method (*B) ArtifactDir() string #71287
+pkg testing, method (*F) ArtifactDir() string #71287
+pkg testing, method (*T) ArtifactDir() string #71287
+pkg testing, type TB interface, ArtifactDir() string #71287

doc/next/6-stdlib/99-minor/testing/71287.md

@@ -0,0 +1,18 @@
+The new methods [T.ArtifactDir], [B.ArtifactDir], and [F.ArtifactDir]
+return a directory in which to write test output files (artifacts).
+
+When the `-artifacts` flag is provided to `go test`,
+this directory will be located under the output directory
+(specified with `-outputdir`, or the current directory by default).
+Otherwise, artifacts are stored in a temporary directory
+which is removed after the test completes.
+
+The first call to `ArtifactDir` when `-artifacts` is provided
+writes the location of the directory to the test log.
+
+For example, in a test named `TestArtifacts`,
+`t.ArtifactDir()` emits:
+
+```
+=== ARTIFACTS Test /path/to/artifact/dir
+```

src/cmd/go/alldocs.go

@@ -649,6 +649,14 @@ func (t *testFuncs) ImportPath() string {
 	return pkg
 }
 
+func (t *testFuncs) ModulePath() string {
+	m := t.Package.Module
+	if m == nil {
+		return ""
+	}
+	return m.Path
+}
+
 // Covered returns a string describing which packages are being tested for coverage.
 // If the covered package is the same as the tested package, it returns the empty string.
 // Otherwise it is a comma-separated human-readable list of packages beginning with
@@ -836,6 +844,7 @@ func init() {
 	testdeps.CoverMarkProfileEmittedFunc = cfile.MarkProfileEmitted
 
 {{end}}
+	testdeps.ModulePath = {{.ModulePath | printf "%q"}}
 	testdeps.ImportPath = {{.ImportPath | printf "%q"}}
 }
 

src/cmd/go/internal/load/test.go

@@ -192,6 +192,10 @@ and -show_bytes options of pprof control how the information is presented.
 The following flags are recognized by the 'go test' command and
 control the execution of any test:
 
+	-artifacts
+	    Save test artifacts in the directory specified by -outputdir.
+	    See 'go doc testing.T.ArtifactDir'.
+
 	-bench regexp
 	    Run only those benchmarks matching a regular expression.
 	    By default, no benchmarks are run.
@@ -286,6 +290,10 @@ control the execution of any test:
 	    This will only list top-level tests. No subtest or subbenchmarks will be
 	    shown.
 
+	-outputdir directory
+	    Place output files from profiling and test artifacts in the
+	    specified directory, by default the directory in which "go test" is running.
+
 	-parallel n
 	    Allow parallel execution of test functions that call t.Parallel, and
 	    fuzz targets that call t.Parallel when running the seed corpus.
@@ -397,10 +405,6 @@ profile the tests during execution:
 	    Sample 1 in n stack traces of goroutines holding a
 	    contended mutex.
 
-	-outputdir directory
-	    Place output files from profiling in the specified directory,
-	    by default the directory in which "go test" is running.
-
 	-trace trace.out
 	    Write an execution trace to the specified file before exiting.
 
@@ -540,6 +544,7 @@ See the documentation of the testing package for more information.
 }
 
 var (
+	testArtifacts    bool                              // -artifacts flag
 	testBench        string                            // -bench flag
 	testC            bool                              // -c flag
 	testCoverPkgs    []*load.Package                   // -coverpkg flag

src/cmd/go/internal/test/flagdefs.go

@@ -44,6 +44,7 @@ func init() {
 	// some of them so that cmd/go knows what to do with the test output, or knows
 	// to build the test in a way that supports the use of the flag.
 
+	cf.BoolVar(&testArtifacts, "artifacts", false, "")
 	cf.StringVar(&testBench, "bench", "", "")
 	cf.Bool("benchmem", false, "")
 	cf.String("benchtime", "", "")
@@ -392,7 +393,8 @@ func testFlags(args []string) (packageNames, passToTest []string) {
 	// directory, but 'go test' defaults it to the working directory of the 'go'
 	// command. Set it explicitly if it is needed due to some other flag that
 	// requests output.
-	if testProfile() != "" && !outputDirSet {
+	needOutputDir := testProfile() != "" || testArtifacts
+	if needOutputDir && !outputDirSet {
 		injectedFlags = append(injectedFlags, "-test.outputdir="+testOutputDir.getAbs())
 	}
 

src/cmd/go/internal/test/test.go

@@ -38,6 +38,7 @@ type event struct {
 	FailedBuild string     `json:",omitempty"`
 	Key         string     `json:",omitempty"`
 	Value       string     `json:",omitempty"`
+	Path        string     `json:",omitempty"`
 }
 
 // textBytes is a hack to get JSON to emit a []byte as a string
@@ -180,6 +181,7 @@ var (
 		[]byte("=== FAIL  "),
 		[]byte("=== SKIP  "),
 		[]byte("=== ATTR  "),
+		[]byte("=== ARTIFACTS "),
 	}
 
 	reports = [][]byte{
@@ -251,7 +253,6 @@ func (c *Converter) handleInputLine(line []byte) {
 	// "=== RUN   "
 	// "=== PAUSE "
 	// "=== CONT  "
-	actionColon := false
 	origLine := line
 	ok := false
 	indent := 0
@@ -273,7 +274,6 @@ func (c *Converter) handleInputLine(line []byte) {
 		}
 		for _, magic := range reports {
 			if bytes.HasPrefix(line, magic) {
-				actionColon = true
 				ok = true
 				break
 			}
@@ -296,16 +296,11 @@ func (c *Converter) handleInputLine(line []byte) {
 		return
 	}
 
-	// Parse out action and test name.
-	i := 0
-	if actionColon {
-		i = bytes.IndexByte(line, ':') + 1
-	}
-	if i == 0 {
-		i = len(updates[0])
-	}
-	action := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(string(line[4:i])), ":"))
-	name := strings.TrimSpace(string(line[i:]))
+	// Parse out action and test name from "=== ACTION: Name".
+	action, name, _ := strings.Cut(string(line[len("=== "):]), " ")
+	action = strings.TrimSuffix(action, ":")
+	action = strings.ToLower(action)
+	name = strings.TrimSpace(name)
 
 	e := &event{Action: action}
 	if line[0] == '-' { // PASS or FAIL report
@@ -336,7 +331,10 @@ func (c *Converter) handleInputLine(line []byte) {
 		c.output.write(origLine)
 		return
 	}
-	if action == "attr" {
+	switch action {
+	case "artifacts":
+		name, e.Path, _ = strings.Cut(name, " ")
+	case "attr":
 		var rest string
 		name, rest, _ = strings.Cut(name, " ")
 		e.Key, e.Value, _ = strings.Cut(rest, " ")

src/cmd/go/internal/test/testflag.go

@@ -66,6 +66,12 @@ func (TestDeps) ImportPath() string {
 	return ImportPath
 }
 
+var ModulePath string
+
+func (TestDeps) ModulePath() string {
+	return ModulePath
+}
+
 // testLog implements testlog.Interface, logging actions by package os.
 type testLog struct {
 	mu  sync.Mutex