Skip to content

fix: remove hardcoded -short from coverage generation (#106) #194

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
em-redhat wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: remove hardcoded -short from coverage generation (#106) #194

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 32 additions & 13 deletions cmd/gaze/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -866,6 +866,7 @@ func newCrapCmd() *cobra.Command {
baselinePath string
analyzerFlag string
languageFlag string
testShort bool
)

cmd := &cobra.Command{
Expand Down Expand Up @@ -894,7 +895,9 @@ automatically.`,
opts.GazeCRAPThreshold = gazeCrapThreshold
opts.Stderr = os.Stderr
opts.ComplexityProvider = goprovider.NewComplexityProvider()
opts.LineCoverageProvider = goprovider.NewLineCoverageProvider(os.Stderr)
lineProv := goprovider.NewLineCoverageProvider(os.Stderr)
lineProv.Short = testShort
opts.LineCoverageProvider = lineProv
return runCrap(crapParams{
patterns: args,
format: format,
Expand Down Expand Up @@ -936,6 +939,8 @@ automatically.`,
"external analyzer binary (e.g., snake-eyes)")
cmd.Flags().StringVar(&languageFlag, "language", "",
"target language for analyzer discovery (e.g., python)")
cmd.Flags().BoolVar(&testShort, "test-short", false,
"pass -short to internal go test invocation (faster, less accurate coverage)")

return cmd
}
Expand Down Expand Up @@ -1460,8 +1465,10 @@ type selfCheckParams struct {
format string
maxCrapload int
maxGazeCrapload int
stdout io.Writer
stderr io.Writer
// testShort passes -short to the internal go test invocation when true.
testShort bool
stdout io.Writer
stderr io.Writer

// thresholdSet is true when any threshold flag was explicitly
// provided on the command line (via cmd.Flags().Changed). Passed
Expand Down Expand Up @@ -1505,7 +1512,9 @@ func runSelfCheck(p selfCheckParams) error {
selfOpts := crap.DefaultOptions()
selfOpts.Stderr = p.stderr
selfOpts.ComplexityProvider = goprovider.NewComplexityProvider()
selfOpts.LineCoverageProvider = goprovider.NewLineCoverageProvider(p.stderr)
lineProv := goprovider.NewLineCoverageProvider(p.stderr)
lineProv.Short = p.testShort
selfOpts.LineCoverageProvider = lineProv

cp := crapParams{
patterns: []string{"./..."},
Expand All @@ -1531,6 +1540,7 @@ func newSelfCheckCmd() *cobra.Command {
format string
maxCrapload int
maxGazeCrapload int
testShort bool
)

cmd := &cobra.Command{
Expand All @@ -1547,6 +1557,7 @@ scores are included when contract coverage data is available
format: format,
maxCrapload: maxCrapload,
maxGazeCrapload: maxGazeCrapload,
testShort: testShort,
stdout: os.Stdout,
stderr: os.Stderr,
thresholdSet: cmd.Flags().Changed("max-crapload") || cmd.Flags().Changed("max-gaze-crapload"),
Expand All @@ -1560,6 +1571,8 @@ scores are included when contract coverage data is available
"fail if CRAPload exceeds this count (0 = no limit)")
cmd.Flags().IntVar(&maxGazeCrapload, "max-gaze-crapload", 0,
"fail if GazeCRAPload exceeds this count (0 = no limit)")
cmd.Flags().BoolVar(&testShort, "test-short", true,
"pass -short to internal go test invocation (default true for self-check to avoid timeouts)")

return cmd
}
Expand All @@ -1579,8 +1592,10 @@ type reportParams struct {
coverProfile string
analyzerFlag string
languageFlag string
stdout io.Writer
stderr io.Writer
// testShort passes -short to internal go test invocations when true.
testShort bool
stdout io.Writer
stderr io.Writer

// runnerFunc overrides aireport.Run for testing. When nil, aireport.Run is called.
runnerFunc func(aireport.RunnerOptions) error
Expand Down Expand Up @@ -1690,6 +1705,7 @@ func runReport(p reportParams) error {
MaxGazeCrapload: p.maxGazeCrapload,
MinContractCoverage: p.minContractCoverage,
},
TestShort: p.testShort,
}

// External analyzer path: when --analyzer is set, override the
Expand Down Expand Up @@ -1787,13 +1803,14 @@ func captureReportJSON(fn func(w io.Writer) error) (json.RawMessage, error) {
// analysis operations and formats the result using an external AI CLI.
func newReportCmd() *cobra.Command {
var (
format string
adapterName string
modelName string
aiTimeout time.Duration
coverProfile string
analyzerFlag string
languageFlag string
format string
adapterName string
modelName string
aiTimeout time.Duration
coverProfile string
analyzerFlag string
languageFlag string
testShortFlag bool

// Threshold raw values and "was set" flags for *int semantics.
maxCraploadVal int
Expand Down Expand Up @@ -1851,6 +1868,7 @@ Examples:
coverProfile: coverProfile,
analyzerFlag: analyzerFlag,
languageFlag: languageFlag,
testShort: testShortFlag,
stdout: cmd.OutOrStdout(),
stderr: cmd.ErrOrStderr(),
}
Expand All @@ -1871,6 +1889,7 @@ Examples:
cmd.Flags().StringVar(&coverProfile, "coverprofile", "", "path to a pre-generated coverage profile (skips internal go test run)")
cmd.Flags().StringVar(&analyzerFlag, "analyzer", "", "external analyzer binary (e.g., snake-eyes)")
cmd.Flags().StringVar(&languageFlag, "language", "", "target language for analyzer discovery (e.g., python)")
cmd.Flags().BoolVar(&testShortFlag, "test-short", false, "pass -short to internal go test invocation (faster, less accurate coverage)")

return cmd
}
Expand Down
199 changes: 193 additions & 6 deletions cmd/gaze/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1430,14 +1430,18 @@ func TestRunSelfCheck_InvalidFormat(t *testing.T) {
}

func TestRunSelfCheck_TextFormat(t *testing.T) {

@jflowers jflowers Jul 20, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[CRITICAL] E2E test missing testShort: true — causes 30-minute CI timeout

The GAZE_COVERAGE_RUN guard prevents recursion, but the selfCheckParams below (line 1440) doesn't set testShort: true. Since testShort defaults to false (Go zero value), the internal go test ./... runs ALL tests without -short, exceeding CI's 30-minute timeout.

The CLI self-check command correctly defaults --test-short to true (line 1571), but the E2E test bypasses the CLI and uses the struct zero value. Add testShort: true to match the CLI default:

err := runSelfCheck(selfCheckParams{
    format:    "text",
    testShort: true, // match CLI default for self-check
    stdout:    &stdout,
    stderr:    &stderr,
})

Same fix needed for TestRunSelfCheck_JSONFormat at line 1461.

Both E2E tests (Go 1.24 and 1.25) panic with test timed out after 30m0s on TestRunSelfCheck_TextFormat. E2E passes on main where -short is still hardcoded.

if os.Getenv("GAZE_COVERAGE_RUN") != "" {
t.Skip("skipping: GAZE_COVERAGE_RUN set (recursion guard)")
}
if testing.Short() {
t.Skip("skipping self-check in short mode")
}
var stdout, stderr bytes.Buffer
err := runSelfCheck(selfCheckParams{
format: "text",
stdout: &stdout,
stderr: &stderr,
format: "text",
testShort: true, // match CLI default for self-check
stdout: &stdout,
stderr: &stderr,
})
if err != nil {
t.Fatalf("self-check text failed: %v", err)
Expand All @@ -1448,14 +1452,18 @@ func TestRunSelfCheck_TextFormat(t *testing.T) {
}

func TestRunSelfCheck_JSONFormat(t *testing.T) {
if os.Getenv("GAZE_COVERAGE_RUN") != "" {
t.Skip("skipping: GAZE_COVERAGE_RUN set (recursion guard)")

@jflowers jflowers Jul 20, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[CRITICAL] Same issue — add testShort: true to selfCheckParams

err := runSelfCheck(selfCheckParams{
    format:    "json",
    testShort: true, // match CLI default for self-check
    stdout:    &stdout,
    stderr:    &stderr,
})

}
if testing.Short() {
t.Skip("skipping self-check in short mode")
}
var stdout, stderr bytes.Buffer
err := runSelfCheck(selfCheckParams{
format: "json",
stdout: &stdout,
stderr: &stderr,
format: "json",
testShort: true, // match CLI default for self-check
stdout: &stdout,
stderr: &stderr,
})
if err != nil {
t.Fatalf("self-check json failed: %v", err)
Expand Down Expand Up @@ -2575,3 +2583,182 @@ func TestBuildAIMapperFunc_OllamaWithModel(t *testing.T) {
t.Fatal("expected non-nil AIMapperFunc for ollama with model")
}
}

// ---------------------------------------------------------------------------
// --test-short flag wiring tests (fix-hardcoded-short-flag Phase 2)
// ---------------------------------------------------------------------------

// TestCrapCmd_TestShortFlag verifies that the --test-short flag is
// recognized by the crap command and sets Short on the
// GoLineCoverageProvider. Uses cobra flag parsing to exercise the
// full newCrapCmd wiring.
func TestCrapCmd_TestShortFlag(t *testing.T) {
cmd := newCrapCmd()
// Verify the flag exists and has the expected default.
f := cmd.Flags().Lookup("test-short")
if f == nil {
t.Fatal("expected --test-short flag on crap command")
}
if f.DefValue != "false" {
t.Errorf("expected default value 'false', got %q", f.DefValue)
}
if f.Usage == "" {
t.Error("expected non-empty usage string for --test-short")
}
}

// TestRunCrap_TestShortThreadsToAnalyze verifies that when
// crapParams.opts.LineCoverageProvider is a *GoLineCoverageProvider
// with Short=true, the value reaches the analyzeFunc unchanged.
func TestRunCrap_TestShortThreadsToAnalyze(t *testing.T) {
var capturedOpts crap.Options
lineProv := goprovider.NewLineCoverageProvider(&bytes.Buffer{})
lineProv.Short = true

opts := crap.DefaultOptions()
opts.LineCoverageProvider = lineProv

var stdout, stderr bytes.Buffer
err := runCrap(crapParams{
patterns: []string{"./..."},
format: "text",
opts: opts,
moduleDir: ".",
stdout: &stdout,
stderr: &stderr,
analyzeFunc: func(_ []string, _ string, o crap.Options) (*crap.Report, error) {
capturedOpts = o
return &crap.Report{
Scores: []crap.Score{{Function: "Foo", Complexity: 1, LineCoverage: 100, CRAP: 1}},
}, nil
},
})
if err != nil {
t.Fatalf("runCrap returned error: %v", err)
}

prov, ok := capturedOpts.LineCoverageProvider.(*goprovider.GoLineCoverageProvider)
if !ok {
t.Fatalf("expected *GoLineCoverageProvider, got %T", capturedOpts.LineCoverageProvider)
}
if !prov.Short {
t.Error("expected GoLineCoverageProvider.Short=true to thread through to analyzeFunc")
}
}

// TestRunSelfCheck_TestShortThreadsToProvider verifies that when
// selfCheckParams.testShort is true, the GoLineCoverageProvider
// constructed in runSelfCheck has Short=true. Uses the runCrapFunc
// injection to capture the constructed crapParams.
func TestRunSelfCheck_TestShortThreadsToProvider(t *testing.T) {
var capturedOpts crap.Options
var stdout, stderr bytes.Buffer
err := runSelfCheck(selfCheckParams{
format: "text",
testShort: true,
stdout: &stdout,
stderr: &stderr,
moduleRootFunc: func() (string, error) {
return "/fake/module/root", nil
},
runCrapFunc: func(p crapParams) error {
capturedOpts = p.opts
return nil
},
})
if err != nil {
t.Fatalf("runSelfCheck returned error: %v", err)
}

// The LineCoverageProvider should be a *GoLineCoverageProvider
// with Short=true.
prov, ok := capturedOpts.LineCoverageProvider.(*goprovider.GoLineCoverageProvider)
if !ok {
t.Fatalf("expected *GoLineCoverageProvider, got %T", capturedOpts.LineCoverageProvider)
}
if !prov.Short {
t.Error("expected GoLineCoverageProvider.Short=true when testShort=true")
}
}

// TestRunSelfCheck_TestShortFalseWhenExplicit verifies that when
// selfCheckParams.testShort is explicitly false, the
// GoLineCoverageProvider has Short=false.
func TestRunSelfCheck_TestShortFalseWhenExplicit(t *testing.T) {
var capturedOpts crap.Options
var stdout, stderr bytes.Buffer
err := runSelfCheck(selfCheckParams{
format: "text",
// testShort not set — defaults to false
stdout: &stdout,
stderr: &stderr,
moduleRootFunc: func() (string, error) {
return "/fake/module/root", nil
},
runCrapFunc: func(p crapParams) error {
capturedOpts = p.opts
return nil
},
})
if err != nil {
t.Fatalf("runSelfCheck returned error: %v", err)
}

prov, ok := capturedOpts.LineCoverageProvider.(*goprovider.GoLineCoverageProvider)
if !ok {
t.Fatalf("expected *GoLineCoverageProvider, got %T", capturedOpts.LineCoverageProvider)
}
if prov.Short {
t.Error("expected GoLineCoverageProvider.Short=false when testShort not set")
}
}

// TestSelfCheckCmd_TestShortFlag verifies that the --test-short flag
// is recognized by the self-check command.
func TestSelfCheckCmd_TestShortFlag(t *testing.T) {
cmd := newSelfCheckCmd()
f := cmd.Flags().Lookup("test-short")
if f == nil {
t.Fatal("expected --test-short flag on self-check command")
}
if f.DefValue != "true" {
t.Errorf("expected default value 'true' for self-check, got %q", f.DefValue)
}
}

// TestRunReport_TestShortThreadsToRunnerOptions verifies that
// reportParams.testShort threads to RunnerOptions.TestShort when
// runReport calls the runner function.
func TestRunReport_TestShortThreadsToRunnerOptions(t *testing.T) {
var capturedTestShort bool
err := runReport(reportParams{
patterns: []string{"./..."},
format: "json",
testShort: true,
stdout: &bytes.Buffer{},
stderr: &bytes.Buffer{},
runnerFunc: func(opts aireport.RunnerOptions) error {
capturedTestShort = opts.TestShort
return nil
},
})
if err != nil {
t.Fatalf("runReport returned error: %v", err)
}
if !capturedTestShort {
t.Error("expected RunnerOptions.TestShort=true when reportParams.testShort=true")
}
}

// TestReportCmd_TestShortFlag verifies that the --test-short flag
// is recognized by the report command.
func TestReportCmd_TestShortFlag(t *testing.T) {
cmd := newReportCmd()
f := cmd.Flags().Lookup("test-short")
if f == nil {
t.Fatal("expected --test-short flag on report command")
}
if f.DefValue != "false" {
t.Errorf("expected default value 'false', got %q", f.DefValue)
}
}
Loading
Loading