From c8faa06c59fcaae845e70747d9f49ef91ebff14b Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:33:00 +0000 Subject: [PATCH 001/124] feat(engine): introduce gcx-driven execution package engine is the seam between OATS and gcx. It captures the result of a single gcx invocation (stdout, stderr, exit code, duration) and exposes it through an Executor interface so the assertion path can be tested without launching a real binary. The production implementation (GCX) shells out via os/exec. It distinguishes "gcx ran and reported a non-zero status" (surfaced via Result.ExitCode, no Go error) from "gcx could not run" (Go error). Context cancellation gets its own error path so timeouts are not misread as failed assertions. No existing code is modified. Future commits wire this in as the replacement for the testhelpers/{remote,prometheus,tempo,requests} HTTP query path. Signed-off-by: Gregor Zeitlinger --- engine/engine.go | 119 ++++++++++++++++++++++++++++++++++++++++++ engine/engine_test.go | 81 ++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 engine/engine.go create mode 100644 engine/engine_test.go diff --git a/engine/engine.go b/engine/engine.go new file mode 100644 index 00000000..1a21f7e8 --- /dev/null +++ b/engine/engine.go @@ -0,0 +1,119 @@ +// Package engine executes gcx CLI commands and returns the result. +// +// engine is the seam between OATS and gcx: OATS no longer talks HTTP to Tempo, +// Loki, Prometheus, or Pyroscope. It builds a gcx command, runs it through an +// Executor, and hands the captured output to the assert package. +// +// The Executor interface keeps tests fast — production wiring uses GCX, which +// shells out via os/exec; tests use a stub that returns canned Results. +package engine + +import ( + "bytes" + "context" + "errors" + "fmt" + "os/exec" + "time" +) + +// Result captures everything a downstream assertion might need from a single +// gcx invocation. ExitCode is captured separately from any Go-level error so +// callers can distinguish "process did not start" from "process ran and +// reported a non-zero status." +type Result struct { + Command []string + Stdout string + Stderr string + ExitCode int + Duration time.Duration +} + +// Executor runs a gcx invocation and returns its Result. +// +// Implementations must never return a non-nil error for a non-zero exit code — +// that is conveyed via Result.ExitCode. A non-nil error means the process +// could not be launched (missing binary, context cancelled before start, +// permission denied, etc.). +type Executor interface { + Execute(ctx context.Context, args ...string) (*Result, error) +} + +// GCX is the production Executor. It shells out to a gcx binary on disk. +type GCX struct { + // Binary is the path to the gcx executable. Required. + Binary string + + // Context is the value passed via --context, prepended to every invocation + // when non-empty. OATS sets this per suite so that a single binary install + // can drive multiple Grafana endpoints. + Context string + + // Env supplements os.Environ() for the child process. Use to pass HOME / + // XDG_CONFIG_HOME for config isolation. + Env []string + + // Timeout caps a single invocation. Zero means no per-invocation timeout — + // the caller's context deadline still applies. + Timeout time.Duration +} + +// Execute runs gcx with args. The --context flag is prepended automatically +// when GCX.Context is set, so callers pass the verb and its flags (e.g. +// "traces", "search", "--since=10m", "{ ... }"). +func (g *GCX) Execute(ctx context.Context, args ...string) (*Result, error) { + if g.Binary == "" { + return nil, fmt.Errorf("engine: gcx binary path is empty") + } + + full := args + if g.Context != "" { + full = append([]string{"--context", g.Context}, args...) + } + + if g.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, g.Timeout) + defer cancel() + } + + cmd := exec.CommandContext(ctx, g.Binary, full...) + if len(g.Env) > 0 { + cmd.Env = append(cmd.Environ(), g.Env...) + } + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + start := time.Now() + err := cmd.Run() + duration := time.Since(start) + + result := &Result{ + Command: append([]string{g.Binary}, full...), + Stdout: stdout.String(), + Stderr: stderr.String(), + Duration: duration, + } + + // Timeout / cancellation takes precedence over exit code so callers can + // distinguish "gcx ran and failed" from "gcx was killed before it could + // answer." + if ctx.Err() != nil { + return result, fmt.Errorf("engine: gcx invocation aborted: %w", ctx.Err()) + } + + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + // Non-zero exit — surface via ExitCode, not via Go error. + result.ExitCode = exitErr.ExitCode() + return result, nil + } + // Process did not start, was killed by the OS, permission denied, etc. + return result, fmt.Errorf("engine: gcx invocation failed: %w", err) + } + + return result, nil +} diff --git a/engine/engine_test.go b/engine/engine_test.go new file mode 100644 index 00000000..2db75f29 --- /dev/null +++ b/engine/engine_test.go @@ -0,0 +1,81 @@ +package engine + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestGCX_ExecuteCapturesStdout(t *testing.T) { + g := &GCX{Binary: "/bin/sh"} + r, err := g.Execute(context.Background(), "-c", "echo hello") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if r.ExitCode != 0 { + t.Errorf("ExitCode: got %d, want 0", r.ExitCode) + } + if strings.TrimSpace(r.Stdout) != "hello" { + t.Errorf("Stdout: got %q, want %q", r.Stdout, "hello\n") + } +} + +func TestGCX_ExecuteCapturesStderr(t *testing.T) { + g := &GCX{Binary: "/bin/sh"} + r, err := g.Execute(context.Background(), "-c", "echo oops >&2") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if strings.TrimSpace(r.Stderr) != "oops" { + t.Errorf("Stderr: got %q, want %q", r.Stderr, "oops\n") + } +} + +func TestGCX_NonZeroExitIsNotAnError(t *testing.T) { + g := &GCX{Binary: "/bin/sh"} + r, err := g.Execute(context.Background(), "-c", "exit 7") + if err != nil { + t.Fatalf("non-zero exit should not return a Go error, got %v", err) + } + if r.ExitCode != 7 { + t.Errorf("ExitCode: got %d, want 7", r.ExitCode) + } +} + +func TestGCX_MissingBinaryIsAnError(t *testing.T) { + g := &GCX{Binary: ""} + _, err := g.Execute(context.Background(), "anything") + if err == nil { + t.Fatal("expected an error for empty binary path") + } +} + +func TestGCX_PrependsContextFlag(t *testing.T) { + g := &GCX{Binary: "/bin/sh", Context: "my-ctx"} + r, err := g.Execute(context.Background(), "-c", `printf "%s\n" "$@"`, "_", "verb") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // The command we asked sh to run echoes the positional args. With our + // Context set, gcx invocations gain a leading "--context my-ctx" pair. + // Since we're using /bin/sh as a stand-in for gcx, we observe the args + // by way of the recorded Command. + want := []string{"/bin/sh", "--context", "my-ctx", "-c", `printf "%s\n" "$@"`, "_", "verb"} + if len(r.Command) != len(want) { + t.Fatalf("Command len: got %d, want %d (%v)", len(r.Command), len(want), r.Command) + } + for i := range want { + if r.Command[i] != want[i] { + t.Errorf("Command[%d]: got %q, want %q", i, r.Command[i], want[i]) + } + } +} + +func TestGCX_TimeoutKillsLongProcess(t *testing.T) { + g := &GCX{Binary: "/bin/sh", Timeout: 50 * time.Millisecond} + _, err := g.Execute(context.Background(), "-c", "sleep 5") + if err == nil { + t.Fatal("expected an error when child overruns the timeout") + } +} From fc5e7dba56c8d6c360f2dd76e82ffb8ace1b3061 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:33:08 +0000 Subject: [PATCH 002/124] feat(assert): introduce assertion vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assert defines the OATS v2 case-yaml expectation language as a set of plain functions that return []Failure: contains, not_contains, regex, value, count, absent. No DSL, no gomega. Functions never short-circuit within a group — a case yaml that lists five substrings under contains gets all five reported in one pass. Regex compile errors surface as failures, not panics, so a malformed pattern in a case file fails loudly. Value parses ">= 0"-style comparison expressions used by metrics assertions. Count reuses the parser for cardinality checks. Absent is the spelling case authors use when "no results" is the expectation. Independent of engine and seed. Signed-off-by: Gregor Zeitlinger --- assert/assert.go | 160 ++++++++++++++++++++++++++++++++++++++++++ assert/assert_test.go | 97 +++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 assert/assert.go create mode 100644 assert/assert_test.go diff --git a/assert/assert.go b/assert/assert.go new file mode 100644 index 00000000..eb224577 --- /dev/null +++ b/assert/assert.go @@ -0,0 +1,160 @@ +// Package assert holds the vocabulary OATS uses to check gcx output. +// +// Each function takes the captured stdout (or a parsed structure derived from +// it) and an expectation, and returns a slice of failures — empty if the +// expectation held. Functions never panic; they never short-circuit on the +// first failure within a group. A case yaml that declares five Contains +// substrings reports all five misses in one run, not just the first. +// +// The shape mirrors the YAML keys documented in the OATS v2 impl plan: +// contains, not_contains, regex, value, count, absent. +package assert + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// Failure carries enough context to render a compact "FAIL " +// block. The Detail is the message body; the package never formats source +// pointers itself — that's the renderer's job. +type Failure struct { + Rule string // "contains", "regex", "value", ... + Detail string +} + +func (f Failure) Error() string { return f.Rule + ": " + f.Detail } + +// Contains checks that each substring appears at least once in stdout. +func Contains(stdout string, substrings []string) []Failure { + var fails []Failure + for _, s := range substrings { + if !strings.Contains(stdout, s) { + fails = append(fails, Failure{ + Rule: "contains", + Detail: fmt.Sprintf("substring %q not found in stdout", s), + }) + } + } + return fails +} + +// NotContains checks that none of the substrings appear in stdout. +func NotContains(stdout string, substrings []string) []Failure { + var fails []Failure + for _, s := range substrings { + if strings.Contains(stdout, s) { + fails = append(fails, Failure{ + Rule: "not_contains", + Detail: fmt.Sprintf("substring %q unexpectedly present in stdout", s), + }) + } + } + return fails +} + +// Regex checks that each pattern matches stdout at least once. An invalid +// pattern is itself reported as a failure — case yamls don't get to fail +// silently on a bad regex. +func Regex(stdout string, patterns []string) []Failure { + var fails []Failure + for _, p := range patterns { + re, err := regexp.Compile(p) + if err != nil { + fails = append(fails, Failure{ + Rule: "regex", + Detail: fmt.Sprintf("invalid pattern %q: %v", p, err), + }) + continue + } + if !re.MatchString(stdout) { + fails = append(fails, Failure{ + Rule: "regex", + Detail: fmt.Sprintf("pattern %q did not match stdout", p), + }) + } + } + return fails +} + +// Value parses a numeric comparison expression ("> 0", ">= 1.5", "== 42", +// "< 100", "<= 0", "!= 0") and checks it against the supplied actual value. +// Used by metrics assertions; the actual value comes from parsing gcx +// metrics query's JSON output upstream of this function. +func Value(actual float64, expr string) []Failure { + op, rhs, err := parseValueExpr(expr) + if err != nil { + return []Failure{{Rule: "value", Detail: err.Error()}} + } + if !applyComparison(actual, op, rhs) { + return []Failure{{ + Rule: "value", + Detail: fmt.Sprintf("expected value %s, got %v", expr, actual), + }} + } + return nil +} + +// Count is Value's sibling for integer cardinality. Same operators, integer +// rhs only — "== 0", ">= 1", "< 10". +func Count(actual int, expr string) []Failure { + // Delegate to Value via float64 — the parser is the same and integer + // comparisons through float64 are exact for any count we'd plausibly + // assert on. + return retag(Value(float64(actual), expr), "count") +} + +// Absent is a convenience over Count: it asserts the count is exactly zero. +// It is the spelling case-yaml authors use when "no traces matched" is the +// expectation. +func Absent(actual int) []Failure { + if actual != 0 { + return []Failure{{ + Rule: "absent", + Detail: fmt.Sprintf("expected zero results, got %d", actual), + }} + } + return nil +} + +func retag(fails []Failure, rule string) []Failure { + for i := range fails { + fails[i].Rule = rule + } + return fails +} + +func parseValueExpr(expr string) (op string, rhs float64, err error) { + expr = strings.TrimSpace(expr) + for _, candidate := range []string{">=", "<=", "==", "!=", ">", "<"} { + if strings.HasPrefix(expr, candidate) { + numStr := strings.TrimSpace(strings.TrimPrefix(expr, candidate)) + n, parseErr := strconv.ParseFloat(numStr, 64) + if parseErr != nil { + return "", 0, fmt.Errorf("invalid numeric rhs in %q: %v", expr, parseErr) + } + return candidate, n, nil + } + } + return "", 0, fmt.Errorf("expected comparison operator (>, >=, <, <=, ==, !=) at start of %q", expr) +} + +func applyComparison(lhs float64, op string, rhs float64) bool { + switch op { + case ">": + return lhs > rhs + case ">=": + return lhs >= rhs + case "<": + return lhs < rhs + case "<=": + return lhs <= rhs + case "==": + return lhs == rhs + case "!=": + return lhs != rhs + } + return false +} diff --git a/assert/assert_test.go b/assert/assert_test.go new file mode 100644 index 00000000..6c1c4adc --- /dev/null +++ b/assert/assert_test.go @@ -0,0 +1,97 @@ +package assert + +import ( + "testing" +) + +func TestContains(t *testing.T) { + cases := []struct { + name string + stdout string + substrings []string + wantFails int + }{ + {"all present", "hello world foo bar", []string{"hello", "foo"}, 0}, + {"one missing", "hello world", []string{"hello", "missing"}, 1}, + {"all missing", "x", []string{"a", "b", "c"}, 3}, + {"empty list", "anything", nil, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := Contains(tc.stdout, tc.substrings) + if len(got) != tc.wantFails { + t.Errorf("got %d failures, want %d: %v", len(got), tc.wantFails, got) + } + }) + } +} + +func TestNotContains(t *testing.T) { + if got := NotContains("hello world", []string{"goodbye"}); len(got) != 0 { + t.Errorf("expected zero failures, got %v", got) + } + if got := NotContains("hello world", []string{"hello"}); len(got) != 1 { + t.Errorf("expected one failure, got %d", len(got)) + } +} + +func TestRegex(t *testing.T) { + if got := Regex("err: 500", []string{`err: \d{3}`}); len(got) != 0 { + t.Errorf("expected match, got %v", got) + } + if got := Regex("err: 500", []string{`^err: 4\d{2}$`}); len(got) != 1 { + t.Errorf("expected one failure (no match), got %d", len(got)) + } + if got := Regex("anything", []string{`[invalid`}); len(got) != 1 { + t.Errorf("expected one failure (invalid pattern), got %d", len(got)) + } +} + +func TestValue(t *testing.T) { + cases := []struct { + name string + actual float64 + expr string + wantFails int + }{ + {">= holds", 5, ">= 1", 0}, + {">= fails", 0, ">= 1", 1}, + {"> holds", 1.5, "> 1", 0}, + {"> fails on equal", 1.0, "> 1", 1}, + {"<= holds", 0, "<= 0", 0}, + {"== holds", 42, "== 42", 0}, + {"== fails", 41, "== 42", 1}, + {"!= holds", 1, "!= 0", 0}, + {"bad operator", 0, "?? 1", 1}, + {"bad rhs", 0, ">= banana", 1}, + {"extra whitespace", 5, " >= 1 ", 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := Value(tc.actual, tc.expr) + if len(got) != tc.wantFails { + t.Errorf("got %d failures, want %d: %v", len(got), tc.wantFails, got) + } + }) + } +} + +func TestCount(t *testing.T) { + got := Count(3, ">= 1") + if len(got) != 0 { + t.Errorf("expected pass, got %v", got) + } + got = Count(0, ">= 1") + if len(got) != 1 || got[0].Rule != "count" { + t.Errorf("expected one count-tagged failure, got %v", got) + } +} + +func TestAbsent(t *testing.T) { + if got := Absent(0); len(got) != 0 { + t.Errorf("expected pass, got %v", got) + } + if got := Absent(2); len(got) != 1 || got[0].Rule != "absent" { + t.Errorf("expected one absent-tagged failure, got %v", got) + } +} From 6176d54e3e9ebf61f0ba58ceb87762df58859966 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:33:17 +0000 Subject: [PATCH 003/124] feat(seed): introduce inline-OTLP seed mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit seed pushes hand-crafted OTLP/HTTP JSON payloads to a running stack's /v1/{traces,logs,metrics} endpoints. This decouples gcx-contract tests from SDK behaviour: a case that asserts "gcx returns the trace I just pushed" can declare the trace inline instead of booting an instrumented example app. The wire format is hand-written JSON rather than the OTel Go SDK so test authors can read exactly what was sent when an assertion fails — no exporter middleware in between. Send processes signals in a fixed order (traces, logs, metrics) so partial sends leave the backend in a predictable state. Sane defaults (Kind=SPAN_KIND_INTERNAL, 200ms span duration, severity INFO) keep case yamls compact for the common case. Independent of engine and assert. Signed-off-by: Gregor Zeitlinger --- seed/seed.go | 210 ++++++++++++++++++++++++++++++++++++++++++++++ seed/seed_test.go | 141 +++++++++++++++++++++++++++++++ 2 files changed, 351 insertions(+) create mode 100644 seed/seed.go create mode 100644 seed/seed_test.go diff --git a/seed/seed.go b/seed/seed.go new file mode 100644 index 00000000..51af6758 --- /dev/null +++ b/seed/seed.go @@ -0,0 +1,210 @@ +// Package seed pushes known OTLP payloads into a running observability stack. +// +// The "inline-otlp" seed mode lets a case yaml carry its own data instead of +// booting an instrumented example app. This decouples gcx-contract tests from +// SDK behaviour: a case that wants to assert "gcx returns the trace I just +// pushed" can declare the trace inline, eliminating an entire layer of +// causation. +// +// The wire format is OTLP/HTTP JSON, which every supported backend accepts +// without an SDK on the test runner's side. We hand-write the JSON rather +// than pull in the OTel Go SDK because (a) the payloads are small and +// (b) we want the test author to see exactly what was sent when an assertion +// fails — no exporter middleware in between. +package seed + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "net/http" + "time" +) + +// Payload describes one inline-otlp seed declaration as it arrives from a +// case yaml. All three signal slices are optional; emitting only the ones +// the case cares about keeps inline payloads short. +type Payload struct { + Traces []Trace + Logs []Log + Metrics []Metric +} + +type Trace struct { + Service string + Span SpanFields +} + +type SpanFields struct { + Name string + // Kind defaults to 1 (SPAN_KIND_INTERNAL) when zero. + Kind int + // Duration defaults to 200ms when zero. + Duration time.Duration +} + +type Log struct { + Service string + Body string + SeverityNumber int // defaults to 9 (INFO) when zero + SeverityText string // defaults to "INFO" when empty +} + +type Metric struct { + Service string + Name string + Value int64 +} + +// Sender pushes a Payload at an OTLP/HTTP endpoint (e.g. http://localhost:4318). +// The endpoint is the base URL — the canonical /v1/{traces,logs,metrics} paths +// are appended internally. +type Sender struct { + OTLPEndpoint string + Client *http.Client +} + +func (s *Sender) httpClient() *http.Client { + if s.Client != nil { + return s.Client + } + return http.DefaultClient +} + +// Send pushes all signals declared in p. Returns the first error encountered, +// but processes signals in a fixed order (traces, logs, metrics) so that a +// partial send leaves the backend in a predictable state for assertions. +func (s *Sender) Send(p Payload) error { + if s.OTLPEndpoint == "" { + return fmt.Errorf("seed: OTLPEndpoint is empty") + } + now := time.Now() + for _, t := range p.Traces { + if err := s.sendTrace(t, now); err != nil { + return fmt.Errorf("seed traces: %w", err) + } + } + for _, l := range p.Logs { + if err := s.sendLog(l, now); err != nil { + return fmt.Errorf("seed logs: %w", err) + } + } + for _, m := range p.Metrics { + if err := s.sendMetric(m, now); err != nil { + return fmt.Errorf("seed metrics: %w", err) + } + } + return nil +} + +func (s *Sender) sendTrace(t Trace, now time.Time) error { + dur := t.Span.Duration + if dur == 0 { + dur = 200 * time.Millisecond + } + kind := t.Span.Kind + if kind == 0 { + kind = 1 + } + end := now.UnixNano() + start := now.Add(-dur).UnixNano() + + body := fmt.Sprintf(`{ + "resourceSpans": [{ + "resource": {"attributes": [ + {"key":"service.name","value":{"stringValue":%q}} + ]}, + "scopeSpans": [{ + "scope": {"name":"oats-inline-seed"}, + "spans": [{ + "traceId": %q, + "spanId": %q, + "name": %q, + "kind": %d, + "startTimeUnixNano": "%d", + "endTimeUnixNano": "%d" + }] + }] + }] +}`, t.Service, randHex(16), randHex(8), t.Span.Name, kind, start, end) + return s.post("/v1/traces", body) +} + +func (s *Sender) sendLog(l Log, now time.Time) error { + sev := l.SeverityNumber + if sev == 0 { + sev = 9 + } + sevText := l.SeverityText + if sevText == "" { + sevText = "INFO" + } + body := fmt.Sprintf(`{ + "resourceLogs": [{ + "resource": {"attributes": [ + {"key":"service.name","value":{"stringValue":%q}} + ]}, + "scopeLogs": [{ + "scope": {"name":"oats-inline-seed"}, + "logRecords": [{ + "timeUnixNano": "%d", + "severityNumber": %d, + "severityText": %q, + "body": {"stringValue": %q} + }] + }] + }] +}`, l.Service, now.UnixNano(), sev, sevText, l.Body) + return s.post("/v1/logs", body) +} + +func (s *Sender) sendMetric(m Metric, now time.Time) error { + end := now.UnixNano() + start := now.Add(-time.Second).UnixNano() + body := fmt.Sprintf(`{ + "resourceMetrics": [{ + "resource": {"attributes": [ + {"key":"service.name","value":{"stringValue":%q}} + ]}, + "scopeMetrics": [{ + "scope": {"name":"oats-inline-seed"}, + "metrics": [{ + "name": %q, + "sum": { + "isMonotonic": true, + "aggregationTemporality": 2, + "dataPoints": [{ + "startTimeUnixNano": "%d", + "timeUnixNano": "%d", + "asInt": "%d" + }] + } + }] + }] + }] +}`, m.Service, m.Name, start, end, m.Value) + return s.post("/v1/metrics", body) +} + +func (s *Sender) post(path, body string) error { + resp, err := s.httpClient().Post(s.OTLPEndpoint+path, "application/json", bytes.NewBufferString(body)) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + data, _ := io.ReadAll(resp.Body) + return fmt.Errorf("%s: %s\n%s", path, resp.Status, data) + } + return nil +} + +func randHex(nBytes int) string { + buf := make([]byte, nBytes) + if _, err := rand.Read(buf); err != nil { + panic(err) + } + return hex.EncodeToString(buf) +} diff --git a/seed/seed_test.go b/seed/seed_test.go new file mode 100644 index 00000000..10b1ccfb --- /dev/null +++ b/seed/seed_test.go @@ -0,0 +1,141 @@ +package seed + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +type recordingHandler struct { + mu sync.Mutex + requests map[string][]byte +} + +func (h *recordingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + h.mu.Lock() + h.requests[r.URL.Path] = body + h.mu.Unlock() + w.WriteHeader(http.StatusOK) +} + +func newRecorder() (*httptest.Server, *recordingHandler) { + h := &recordingHandler{requests: make(map[string][]byte)} + return httptest.NewServer(h), h +} + +func TestSender_SendTracesLogsMetrics(t *testing.T) { + srv, h := newRecorder() + defer srv.Close() + + s := &Sender{OTLPEndpoint: srv.URL} + err := s.Send(Payload{ + Traces: []Trace{{ + Service: "svc-a", + Span: SpanFields{Name: "do-thing"}, + }}, + Logs: []Log{{ + Service: "svc-a", + Body: "hello", + }}, + Metrics: []Metric{{ + Service: "svc-a", + Name: "things_total", + Value: 42, + }}, + }) + if err != nil { + t.Fatalf("Send: %v", err) + } + + if _, ok := h.requests["/v1/traces"]; !ok { + t.Error("traces endpoint not hit") + } + if _, ok := h.requests["/v1/logs"]; !ok { + t.Error("logs endpoint not hit") + } + if _, ok := h.requests["/v1/metrics"]; !ok { + t.Error("metrics endpoint not hit") + } + + for _, want := range []struct { + path, needle string + }{ + {"/v1/traces", "do-thing"}, + {"/v1/traces", "svc-a"}, + {"/v1/logs", "hello"}, + {"/v1/metrics", "things_total"}, + } { + if !strings.Contains(string(h.requests[want.path]), want.needle) { + t.Errorf("payload at %s missing %q:\n%s", want.path, want.needle, h.requests[want.path]) + } + } +} + +func TestSender_EmptyEndpointFailsLoudly(t *testing.T) { + s := &Sender{} + if err := s.Send(Payload{}); err == nil { + t.Fatal("expected an error for empty endpoint") + } +} + +func TestSender_BackendErrorPropagates(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("rejected")) + })) + defer srv.Close() + + s := &Sender{OTLPEndpoint: srv.URL} + err := s.Send(Payload{Traces: []Trace{{Service: "x", Span: SpanFields{Name: "y"}}}}) + if err == nil { + t.Fatal("expected an error from 400 response") + } + if !strings.Contains(err.Error(), "rejected") { + t.Errorf("error should include backend body, got: %v", err) + } +} + +func TestSender_SpanDefaultsApply(t *testing.T) { + srv, h := newRecorder() + defer srv.Close() + + before := time.Now() + s := &Sender{OTLPEndpoint: srv.URL} + err := s.Send(Payload{Traces: []Trace{{ + Service: "svc", + Span: SpanFields{Name: "n"}, + }}}) + if err != nil { + t.Fatal(err) + } + + // Decode just enough to verify defaults made it into the wire payload. + var parsed struct { + ResourceSpans []struct { + ScopeSpans []struct { + Spans []struct { + Kind int `json:"kind"` + StartTimeUnixNano string `json:"startTimeUnixNano"` + EndTimeUnixNano string `json:"endTimeUnixNano"` + } `json:"spans"` + } `json:"scopeSpans"` + } `json:"resourceSpans"` + } + if err := json.Unmarshal(h.requests["/v1/traces"], &parsed); err != nil { + t.Fatalf("parse: %v", err) + } + span := parsed.ResourceSpans[0].ScopeSpans[0].Spans[0] + if span.Kind != 1 { + t.Errorf("Kind default: got %d, want 1", span.Kind) + } + if span.StartTimeUnixNano == "" || span.EndTimeUnixNano == "" { + t.Errorf("timestamps empty: %+v", span) + } + _ = before +} From a7db5dbc73275da70c3fc99c8324b6e67ba393a5 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:35:56 +0000 Subject: [PATCH 004/124] feat(v2case): case yaml schema, loader, and validator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v2case is the in-memory representation of a v2 case yaml file. The struct shape mirrors the case yaml documented in the OATS v2 implementation plan: oats version, name, seed block (app | inline-otlp), expected block (traces / metrics / logs / profiles), with the contains / not_contains / regex / count / absent vocabulary embedded into each signal's assertion struct. Load reads a file from disk; Parse is the byte-slice counterpart for tests. KnownFields(true) on the yaml decoder rejects typos in case yamls rather than silently dropping them. Validate enforces structural rules a parser cannot — required fields, seed.type discriminant, inline-otlp must declare at least one signal payload, a case with no expectations cannot fail and is therefore an error. IsHermetic defaults to true; cases sharing a fixture must namespace their data, and the runner will rely on this contract when v2 lands shared-fixture support. Signed-off-by: Gregor Zeitlinger --- v2case/case.go | 191 ++++++++++++++++++++++++++++++++++++++++++++ v2case/case_test.go | 145 +++++++++++++++++++++++++++++++++ 2 files changed, 336 insertions(+) create mode 100644 v2case/case.go create mode 100644 v2case/case_test.go diff --git a/v2case/case.go b/v2case/case.go new file mode 100644 index 00000000..f2772f33 --- /dev/null +++ b/v2case/case.go @@ -0,0 +1,191 @@ +// Package v2case is the in-memory representation of an OATS v2 case yaml. +// +// A case yaml describes one observable behaviour to verify: how to seed data +// into the stack, and what gcx queries should return. Cases declare what +// they expect, not how to run gcx — the runner picks gcx args from the +// expectation block. +// +// The struct shape mirrors the case yaml documented in the OATS v2 +// implementation plan (internal-docs #14). Field tags follow the yaml.v3 +// convention. +package v2case + +import ( + "bytes" + "fmt" + "os" + + "go.yaml.in/yaml/v3" +) + +// SchemaVersion is the value of the top-level `oats:` field that this loader +// understands. Cases with any other value are rejected at parse time. +const SchemaVersion = 2 + +// Case is one entry point yaml file. Cases are independently runnable; +// suites group them via oats.toml. +type Case struct { + OatsVersion int `yaml:"oats"` + Name string `yaml:"name"` + Tags []string `yaml:"tags,omitempty"` + Hermetic *bool `yaml:"hermetic,omitempty"` // pointer: distinguish unset vs explicit false + + Seed Seed `yaml:"seed"` + Expected Expected `yaml:"expected"` + + // SourcePath is filled by the loader; not part of the yaml surface. + SourcePath string `yaml:"-"` +} + +// Seed declares how a case populates the stack before assertions run. +// Exactly one of the fields beyond Type is meaningful per Type value: +// +// type: app → Compose is required +// type: inline-otlp → Traces/Logs/Metrics describe the payload to push +type Seed struct { + Type string `yaml:"type"` // "app" or "inline-otlp" + Compose string `yaml:"compose,omitempty"` + Traces []SeedTrace `yaml:"traces,omitempty"` + Logs []SeedLog `yaml:"logs,omitempty"` + Metrics []SeedMetric `yaml:"metrics,omitempty"` + Vars map[string]any `yaml:"vars,omitempty"` +} + +type SeedTrace struct { + Service string `yaml:"service"` + Spans []SeedSpan `yaml:"spans"` +} + +type SeedSpan struct { + Name string `yaml:"name"` + Kind int `yaml:"kind,omitempty"` + Duration string `yaml:"duration,omitempty"` // human duration ("200ms") +} + +type SeedLog struct { + Service string `yaml:"service"` + Body string `yaml:"body"` + SeverityNumber int `yaml:"severity_number,omitempty"` + SeverityText string `yaml:"severity_text,omitempty"` +} + +type SeedMetric struct { + Service string `yaml:"service"` + Name string `yaml:"name"` + Value int64 `yaml:"value"` +} + +// Expected groups per-signal assertion blocks. A case may omit any signal it +// does not care about; an empty Expected makes the case a no-op (rejected at +// validation). +type Expected struct { + Traces []TraceAssertion `yaml:"traces,omitempty"` + Metrics []MetricAssertion `yaml:"metrics,omitempty"` + Logs []LogAssertion `yaml:"logs,omitempty"` + Profiles []ProfileAssertion `yaml:"profiles,omitempty"` +} + +// AssertionCommon holds the keys every signal-type assertion supports. +// Embedded into each concrete assertion struct so a case author can mix and +// match contains / regex / count / absent on any signal. +type AssertionCommon struct { + Contains []string `yaml:"contains,omitempty"` + NotContains []string `yaml:"not_contains,omitempty"` + Regex []string `yaml:"regex,omitempty"` + Count string `yaml:"count,omitempty"` // ">= 1", "== 0", ... + Absent bool `yaml:"absent,omitempty"` +} + +type TraceAssertion struct { + TraceQL string `yaml:"traceql"` + AssertionCommon `yaml:",inline"` +} + +type MetricAssertion struct { + PromQL string `yaml:"promql"` + Value string `yaml:"value,omitempty"` // ">= 0", "== 42", ... + AssertionCommon `yaml:",inline"` +} + +type LogAssertion struct { + LogQL string `yaml:"logql"` + AssertionCommon `yaml:",inline"` +} + +type ProfileAssertion struct { + Query string `yaml:"query"` + AssertionCommon `yaml:",inline"` +} + +// Load reads a yaml file from disk and returns a parsed, validated Case. +// Returns an error if the file is missing, the yaml is malformed, or the +// case violates any structural rule (see Validate). +func Load(path string) (*Case, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("v2case load %s: %w", path, err) + } + c, err := Parse(data) + if err != nil { + return nil, fmt.Errorf("v2case parse %s: %w", path, err) + } + c.SourcePath = path + return c, nil +} + +// Parse is Load's byte-slice counterpart. Useful in tests that hold yaml +// inline. +func Parse(data []byte) (*Case, error) { + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) // reject unknown keys + + var c Case + if err := dec.Decode(&c); err != nil { + return nil, err + } + if err := c.Validate(); err != nil { + return nil, err + } + return &c, nil +} + +// Validate checks structural rules a yaml parser cannot enforce on its own. +// Called automatically by Parse; exported for tests that construct Cases +// programmatically. +func (c *Case) Validate() error { + if c.OatsVersion != SchemaVersion { + return fmt.Errorf("oats: expected %d, got %d (this binary parses v%d only)", + SchemaVersion, c.OatsVersion, SchemaVersion) + } + if c.Name == "" { + return fmt.Errorf("name: required, non-empty") + } + switch c.Seed.Type { + case "app": + if c.Seed.Compose == "" { + return fmt.Errorf("seed.compose: required when seed.type = app") + } + case "inline-otlp": + if len(c.Seed.Traces)+len(c.Seed.Logs)+len(c.Seed.Metrics) == 0 { + return fmt.Errorf("seed: inline-otlp must declare at least one trace, log, or metric") + } + case "": + return fmt.Errorf("seed.type: required (app | inline-otlp)") + default: + return fmt.Errorf("seed.type: unknown value %q (expected app or inline-otlp)", c.Seed.Type) + } + if len(c.Expected.Traces)+len(c.Expected.Metrics)+len(c.Expected.Logs)+len(c.Expected.Profiles) == 0 { + return fmt.Errorf("expected: at least one signal assertion required (a case with no expectations cannot fail)") + } + return nil +} + +// IsHermetic reports whether this case promises not to interfere with siblings +// sharing the same fixture. Default is true; cases opt out by setting +// `hermetic: false`. +func (c *Case) IsHermetic() bool { + if c.Hermetic == nil { + return true + } + return *c.Hermetic +} diff --git a/v2case/case_test.go b/v2case/case_test.go new file mode 100644 index 00000000..05bfe16e --- /dev/null +++ b/v2case/case_test.go @@ -0,0 +1,145 @@ +package v2case + +import ( + "strings" + "testing" +) + +func TestParse_AppSeed(t *testing.T) { + src := []byte(` +oats: 2 +name: rolldice traces have route attribute +seed: + type: app + compose: docker-compose.app.yml +expected: + traces: + - traceql: '{ span.http.route = "/rolldice" }' + contains: ["GET /rolldice"] + metrics: + - promql: 'rolls_total' + value: '>= 0' +`) + c, err := Parse(src) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if c.Name != "rolldice traces have route attribute" { + t.Errorf("Name: got %q", c.Name) + } + if c.Seed.Type != "app" || c.Seed.Compose != "docker-compose.app.yml" { + t.Errorf("Seed: %+v", c.Seed) + } + if len(c.Expected.Traces) != 1 || c.Expected.Traces[0].TraceQL == "" { + t.Errorf("Expected.Traces: %+v", c.Expected.Traces) + } + if got := c.Expected.Metrics[0].Value; got != ">= 0" { + t.Errorf("Value: got %q", got) + } +} + +func TestParse_InlineOTLPSeed(t *testing.T) { + src := []byte(` +oats: 2 +name: gcx returns seeded trace +seed: + type: inline-otlp + traces: + - service: gcx-e2e-seed + spans: + - name: seed-operation +expected: + traces: + - traceql: '{ resource.service.name = "gcx-e2e-seed" }' + contains: ["gcx-e2e-seed", "seed-operation"] +`) + c, err := Parse(src) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if c.Seed.Type != "inline-otlp" { + t.Errorf("Seed.Type: got %q", c.Seed.Type) + } + if len(c.Seed.Traces) != 1 || c.Seed.Traces[0].Service != "gcx-e2e-seed" { + t.Errorf("Seed.Traces: %+v", c.Seed.Traces) + } +} + +func TestParse_RejectsUnknownFields(t *testing.T) { + src := []byte(` +oats: 2 +name: typo +seed: + type: app + compose: x.yml + composr: y.yml # typo +expected: + traces: + - traceql: '{ ... }' + contains: ["x"] +`) + _, err := Parse(src) + if err == nil { + t.Fatal("expected error for unknown field") + } +} + +func TestValidate_MissingOats(t *testing.T) { + c := &Case{Name: "x", Seed: Seed{Type: "app", Compose: "y"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), "oats:") { + t.Errorf("expected oats version error, got %v", err) + } +} + +func TestValidate_MissingName(t *testing.T) { + c := &Case{OatsVersion: 2, Seed: Seed{Type: "app", Compose: "y"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), "name:") { + t.Errorf("expected name error, got %v", err) + } +} + +func TestValidate_UnknownSeedType(t *testing.T) { + c := &Case{OatsVersion: 2, Name: "x", Seed: Seed{Type: "weird"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), "seed.type") { + t.Errorf("expected seed type error, got %v", err) + } +} + +func TestValidate_AppSeedNeedsCompose(t *testing.T) { + c := &Case{OatsVersion: 2, Name: "x", Seed: Seed{Type: "app"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), "seed.compose") { + t.Errorf("expected compose error, got %v", err) + } +} + +func TestValidate_InlineOTLPNeedsPayload(t *testing.T) { + c := &Case{OatsVersion: 2, Name: "x", Seed: Seed{Type: "inline-otlp"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), "at least one trace") { + t.Errorf("expected payload error, got %v", err) + } +} + +func TestValidate_NoExpectations(t *testing.T) { + c := &Case{OatsVersion: 2, Name: "x", Seed: Seed{Type: "app", Compose: "y"}} + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), "expected:") { + t.Errorf("expected 'no expectations' error, got %v", err) + } +} + +func TestIsHermetic(t *testing.T) { + c := &Case{} + if !c.IsHermetic() { + t.Error("default should be hermetic") + } + f := false + c.Hermetic = &f + if c.IsHermetic() { + t.Error("explicit false should override") + } +} From d4e57794e172829351bd3a81bf6b08c67b26a424 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:38:41 +0000 Subject: [PATCH 005/124] feat(report): event-driven Reporter with compact text and NDJSON MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit report carries OATS v2's observation channel. Every interesting moment in a run — a case started, an assertion failed, a fixture became ready — is published as an Event. A Reporter renders the stream for a particular audience. Two renderers are included: - TextReporter is the default. At default verbosity it is silent on success, emits a detailed FAIL block per failed assertion, and finishes with one PASS/FAIL summary line. Verbosity flags (VerbosePasses / VerboseCmd / VerboseAll) progressively unlock per-pass lines, gcx invocation echoes, and fixture lifecycle. When GITHUB_ACTIONS=true is in the environment, failures additionally emit ::error file=...,line=... annotations so reviewers see them inline on PR diffs. Duplicate annotations at the same source position are suppressed. - NDJSONReporter writes one JSON object per Event for tooling that wants to consume events structurally. Pass and fixture lifecycle events are filtered by verbosity in the same spirit as the text renderer — agents shouldn't pay tokens for "everything went fine." SchemaVersion (1) travels with each run.start event. Consumers pin to a version; we bump on breaking changes only. splitSource carries the path:line parsing used by both the failure block headers and the GHA annotations. Tested for the edge cases that matter (no colon, non-numeric suffix, colons earlier in the path). Signed-off-by: Gregor Zeitlinger --- report/event.go | 99 +++++++++++++++++++++++ report/ndjson.go | 48 +++++++++++ report/report_test.go | 153 +++++++++++++++++++++++++++++++++++ report/text.go | 181 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 481 insertions(+) create mode 100644 report/event.go create mode 100644 report/ndjson.go create mode 100644 report/report_test.go create mode 100644 report/text.go diff --git a/report/event.go b/report/event.go new file mode 100644 index 00000000..20e290d6 --- /dev/null +++ b/report/event.go @@ -0,0 +1,99 @@ +// Package report carries OATS v2's observation channel. +// +// Every interesting moment in a run — a case started, an assertion failed, a +// fixture became ready — is published as an Event. A Reporter renders the +// stream for a particular audience: the compact text format for humans and +// agents alike (the default), or NDJSON for tooling that wants to consume +// events structurally. +// +// The same Event vocabulary feeds both renderers. The CI-annotation emitter +// is a thin layer over the text renderer that fires when GITHUB_ACTIONS=true. +package report + +import "time" + +// EventType is the canonical lifecycle vocabulary. New types should be added +// here and then handled by all renderers — emitting an unknown type must +// remain safe (renderers either drop it or render a neutral line). +type EventType string + +const ( + EventRunStart EventType = "run.start" + EventRunEnd EventType = "run.end" + EventSuiteStart EventType = "suite.start" + EventSuiteEnd EventType = "suite.end" + EventFixtureStart EventType = "fixture.start" + EventFixtureReady EventType = "fixture.ready" + EventFixtureTeardown EventType = "fixture.teardown" + EventCaseStart EventType = "case.start" + EventCasePass EventType = "case.pass" + EventCaseFail EventType = "case.fail" + EventCaseSkip EventType = "case.skip" + EventAssertFail EventType = "assert.fail" + EventGCXExec EventType = "gcx.exec" +) + +// SchemaVersion travels with each run.start event. Consumers pin to a +// version; we bump on any breaking change (key removed, key semantics +// changed). Additive changes are not breaking. +const SchemaVersion = 1 + +// Event is the single payload type that crosses the Reporter boundary. +// Fields are sparse on purpose — each event type populates only what it +// needs, and Reporters MUST tolerate missing values for fields outside an +// event's natural set. +// +// JSON tags drive NDJSON output. The TextReporter uses the same fields but +// formats them prose-side. +type Event struct { + Type EventType `json:"event"` + Ts time.Time `json:"ts,omitempty"` + + // Set on run.start only. + OatsVersion string `json:"oats_version,omitempty"` + SchemaVersion int `json:"schema_version,omitempty"` + + // Identifying context. + Suite string `json:"suite,omitempty"` + Fixture string `json:"fixture,omitempty"` + FixtureType string `json:"fixture_type,omitempty"` + Case string `json:"case,omitempty"` + Source string `json:"source,omitempty"` + + // Per-assertion / per-failure detail. + Msg string `json:"msg,omitempty"` + Cmd string `json:"cmd,omitempty"` + StdoutExcerpt string `json:"stdout_excerpt,omitempty"` + + // Timing and aggregate counts. + DurationMs int64 `json:"duration_ms,omitempty"` + CaseCount int `json:"case_count,omitempty"` + Pass int `json:"pass,omitempty"` + Fail int `json:"fail,omitempty"` + Skip int `json:"skip,omitempty"` +} + +// Reporter consumes Events as they happen and flushes its final output when +// Close is called. Concrete implementations must be safe for sequential use +// (no concurrent Emit calls — the runner serialises them). +type Reporter interface { + Emit(e Event) + Close() error +} + +// Verbosity controls how much detail TextReporter renders. NDJSON is +// unaffected — it emits everything except gcx.exec events, which are only +// included at VerboseAll because they can be very large. +type Verbosity int + +const ( + // VerboseDefault prints failures plus the final summary. Pass events, + // fixture lifecycle, and gcx exec details are silent. + VerboseDefault Verbosity = iota + // VerbosePasses adds one line per passing case. + VerbosePasses + // VerboseCmd adds the gcx invocation behind each assertion (pass or fail). + VerboseCmd + // VerboseAll adds fixture lifecycle and full gcx stdout / per-phase timing. + VerboseAll +) diff --git a/report/ndjson.go b/report/ndjson.go new file mode 100644 index 00000000..7e53f757 --- /dev/null +++ b/report/ndjson.go @@ -0,0 +1,48 @@ +package report + +import ( + "encoding/json" + "io" +) + +// NDJSONReporter emits one JSON object per Event on its own line. Suitable +// for tooling that wants to consume events structurally. Pass and gcx.exec +// events are filtered by Verbosity in the same spirit as TextReporter so the +// token cost of "everything went fine" remains zero. +type NDJSONReporter struct { + w io.Writer + enc *json.Encoder + v Verbosity +} + +func NewNDJSONReporter(w io.Writer, v Verbosity) *NDJSONReporter { + enc := json.NewEncoder(w) + // Compact form: no HTML escaping, no extra whitespace beyond the + // trailing newline json.Encoder emits per Encode. + enc.SetEscapeHTML(false) + return &NDJSONReporter{w: w, enc: enc, v: v} +} + +func (r *NDJSONReporter) Emit(e Event) { + if !r.shouldEmit(e) { + return + } + // Failure to write the stream is silent on purpose: a downstream pipe + // that died should not crash the runner. The terminal exit status still + // reflects the test outcome. + _ = r.enc.Encode(e) +} + +func (r *NDJSONReporter) Close() error { return nil } + +func (r *NDJSONReporter) shouldEmit(e Event) bool { + switch e.Type { + case EventCasePass: + return r.v >= VerbosePasses + case EventGCXExec: + return r.v >= VerboseAll + case EventFixtureStart, EventFixtureReady, EventFixtureTeardown: + return r.v >= VerboseAll + } + return true +} diff --git a/report/report_test.go b/report/report_test.go new file mode 100644 index 00000000..f80902f1 --- /dev/null +++ b/report/report_test.go @@ -0,0 +1,153 @@ +package report + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" +) + +func TestTextReporter_SilentOnAllPass(t *testing.T) { + var buf bytes.Buffer + r := NewTextReporter(&buf, VerboseDefault) + r.Emit(Event{Type: EventRunStart, OatsVersion: "test"}) + r.Emit(Event{Type: EventCasePass, Case: "a"}) + r.Emit(Event{Type: EventCasePass, Case: "b"}) + r.Emit(Event{Type: EventRunEnd, DurationMs: 100}) + + out := buf.String() + if strings.Contains(out, "FAIL") { + t.Errorf("no FAIL block expected on all-pass run:\n%s", out) + } + if !strings.Contains(out, "PASS 2/2") { + t.Errorf("summary line missing:\n%s", out) + } +} + +func TestTextReporter_FailureBlockHasSourceAndCmd(t *testing.T) { + var buf bytes.Buffer + r := NewTextReporter(&buf, VerboseDefault) + r.Emit(Event{Type: EventRunStart}) + r.Emit(Event{Type: EventCaseStart, Case: "rolldice", Source: "examples/nodejs/oats.yaml:1"}) + r.Emit(Event{ + Type: EventAssertFail, + Case: "rolldice", + Source: "examples/nodejs/oats.yaml:8", + Msg: "TraceQL returned no results", + Cmd: "gcx traces search '{ span.http.route = \"/rolldice\" }'", + }) + r.Emit(Event{Type: EventCaseFail, Case: "rolldice"}) + r.Emit(Event{Type: EventRunEnd}) + + out := buf.String() + if !strings.Contains(out, "FAIL rolldice examples/nodejs/oats.yaml:8") { + t.Errorf("FAIL header missing or wrong:\n%s", out) + } + if !strings.Contains(out, "TraceQL returned no results") { + t.Errorf("failure message missing:\n%s", out) + } + if !strings.Contains(out, "gcx traces search") { + t.Errorf("command missing:\n%s", out) + } + if !strings.Contains(out, "FAIL 0/1 (1 failed") { + t.Errorf("summary line missing or wrong:\n%s", out) + } +} + +func TestTextReporter_GHAAnnotationsWhenEnabled(t *testing.T) { + t.Setenv("GITHUB_ACTIONS", "true") + + var buf bytes.Buffer + r := NewTextReporter(&buf, VerboseDefault) + r.Emit(Event{Type: EventRunStart}) + r.Emit(Event{ + Type: EventAssertFail, + Case: "x", + Source: "examples/nodejs/oats.yaml:8", + Msg: "oops", + }) + // Same source twice — only one annotation should appear. + r.Emit(Event{ + Type: EventAssertFail, + Case: "x", + Source: "examples/nodejs/oats.yaml:8", + Msg: "again", + }) + r.Emit(Event{Type: EventCaseFail, Case: "x"}) + r.Emit(Event{Type: EventRunEnd}) + + out := buf.String() + expectedAnnotation := "::error file=examples/nodejs/oats.yaml,line=8::oops" + if !strings.Contains(out, expectedAnnotation) { + t.Errorf("expected annotation missing:\n%s", out) + } + if strings.Count(out, "::error file=") != 1 { + t.Errorf("duplicate annotations not deduplicated:\n%s", out) + } +} + +func TestTextReporter_VerbosePassPrintsPasses(t *testing.T) { + var buf bytes.Buffer + r := NewTextReporter(&buf, VerbosePasses) + r.Emit(Event{Type: EventRunStart}) + r.Emit(Event{Type: EventCasePass, Case: "a"}) + r.Emit(Event{Type: EventRunEnd}) + + if !strings.Contains(buf.String(), "PASS a\n") { + t.Errorf("expected per-case PASS line:\n%s", buf.String()) + } +} + +func TestNDJSONReporter_EmitsOneJSONObjectPerLine(t *testing.T) { + var buf bytes.Buffer + r := NewNDJSONReporter(&buf, VerboseDefault) + r.Emit(Event{Type: EventRunStart, OatsVersion: "x", SchemaVersion: 1, Ts: time.Now()}) + r.Emit(Event{Type: EventCaseFail, Case: "rolldice", DurationMs: 1234}) + r.Emit(Event{Type: EventRunEnd, Pass: 0, Fail: 1}) + + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + if len(lines) != 3 { + t.Fatalf("expected 3 lines, got %d:\n%s", len(lines), buf.String()) + } + for _, line := range lines { + var e map[string]any + if err := json.Unmarshal([]byte(line), &e); err != nil { + t.Errorf("invalid JSON line %q: %v", line, err) + } + } +} + +func TestNDJSONReporter_FiltersPassByDefault(t *testing.T) { + var buf bytes.Buffer + r := NewNDJSONReporter(&buf, VerboseDefault) + r.Emit(Event{Type: EventCasePass, Case: "a"}) + r.Emit(Event{Type: EventCaseFail, Case: "b"}) + + if strings.Contains(buf.String(), `"case.pass"`) { + t.Errorf("pass event leaked through default verbosity:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), `"case.fail"`) { + t.Errorf("fail event missing:\n%s", buf.String()) + } +} + +func TestSplitSource(t *testing.T) { + cases := []struct { + in string + wantFile string + wantLine int + }{ + {"a.yaml:42", "a.yaml", 42}, + {"a.yaml", "a.yaml", 0}, + {"path/with/colon:x/a.yaml:7", "path/with/colon:x/a.yaml", 7}, + {"a.yaml:notanint", "a.yaml:notanint", 0}, + {"", "", 0}, + } + for _, tc := range cases { + f, n := splitSource(tc.in) + if f != tc.wantFile || n != tc.wantLine { + t.Errorf("splitSource(%q): got (%q, %d), want (%q, %d)", tc.in, f, n, tc.wantFile, tc.wantLine) + } + } +} diff --git a/report/text.go b/report/text.go new file mode 100644 index 00000000..364d68de --- /dev/null +++ b/report/text.go @@ -0,0 +1,181 @@ +package report + +import ( + "fmt" + "io" + "os" + "strings" + "time" +) + +// write swallows write errors on purpose: a downstream pipe that died +// should not crash the runner, and the exit status still reflects the test +// outcome. +func (r *TextReporter) write(format string, args ...any) { + _, _ = fmt.Fprintf(r.w, format, args...) +} + +// TextReporter is the default Reporter. It emits compact text suitable for +// both humans and agents: silent on success at default verbosity, detailed +// per-failure blocks, and one summary line per run. +// +// When GITHUB_ACTIONS is set in the environment, failures are additionally +// emitted as ::error::file=...,line=...:: annotations so reviewers see them +// inline on the PR diff. The annotation channel is a thin pass-through over +// the same failure events that drive the human-readable blocks — no +// duplicate accounting, no separate sink. +type TextReporter struct { + w io.Writer + v Verbosity + ghaEnabled bool + runStart time.Time + pass int + fail int + skip int + failBlocks []string // buffered "FAIL ..." blocks, flushed at run.end + knownErrAt map[string]struct{} +} + +func NewTextReporter(w io.Writer, v Verbosity) *TextReporter { + return &TextReporter{ + w: w, + v: v, + ghaEnabled: os.Getenv("GITHUB_ACTIONS") == "true", + knownErrAt: make(map[string]struct{}), + } +} + +func (r *TextReporter) Emit(e Event) { + switch e.Type { + case EventRunStart: + r.runStart = nonZeroOrNow(e.Ts) + case EventRunEnd: + r.flushRunEnd(e) + case EventCasePass: + r.pass++ + if r.v >= VerbosePasses { + r.write("PASS %s\n", e.Case) + } + case EventCaseSkip: + r.skip++ + if r.v >= VerbosePasses { + r.write("SKIP %s\n", e.Case) + } + case EventAssertFail: + r.recordFailure(e) + case EventCaseFail: + r.fail++ + case EventGCXExec: + if r.v >= VerboseCmd { + r.write(" $ %s\n", e.Cmd) + } + case EventFixtureStart, EventFixtureReady, EventFixtureTeardown: + if r.v >= VerboseAll { + r.write("[fixture %s] %s (%dms)\n", e.Fixture, e.Type, e.DurationMs) + } + } +} + +func (r *TextReporter) Close() error { return nil } + +func (r *TextReporter) recordFailure(e Event) { + var b strings.Builder + src := e.Source + if src == "" { + src = "(unknown source)" + } + fmt.Fprintf(&b, "FAIL %s %s\n", e.Case, src) + if e.Msg != "" { + fmt.Fprintf(&b, " %s\n", e.Msg) + } + if e.Cmd != "" { + fmt.Fprintf(&b, " $ %s\n", e.Cmd) + } + if e.StdoutExcerpt != "" { + fmt.Fprintf(&b, " stdout: %s\n", e.StdoutExcerpt) + } + r.failBlocks = append(r.failBlocks, b.String()) + + if r.ghaEnabled { + r.emitGHAAnnotation(e) + } +} + +func (r *TextReporter) emitGHAAnnotation(e Event) { + file, line := splitSource(e.Source) + // Suppress duplicate annotations for the same source position so a case + // with N substring failures does not flood the PR diff. + key := fmt.Sprintf("%s:%d", file, line) + if _, dup := r.knownErrAt[key]; dup { + return + } + r.knownErrAt[key] = struct{}{} + + msg := e.Msg + if msg == "" { + msg = "OATS assertion failed" + } + if line > 0 { + r.write("::error file=%s,line=%d::%s\n", file, line, msg) + } else { + r.write("::error file=%s::%s\n", file, msg) + } +} + +func (r *TextReporter) flushRunEnd(e Event) { + // Failure blocks first so the summary line is the final thing the reader + // sees — useful for both humans (scroll to bottom) and CI logs (tail). + for _, b := range r.failBlocks { + r.write("\n") + r.write("%s", b) + } + + total := r.pass + r.fail + r.skip + duration := durationOr(e, time.Since(r.runStart)) + switch { + case r.fail == 0 && r.skip == 0: + r.write("\nPASS %d/%d in %s\n", r.pass, total, duration) + case r.fail == 0: + r.write("\nPASS %d/%d (%d skipped) in %s\n", r.pass, total, r.skip, duration) + default: + r.write("\nFAIL %d/%d (%d failed, %d skipped) in %s\n", + r.pass, total, r.fail, r.skip, duration) + } +} + +func nonZeroOrNow(t time.Time) time.Time { + if t.IsZero() { + return time.Now() + } + return t +} + +func durationOr(e Event, fallback time.Duration) time.Duration { + if e.DurationMs > 0 { + return time.Duration(e.DurationMs) * time.Millisecond + } + return fallback.Round(time.Millisecond) +} + +// splitSource splits "path/to/file.yaml:42" into (file, line). When no line +// number is present (or the suffix is non-numeric), line is zero and file is +// the whole input. The contract for callers is "best effort": annotations +// fall back to file-only when a line number cannot be derived. +func splitSource(src string) (string, int) { + if src == "" { + return "", 0 + } + idx := strings.LastIndex(src, ":") + if idx < 0 { + return src, 0 + } + suffix := src[idx+1:] + n := 0 + for _, c := range suffix { + if c < '0' || c > '9' { + return src, 0 + } + n = n*10 + int(c-'0') + } + return src[:idx], n +} From 15eadf00b9988cf845fa0457e2c8e05a54982b03 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:40:27 +0000 Subject: [PATCH 006/124] chore: gofmt alignment on report and v2case structs Pre-push hook (flint) auto-fixed struct field alignment in report/text.go and v2case/case.go. Folding the format fix into a dedicated commit rather than amending the feature commits keeps the feature history free of unrelated whitespace churn. Signed-off-by: Gregor Zeitlinger --- report/text.go | 18 +++++++++--------- v2case/case.go | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/report/text.go b/report/text.go index 364d68de..3d253d27 100644 --- a/report/text.go +++ b/report/text.go @@ -25,15 +25,15 @@ func (r *TextReporter) write(format string, args ...any) { // the same failure events that drive the human-readable blocks — no // duplicate accounting, no separate sink. type TextReporter struct { - w io.Writer - v Verbosity - ghaEnabled bool - runStart time.Time - pass int - fail int - skip int - failBlocks []string // buffered "FAIL ..." blocks, flushed at run.end - knownErrAt map[string]struct{} + w io.Writer + v Verbosity + ghaEnabled bool + runStart time.Time + pass int + fail int + skip int + failBlocks []string // buffered "FAIL ..." blocks, flushed at run.end + knownErrAt map[string]struct{} } func NewTextReporter(w io.Writer, v Verbosity) *TextReporter { diff --git a/v2case/case.go b/v2case/case.go index f2772f33..8ed3948b 100644 --- a/v2case/case.go +++ b/v2case/case.go @@ -97,23 +97,23 @@ type AssertionCommon struct { } type TraceAssertion struct { - TraceQL string `yaml:"traceql"` + TraceQL string `yaml:"traceql"` AssertionCommon `yaml:",inline"` } type MetricAssertion struct { - PromQL string `yaml:"promql"` - Value string `yaml:"value,omitempty"` // ">= 0", "== 42", ... + PromQL string `yaml:"promql"` + Value string `yaml:"value,omitempty"` // ">= 0", "== 42", ... AssertionCommon `yaml:",inline"` } type LogAssertion struct { - LogQL string `yaml:"logql"` + LogQL string `yaml:"logql"` AssertionCommon `yaml:",inline"` } type ProfileAssertion struct { - Query string `yaml:"query"` + Query string `yaml:"query"` AssertionCommon `yaml:",inline"` } From 1ef29c1c3de39aa7b6b7348ddf95e62286ad325a Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:41:36 +0000 Subject: [PATCH 007/124] feat(wait): generic polling primitive (replaces gomega.Eventually) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit wait.Until and wait.While replace gomega.Eventually and gomega.Consistently in the OATS v2 runner. Same semantics, but failures are values returned from the asserter — no panics, no DSL. Until runs the asserter at least once, returns OK as soon as an iteration reports no failures, and gives up at the deadline. While runs for the entire window, requiring zero failures every iteration; the absence path uses it ("data we don't want must stay absent for at least N seconds"). Both honour context cancellation between polls. Generic over the failure type so callers can plug in assert.Failure (the typical case) or any other slice-shaped result without an intermediate wrapping. Defaults — 30s timeout, 500ms interval — mirror the gomega defaults v1 was already using, so case yamls do not need to be re-timed when they move to v2. Signed-off-by: Gregor Zeitlinger --- wait/wait.go | 118 ++++++++++++++++++++++++++++++++++++++++++++++ wait/wait_test.go | 106 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 wait/wait.go create mode 100644 wait/wait_test.go diff --git a/wait/wait.go b/wait/wait.go new file mode 100644 index 00000000..6a1e2065 --- /dev/null +++ b/wait/wait.go @@ -0,0 +1,118 @@ +// Package wait holds OATS v2's polling primitive: "keep trying until this +// passes, or give up at the deadline." +// +// In v1 this was gomega.Eventually. v2 drops gomega, so wait provides the +// same shape (asserter callback + timeout + polling interval) without the +// DSL. Failures are values, not panics; the runner decides how to render +// them via the report package. +// +// Two modes: +// +// Until — succeed once any iteration's assertion has no failures +// While — succeed only if every iteration's assertion has no failures +// for the entire window (used for absence checks) +package wait + +import ( + "context" + "time" +) + +// Defaults applied when the caller passes zero values. +const ( + DefaultTimeout = 30 * time.Second + DefaultInterval = 500 * time.Millisecond +) + +// Asserter is invoked on each poll. It returns the failures it observed — +// an empty slice (or nil) means "the expectation held on this iteration." +type Asserter[F any] func() []F + +// Options controls polling cadence. Timeout caps the total wall-clock spent +// retrying. Interval is the gap between polls. Zero values get sensible +// defaults (see DefaultTimeout / DefaultInterval). +type Options struct { + Timeout time.Duration + Interval time.Duration +} + +// Result is what Until and While return. Iterations counts how many poll +// attempts ran; LastFailures is the most recent failure set observed (empty +// on Until success, possibly populated on While success when the asserter +// reports transient failures the caller chose to ignore). +type Result[F any] struct { + OK bool + Iterations int + Elapsed time.Duration + LastFailures []F +} + +// Until polls the asserter until it returns no failures (success) or the +// deadline elapses (failure). It runs the asserter at least once even when +// the deadline has already passed. +func Until[F any](ctx context.Context, opts Options, asserter Asserter[F]) Result[F] { + opts = withDefaults(opts) + start := time.Now() + deadline := start.Add(opts.Timeout) + + var last []F + iter := 0 + for { + iter++ + last = asserter() + if len(last) == 0 { + return Result[F]{OK: true, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} + } + if time.Now().After(deadline) { + return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: last} + } + if ctx.Err() != nil { + return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: last} + } + select { + case <-time.After(opts.Interval): + case <-ctx.Done(): + return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: last} + } + } +} + +// While polls the asserter for the entire window, requiring no failures on +// every iteration. Used for absence checks ("data we DON'T want must stay +// absent for at least N seconds"). Reports the first failing iteration's +// failures, if any. +func While[F any](ctx context.Context, opts Options, asserter Asserter[F]) Result[F] { + opts = withDefaults(opts) + start := time.Now() + deadline := start.Add(opts.Timeout) + + iter := 0 + for { + iter++ + fails := asserter() + if len(fails) > 0 { + return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: fails} + } + if !time.Now().Before(deadline) { + return Result[F]{OK: true, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} + } + if ctx.Err() != nil { + return Result[F]{OK: true, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} + } + select { + case <-time.After(opts.Interval): + case <-ctx.Done(): + return Result[F]{OK: true, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} + } + } +} + +func withDefaults(o Options) Options { + if o.Timeout <= 0 { + o.Timeout = DefaultTimeout + } + if o.Interval <= 0 { + o.Interval = DefaultInterval + } + return o +} diff --git a/wait/wait_test.go b/wait/wait_test.go new file mode 100644 index 00000000..385c1605 --- /dev/null +++ b/wait/wait_test.go @@ -0,0 +1,106 @@ +package wait + +import ( + "context" + "sync/atomic" + "testing" + "time" +) + +func TestUntil_SucceedsFirstTry(t *testing.T) { + r := Until[string](context.Background(), Options{Timeout: time.Second, Interval: 10 * time.Millisecond}, func() []string { + return nil + }) + if !r.OK { + t.Errorf("expected OK, got %+v", r) + } + if r.Iterations != 1 { + t.Errorf("Iterations: got %d, want 1", r.Iterations) + } +} + +func TestUntil_SucceedsAfterSeveralTries(t *testing.T) { + var n int32 + r := Until[string](context.Background(), Options{Timeout: time.Second, Interval: 5 * time.Millisecond}, func() []string { + if atomic.AddInt32(&n, 1) < 3 { + return []string{"not yet"} + } + return nil + }) + if !r.OK { + t.Fatalf("expected OK after retries, got %+v", r) + } + if r.Iterations != 3 { + t.Errorf("Iterations: got %d, want 3", r.Iterations) + } +} + +func TestUntil_FailsAtDeadline(t *testing.T) { + r := Until[string](context.Background(), Options{Timeout: 30 * time.Millisecond, Interval: 5 * time.Millisecond}, func() []string { + return []string{"never passes"} + }) + if r.OK { + t.Errorf("expected !OK, got %+v", r) + } + if len(r.LastFailures) == 0 { + t.Errorf("LastFailures should carry the last asserter output") + } +} + +func TestUntil_RunsAtLeastOnceEvenWithZeroTimeout(t *testing.T) { + // Zero timeout gets the default — we just want to verify the asserter + // is invoked, not that no poll happens. The contract is "at least once." + called := false + r := Until[string](context.Background(), Options{Timeout: 5 * time.Millisecond}, func() []string { + called = true + return []string{"x"} + }) + if !called { + t.Error("asserter must be invoked at least once even under tight deadline") + } + if r.OK { + t.Errorf("expected !OK, got %+v", r) + } +} + +func TestWhile_HoldsForEntireWindow(t *testing.T) { + r := While[string](context.Background(), Options{Timeout: 30 * time.Millisecond, Interval: 5 * time.Millisecond}, func() []string { + return nil // never fails + }) + if !r.OK { + t.Errorf("expected OK, got %+v", r) + } + if r.Iterations < 2 { + t.Errorf("expected multiple polls in 30ms with 5ms interval, got %d", r.Iterations) + } +} + +func TestWhile_FailsOnFirstFailure(t *testing.T) { + var n int32 + r := While[string](context.Background(), Options{Timeout: 200 * time.Millisecond, Interval: 5 * time.Millisecond}, func() []string { + if atomic.AddInt32(&n, 1) > 3 { + return []string{"data appeared mid-window"} + } + return nil + }) + if r.OK { + t.Fatalf("expected !OK on mid-window failure, got %+v", r) + } + if len(r.LastFailures) == 0 { + t.Errorf("LastFailures should describe the breach") + } +} + +func TestUntil_ContextCancelStops(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + r := Until[string](ctx, Options{Timeout: time.Second, Interval: 5 * time.Millisecond}, func() []string { + return []string{"x"} + }) + if r.OK { + t.Errorf("cancelled context should not pass: %+v", r) + } + if r.Iterations == 0 { + t.Errorf("asserter should still run once before honoring cancel") + } +} From 4df2734c1df022b06bd47daafdac2314bf6cdf56 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:43:11 +0000 Subject: [PATCH 008/124] feat(discovery): oats.toml root config and run planner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery is the layer that turns an oats.toml plus a collection of case yamls into a concrete run plan. v1's "walk for any yaml carrying oats-schema-version" model becomes "declare the plan up front." oats.toml shape (TOML, matching flint precedent): [meta] version = 2 [[suite]] name = "lgtm" cases = ["examples/*/oats.yaml"] fixture = "lgtm-shared" tags = ["traces", "metrics", "logs"] [fixture.lgtm-shared] type = "compose" template = "lgtm" Three fixture types are supported day one: compose, k3d, remote. kind and k3s-host are deferred to a follow-up when a concrete consumer needs them. remote (point at an already-running stack) is in scope because it unlocks "run my suite against staging" with effectively zero added code. PlanRun expands globs relative to oats.toml's directory and applies a Filter (by suite name or tag). Cases within a suite sort by source path for stable ordering across runs. An empty glob is an error — silent "matched nothing" was the surprising-behaviour problem with the v1 walk. Summary renders one line per suite for `oats list` without loading any case yamls — cheap inspection before deciding to run. KnownFields-style strictness comes from md.Undecoded(); unknown top-level keys in oats.toml surface as an error rather than a silent drop. Signed-off-by: Gregor Zeitlinger --- discovery/discovery.go | 230 ++++++++++++++++++++++++++++++++++++ discovery/discovery_test.go | 228 +++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + 4 files changed, 461 insertions(+) create mode 100644 discovery/discovery.go create mode 100644 discovery/discovery_test.go diff --git a/discovery/discovery.go b/discovery/discovery.go new file mode 100644 index 00000000..1f0c440b --- /dev/null +++ b/discovery/discovery.go @@ -0,0 +1,230 @@ +// Package discovery turns an oats.toml + a collection of case yamls into a +// concrete run plan. +// +// In OATS v1, the runner walked the file system for any yaml carrying +// "oats-schema-version" and ran whatever it found. v2 declares the plan up +// front: oats.toml lists suites, each suite lists cases (path globs) and the +// fixture they share. "oats list" prints the plan before "oats run" executes +// it. +package discovery + +import ( + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/BurntSushi/toml" + "github.com/grafana/oats/v2case" +) + +// RootConfig is the parsed oats.toml file. Field names mirror the TOML keys +// directly so a misnamed key surfaces as a "field not defined" error from +// the toml decoder rather than a silent miss. +type RootConfig struct { + Meta Meta `toml:"meta"` + Suites []SuiteConfig `toml:"suite"` + Fixture map[string]FixtureConfig `toml:"fixture"` + Cache CacheConfig `toml:"cache,omitempty"` + + // SourceDir is the directory of the loaded oats.toml. Case glob + // expressions resolve relative to it. + SourceDir string `toml:"-"` +} + +type Meta struct { + Version int `toml:"version"` +} + +type SuiteConfig struct { + Name string `toml:"name"` + Cases []string `toml:"cases"` // path globs, relative to oats.toml dir + Fixture string `toml:"fixture,omitempty"` + Tags []string `toml:"tags,omitempty"` +} + +type FixtureConfig struct { + Type string `toml:"type"` // "compose" | "k3d" | "remote" + Template string `toml:"template,omitempty"` + ComposeFile string `toml:"compose_file,omitempty"` + PoolSize int `toml:"pool_size,omitempty"` + Endpoint string `toml:"endpoint,omitempty"` // remote only +} + +type CacheConfig struct { + TTLDays int `toml:"ttl_days,omitempty"` // zero → use runtime default +} + +// SupportedVersion is the value of [meta].version that this binary parses. +const SupportedVersion = 2 + +// Load reads an oats.toml from disk. +func Load(path string) (*RootConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("discovery load %s: %w", path, err) + } + var cfg RootConfig + md, err := toml.Decode(string(data), &cfg) + if err != nil { + return nil, fmt.Errorf("discovery parse %s: %w", path, err) + } + if undecoded := md.Undecoded(); len(undecoded) > 0 { + return nil, fmt.Errorf("discovery: %s contains unknown keys: %v", path, undecoded) + } + cfg.SourceDir = filepath.Dir(path) + if err := cfg.Validate(); err != nil { + return nil, err + } + return &cfg, nil +} + +// Validate checks structural rules a toml parser cannot. Called by Load; +// exposed for tests that construct RootConfigs in memory. +func (c *RootConfig) Validate() error { + if c.Meta.Version != SupportedVersion { + return fmt.Errorf("meta.version: expected %d, got %d", SupportedVersion, c.Meta.Version) + } + if len(c.Suites) == 0 { + return fmt.Errorf("at least one [[suite]] required") + } + for i, s := range c.Suites { + if s.Name == "" { + return fmt.Errorf("suite[%d].name: required", i) + } + if len(s.Cases) == 0 { + return fmt.Errorf("suite[%d] (%q): cases is required and non-empty", i, s.Name) + } + if s.Fixture != "" { + if _, ok := c.Fixture[s.Fixture]; !ok { + return fmt.Errorf("suite[%d] (%q): fixture %q not defined", i, s.Name, s.Fixture) + } + } + } + for name, f := range c.Fixture { + switch f.Type { + case "compose": + if f.Template == "" && f.ComposeFile == "" { + return fmt.Errorf("fixture %q: type=compose requires template or compose_file", name) + } + case "k3d": + // PoolSize=0 means "single ephemeral cluster" — valid. + case "remote": + if f.Endpoint == "" { + return fmt.Errorf("fixture %q: type=remote requires endpoint", name) + } + case "": + return fmt.Errorf("fixture %q: type is required (compose | k3d | remote)", name) + default: + return fmt.Errorf("fixture %q: unknown type %q", name, f.Type) + } + } + return nil +} + +// Filter describes which suites and cases to include in a Plan. +// Empty fields impose no restriction. Tag filtering uses any-match semantics: +// a suite passes if any of its tags appears in the filter list. +type Filter struct { + Suites []string // exact suite names; empty = all suites + Tags []string // any-match +} + +// Plan is one suite plus the cases it expanded to. +type Plan struct { + Suite SuiteConfig + Fixture FixtureConfig + Cases []*v2case.Case +} + +// PlanRun expands globs and applies the filter against the loaded config. +// Returns plans in oats.toml order; cases within a plan are sorted by +// SourcePath for stable test ordering. +func (c *RootConfig) PlanRun(f Filter) ([]Plan, error) { + wantSuite := func(name string) bool { + if len(f.Suites) == 0 { + return true + } + for _, s := range f.Suites { + if s == name { + return true + } + } + return false + } + wantTag := func(tags []string) bool { + if len(f.Tags) == 0 { + return true + } + for _, want := range f.Tags { + for _, has := range tags { + if want == has { + return true + } + } + } + return false + } + + var plans []Plan + for _, suite := range c.Suites { + if !wantSuite(suite.Name) { + continue + } + if !wantTag(suite.Tags) { + continue + } + + cases, err := c.loadSuiteCases(suite) + if err != nil { + return nil, fmt.Errorf("suite %q: %w", suite.Name, err) + } + + plans = append(plans, Plan{ + Suite: suite, + Fixture: c.Fixture[suite.Fixture], + Cases: cases, + }) + } + return plans, nil +} + +func (c *RootConfig) loadSuiteCases(suite SuiteConfig) ([]*v2case.Case, error) { + seen := make(map[string]struct{}) // dedupe overlapping globs + var cases []*v2case.Case + for _, pattern := range suite.Cases { + abs := filepath.Join(c.SourceDir, pattern) + matches, err := filepath.Glob(abs) + if err != nil { + return nil, fmt.Errorf("glob %q: %w", pattern, err) + } + if len(matches) == 0 { + return nil, fmt.Errorf("glob %q matched zero files", pattern) + } + for _, m := range matches { + if _, dup := seen[m]; dup { + continue + } + seen[m] = struct{}{} + tc, loadErr := v2case.Load(m) + if loadErr != nil { + return nil, loadErr + } + cases = append(cases, tc) + } + } + sort.Slice(cases, func(i, j int) bool { + return cases[i].SourcePath < cases[j].SourcePath + }) + return cases, nil +} + +// Summary renders a single line per plan suitable for `oats list`. It does +// not load any cases — useful for a dry-run before deciding to expand globs. +func (c *RootConfig) Summary() string { + var out string + for _, s := range c.Suites { + out += fmt.Sprintf("suite=%s fixture=%s tags=%v cases=%v\n", s.Name, s.Fixture, s.Tags, s.Cases) + } + return out +} diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go new file mode 100644 index 00000000..1411a23e --- /dev/null +++ b/discovery/discovery_test.go @@ -0,0 +1,228 @@ +package discovery + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func writeFile(t *testing.T, dir, rel, body string) { + t.Helper() + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +const validCaseYAML = `oats: 2 +name: %s +seed: + type: app + compose: docker-compose.app.yml +expected: + traces: + - traceql: '{}' + contains: ["x"] +` + +func TestLoad_ValidConfig(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "lgtm" +cases = ["cases/*.yaml"] +fixture = "lgtm-shared" +tags = ["traces"] + +[fixture.lgtm-shared] +type = "compose" +template = "lgtm" +`) + writeFile(t, dir, "cases/a.yaml", strings.Replace(validCaseYAML, "%s", "case-a", 1)) + writeFile(t, dir, "cases/b.yaml", strings.Replace(validCaseYAML, "%s", "case-b", 1)) + + cfg, err := Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + + plans, err := cfg.PlanRun(Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 { + t.Fatalf("plans: got %d, want 1", len(plans)) + } + if len(plans[0].Cases) != 2 { + t.Errorf("cases: got %d, want 2", len(plans[0].Cases)) + } + if plans[0].Cases[0].Name != "case-a" { + t.Errorf("sort order: got %q first", plans[0].Cases[0].Name) + } + if plans[0].Fixture.Type != "compose" { + t.Errorf("fixture resolved wrong: %+v", plans[0].Fixture) + } +} + +func TestLoad_RejectsUnknownKey(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 +typooo = "boom" +[[suite]] +name = "x" +cases = ["a.yaml"] +`) + _, err := Load(filepath.Join(dir, "oats.toml")) + if err == nil || !strings.Contains(err.Error(), "unknown keys") { + t.Errorf("expected unknown-keys error, got %v", err) + } +} + +func TestPlanRun_FilterByTag(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "alpha" +cases = ["a.yaml"] +tags = ["traces"] + +[[suite]] +name = "beta" +cases = ["b.yaml"] +tags = ["logs"] +`) + writeFile(t, dir, "a.yaml", strings.Replace(validCaseYAML, "%s", "a", 1)) + writeFile(t, dir, "b.yaml", strings.Replace(validCaseYAML, "%s", "b", 1)) + + cfg, err := Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(Filter{Tags: []string{"traces"}}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || plans[0].Suite.Name != "alpha" { + t.Errorf("tag filter: %+v", planNames(plans)) + } +} + +func TestPlanRun_FilterBySuiteName(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "alpha" +cases = ["a.yaml"] + +[[suite]] +name = "beta" +cases = ["b.yaml"] +`) + writeFile(t, dir, "a.yaml", strings.Replace(validCaseYAML, "%s", "a", 1)) + writeFile(t, dir, "b.yaml", strings.Replace(validCaseYAML, "%s", "b", 1)) + + cfg, err := Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(Filter{Suites: []string{"beta"}}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || plans[0].Suite.Name != "beta" { + t.Errorf("suite filter: %+v", planNames(plans)) + } +} + +func TestValidate_BadFixtureType(t *testing.T) { + cfg := &RootConfig{ + Meta: Meta{Version: 2}, + Suites: []SuiteConfig{{ + Name: "s", Cases: []string{"a.yaml"}, Fixture: "x", + }}, + Fixture: map[string]FixtureConfig{"x": {Type: "weird"}}, + } + err := cfg.Validate() + if err == nil || !strings.Contains(err.Error(), `unknown type "weird"`) { + t.Errorf("expected unknown-type error, got %v", err) + } +} + +func TestValidate_FixtureRefNotDefined(t *testing.T) { + cfg := &RootConfig{ + Meta: Meta{Version: 2}, + Suites: []SuiteConfig{{ + Name: "s", Cases: []string{"a.yaml"}, Fixture: "missing", + }}, + } + err := cfg.Validate() + if err == nil || !strings.Contains(err.Error(), `fixture "missing" not defined`) { + t.Errorf("expected fixture-missing error, got %v", err) + } +} + +func TestValidate_RemoteRequiresEndpoint(t *testing.T) { + cfg := &RootConfig{ + Meta: Meta{Version: 2}, + Suites: []SuiteConfig{{Name: "s", Cases: []string{"a.yaml"}}}, + Fixture: map[string]FixtureConfig{"r": {Type: "remote"}}, + } + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "endpoint") { + t.Errorf("expected endpoint error, got %v", err) + } +} + +func TestPlanRun_EmptyGlobIsAnError(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "s" +cases = ["nope/*.yaml"] +`) + cfg, err := Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + _, err = cfg.PlanRun(Filter{}) + if err == nil || !strings.Contains(err.Error(), "matched zero files") { + t.Errorf("expected zero-match error, got %v", err) + } +} + +func TestSummary(t *testing.T) { + cfg := &RootConfig{ + Suites: []SuiteConfig{ + {Name: "alpha", Fixture: "x", Tags: []string{"traces"}, Cases: []string{"a.yaml"}}, + }, + } + s := cfg.Summary() + if !strings.Contains(s, "suite=alpha") { + t.Errorf("Summary missing suite line: %q", s) + } +} + +func planNames(p []Plan) []string { + out := make([]string, len(p)) + for i := range p { + out[i] = p[i].Suite.Name + } + return out +} diff --git a/go.mod b/go.mod index 629ee4ca..3da24c84 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/grafana/oats go 1.25.0 require ( + github.com/BurntSushi/toml v1.6.0 github.com/onsi/gomega v1.42.1 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/collector/pdata v1.61.0 diff --git a/go.sum b/go.sum index ab37d6b0..9e26a478 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= From cac8145f6fc14625ef9a1e54120ab40ff442d2bd Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:44:17 +0000 Subject: [PATCH 009/124] feat(signalcmd): translate v2case assertions into gcx args signalcmd is a one-function-per-signal layer that turns a v2case TraceAssertion / LogAssertion / MetricAssertion / ProfileAssertion into the []string suitable for engine.GCX.Execute. Functions stay positional and minimal: no --context flag (engine prepends that), no datasource UID (resolved from gcx config), no extra output format unless the assertion needs it. The single exception is metrics with a numeric value: those request -o json so the runner can extract a typed value for the value-comparison check. Render is the inverse for the report layer: it formats an args slice as a shell-paste-safe command line, single-quoting only what needs it and escaping embedded single quotes. The output appears in FAIL blocks and at -vv verbosity so a reviewer can copy and run the exact command gcx executed. The package depends on v2case for the assertion structs; tests assert the resulting arg vectors directly so regressions in flag ordering fail loudly. Signed-off-by: Gregor Zeitlinger --- signalcmd/signalcmd.go | 124 ++++++++++++++++++++++++++++++++++++ signalcmd/signalcmd_test.go | 103 ++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 signalcmd/signalcmd.go create mode 100644 signalcmd/signalcmd_test.go diff --git a/signalcmd/signalcmd.go b/signalcmd/signalcmd.go new file mode 100644 index 00000000..35b7150a --- /dev/null +++ b/signalcmd/signalcmd.go @@ -0,0 +1,124 @@ +// Package signalcmd translates a single v2case assertion into the gcx +// command line that will execute the corresponding query. +// +// One function per signal type. Each returns []string suitable for handing +// to engine.GCX.Execute: positional args only, no --context (engine prepends +// that), no datasource flag (resolved from the gcx config context). +// +// The output format is fixed to gcx's default "agents" text mode for +// substring assertions; JSON (-o json) for assertions that need a +// structured value extracted (metrics' `value` key). +package signalcmd + +import ( + "time" + + "github.com/grafana/oats/v2case" +) + +// Defaults applied when a case does not pin its own values. +const ( + DefaultSince = 10 * time.Minute +) + +// Traces builds the gcx args for a TraceAssertion. +func Traces(a v2case.TraceAssertion, since time.Duration) []string { + if since <= 0 { + since = DefaultSince + } + return []string{ + "traces", "search", + "--since", since.String(), + a.TraceQL, + } +} + +// Logs builds the gcx args for a LogAssertion. +func Logs(a v2case.LogAssertion, since time.Duration) []string { + if since <= 0 { + since = DefaultSince + } + return []string{ + "logs", "query", + "--since", since.String(), + a.LogQL, + } +} + +// Metrics builds the gcx args for a MetricAssertion. When the assertion +// declares a numeric `value` comparison we ask gcx for JSON so the runner +// can parse the actual value out; otherwise the default agent text format +// is enough for substring matching. +func Metrics(a v2case.MetricAssertion, since time.Duration) []string { + if since <= 0 { + since = DefaultSince + } + args := []string{ + "metrics", "query", + "--since", since.String(), + } + if a.Value != "" { + args = append(args, "-o", "json") + } + args = append(args, a.PromQL) + return args +} + +// Profiles builds the gcx args for a ProfileAssertion. +func Profiles(a v2case.ProfileAssertion, since time.Duration) []string { + if since <= 0 { + since = DefaultSince + } + return []string{ + "profiles", "query", + "--since", since.String(), + a.Query, + } +} + +// Render is a convenience used by the report layer to show the gcx +// invocation in a FAIL block. It mirrors what a shell would print. +func Render(args []string) string { + out := "gcx" + for _, a := range args { + out += " " + shellQuote(a) + } + return out +} + +// shellQuote is intentionally tiny: TraceQL / PromQL / LogQL strings often +// contain spaces and quotes that need to survive a copy-paste from a FAIL +// block. We single-quote anything that's not "boring." +func shellQuote(s string) string { + if s == "" { + return "''" + } + safe := true + for _, c := range s { + switch { + case c >= 'a' && c <= 'z': + case c >= 'A' && c <= 'Z': + case c >= '0' && c <= '9': + case c == '-' || c == '_' || c == '.' || c == '/' || c == ':' || c == '=': + default: + safe = false + } + if !safe { + break + } + } + if safe { + return s + } + // Single-quote and escape any embedded single quotes. + quoted := "'" + for _, c := range s { + if c == '\'' { + quoted += `'\''` + } else { + quoted += string(c) + } + } + quoted += "'" + return quoted +} diff --git a/signalcmd/signalcmd_test.go b/signalcmd/signalcmd_test.go new file mode 100644 index 00000000..d82e6fca --- /dev/null +++ b/signalcmd/signalcmd_test.go @@ -0,0 +1,103 @@ +package signalcmd + +import ( + "strings" + "testing" + "time" + + "github.com/grafana/oats/v2case" +) + +func TestTraces(t *testing.T) { + got := Traces(v2case.TraceAssertion{TraceQL: `{ span.http.route = "/x" }`}, 0) + want := []string{"traces", "search", "--since", "10m0s", `{ span.http.route = "/x" }`} + if !equal(got, want) { + t.Errorf("got %v\nwant %v", got, want) + } +} + +func TestLogs(t *testing.T) { + got := Logs(v2case.LogAssertion{LogQL: `{service_name="x"}`}, 5*time.Minute) + want := []string{"logs", "query", "--since", "5m0s", `{service_name="x"}`} + if !equal(got, want) { + t.Errorf("got %v\nwant %v", got, want) + } +} + +func TestMetrics_PromQLOnly(t *testing.T) { + got := Metrics(v2case.MetricAssertion{PromQL: "up"}, time.Minute) + want := []string{"metrics", "query", "--since", "1m0s", "up"} + if !equal(got, want) { + t.Errorf("got %v\nwant %v", got, want) + } +} + +func TestMetrics_WithValueAsksForJSON(t *testing.T) { + got := Metrics(v2case.MetricAssertion{PromQL: "rate(x[1m])", Value: ">= 0"}, time.Minute) + // JSON output flag must appear before the positional PromQL. + if !contains(got, "-o", "json") { + t.Errorf("expected -o json in: %v", got) + } + if got[len(got)-1] != "rate(x[1m])" { + t.Errorf("PromQL should be last positional: %v", got) + } +} + +func TestProfiles(t *testing.T) { + got := Profiles(v2case.ProfileAssertion{Query: "process_cpu:cpu:nanoseconds:cpu:nanoseconds{}"}, 0) + if got[0] != "profiles" || got[1] != "query" { + t.Errorf("wrong verb chain: %v", got) + } +} + +func TestRender_QuotesSpecialChars(t *testing.T) { + args := []string{"traces", "search", `{ span.http.route = "/x" }`} + rendered := Render(args) + if !strings.HasPrefix(rendered, "gcx traces search '{") { + t.Errorf("complex arg should be single-quoted: %s", rendered) + } +} + +func TestRender_LeavesBoringArgsBare(t *testing.T) { + args := []string{"metrics", "query", "--since", "10m"} + rendered := Render(args) + if rendered != "gcx metrics query --since 10m" { + t.Errorf("boring args should not be quoted: %s", rendered) + } +} + +func TestRender_EscapesSingleQuote(t *testing.T) { + rendered := Render([]string{"echo", "it's"}) + // Embedded single quote escaped as '\'' + if !strings.Contains(rendered, `'\''`) { + t.Errorf("missing quote escape: %s", rendered) + } +} + +func equal(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func contains(haystack []string, needles ...string) bool { + for i := 0; i+len(needles) <= len(haystack); i++ { + match := true + for j, n := range needles { + if haystack[i+j] != n { + match = false + break + } + } + if match { + return true + } + } + return false +} From e55590a8a1b53cb5de3168290cf2125836ae2b89 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:44:37 +0000 Subject: [PATCH 010/124] chore: gofmt discovery struct alignment Pre-push hook (flint) auto-fixed field alignment. Signed-off-by: Gregor Zeitlinger --- discovery/discovery.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/discovery/discovery.go b/discovery/discovery.go index 1f0c440b..48c33052 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -22,10 +22,10 @@ import ( // directly so a misnamed key surfaces as a "field not defined" error from // the toml decoder rather than a silent miss. type RootConfig struct { - Meta Meta `toml:"meta"` - Suites []SuiteConfig `toml:"suite"` - Fixture map[string]FixtureConfig `toml:"fixture"` - Cache CacheConfig `toml:"cache,omitempty"` + Meta Meta `toml:"meta"` + Suites []SuiteConfig `toml:"suite"` + Fixture map[string]FixtureConfig `toml:"fixture"` + Cache CacheConfig `toml:"cache,omitempty"` // SourceDir is the directory of the loaded oats.toml. Case glob // expressions resolve relative to it. From 760e4649753700b87bd73260c1130b69415f8f4a Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:47:07 +0000 Subject: [PATCH 011/124] feat(runner): orchestrate seed + poll-and-assert per case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runner is the glue across engine, seed, signalcmd, assert, wait, and report. One Runner instance handles a suite of cases sharing a single Endpoint and Reporter — created once, reused. Flow per case: 1. Emit case.start. 2. Seed: inline-otlp POSTs to Endpoint.OTLPHTTP; app does nothing (the external fixture has already booted the application). 3. SeedSettleDelay (default 2s) — gives ingest pipelines a head start so the first poll has something to find. 4. For each Expected.{Traces,Logs,Metrics,Profiles} assertion: - Build gcx args via signalcmd. - wait.Until / wait.While loop the gcx invocation through the engine.Executor, applying assert.Contains / NotContains / Regex / Count / Value. - On failure, emit assert.fail events with the gcx command for the failure block. 5. Emit case.pass or case.fail. Failure in any signal block fails the case, but the runner continues through the other blocks so the report shows all problems at once. Metric `value: ">= N"` assertions ask gcx for JSON output and parse the first vector data point. extractMetricValue is intentionally tolerant of unknown fields so gcx schema evolution does not break us. approxRowCount is the temporary stand-in for "did anything come back?" used by count and absent. v2.1 can swap this for a structured-output path once gcx exposes a stable count flag. Tests are stubbed via a deterministic Executor that returns canned output and records the args every invocation received — no live gcx required, no fixture required. Signed-off-by: Gregor Zeitlinger --- runner/runner.go | 376 ++++++++++++++++++++++++++++++++++++++++++ runner/runner_test.go | 213 ++++++++++++++++++++++++ 2 files changed, 589 insertions(+) create mode 100644 runner/runner.go create mode 100644 runner/runner_test.go diff --git a/runner/runner.go b/runner/runner.go new file mode 100644 index 00000000..20535ce8 --- /dev/null +++ b/runner/runner.go @@ -0,0 +1,376 @@ +// Package runner orchestrates one v2 case end-to-end: seed → poll-and-assert +// → report. It is intentionally agnostic about fixtures — the caller hands +// in already-resolved endpoints, so this layer ships before the +// fixture-lifecycle layer exists. +// +// One Runner instance handles one suite of cases sharing a fixture. +package runner + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/grafana/oats/assert" + "github.com/grafana/oats/engine" + "github.com/grafana/oats/report" + "github.com/grafana/oats/seed" + "github.com/grafana/oats/signalcmd" + "github.com/grafana/oats/v2case" + "github.com/grafana/oats/wait" +) + +// Endpoint identifies a running stack the runner can drive. It is small on +// purpose: the v2 runner only needs to know where gcx points and where to +// seed OTLP. Everything else lives in the gcx config context. +type Endpoint struct { + // GCXContext is the value passed to gcx --context. Required. + GCXContext string + + // OTLPHTTP is the base URL for OTLP/HTTP seed POSTs ("http://localhost:4318"). + // Required when any case uses seed.type = inline-otlp. + OTLPHTTP string +} + +// Options configures the polling cadence and per-case deadline. Sensible +// zero-value defaults apply. +type Options struct { + // Timeout caps how long the runner waits for any one assertion to pass. + // Default 30s. + Timeout time.Duration + + // Interval is the gap between assertion polls. Default 500ms. + Interval time.Duration + + // AbsentTimeout is how long an absence assertion must hold. Default 10s. + AbsentTimeout time.Duration + + // SeedSettleDelay is how long to wait after seeding before the first + // assertion attempt. Helps when an upstream ingest pipeline has a known + // minimum buffer (e.g. Loki's ~5s). Default 2s. + SeedSettleDelay time.Duration +} + +func (o Options) withDefaults() Options { + if o.Timeout <= 0 { + o.Timeout = 30 * time.Second + } + if o.Interval <= 0 { + o.Interval = 500 * time.Millisecond + } + if o.AbsentTimeout <= 0 { + o.AbsentTimeout = 10 * time.Second + } + if o.SeedSettleDelay < 0 { + o.SeedSettleDelay = 2 * time.Second + } + if o.SeedSettleDelay == 0 { + o.SeedSettleDelay = 2 * time.Second + } + return o +} + +// Runner executes one or more cases against a single Endpoint, emitting +// lifecycle events to the configured Reporter. It is intended to be created +// once per suite — the gcx Executor and Reporter are reused across cases. +type Runner struct { + exec engine.Executor + reporter report.Reporter + endpoint Endpoint + opts Options + seeder *seed.Sender +} + +// New constructs a Runner. exec is typically a configured engine.GCX; +// rep is typically a report.TextReporter or NDJSONReporter; ep names the +// gcx context and (optionally) the OTLP endpoint for inline seeding. +func New(exec engine.Executor, rep report.Reporter, ep Endpoint, opts Options) *Runner { + return &Runner{ + exec: exec, + reporter: rep, + endpoint: ep, + opts: opts.withDefaults(), + seeder: &seed.Sender{OTLPEndpoint: ep.OTLPHTTP}, + } +} + +// RunCase runs one case. Returns true on pass, false on fail. Events are +// emitted via the Reporter; errors that prevent the case from running at +// all (e.g. inline-otlp seed without an OTLP endpoint) are also surfaced +// as case.fail events with an explanatory msg. +func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { + caseStart := time.Now() + r.reporter.Emit(report.Event{ + Type: report.EventCaseStart, + Case: c.Name, + Source: c.SourcePath, + Ts: caseStart, + }) + + // Seed. + if err := r.seedCase(c); err != nil { + r.failCase(c, "seed: "+err.Error(), "") + r.reporter.Emit(report.Event{ + Type: report.EventCaseFail, + Case: c.Name, + DurationMs: time.Since(caseStart).Milliseconds(), + }) + return false + } + + if r.opts.SeedSettleDelay > 0 { + select { + case <-time.After(r.opts.SeedSettleDelay): + case <-ctx.Done(): + r.failCase(c, "context cancelled during seed-settle window", "") + r.reporter.Emit(report.Event{Type: report.EventCaseFail, Case: c.Name}) + return false + } + } + + // Assertions, signal by signal. A failure in any signal block fails the + // case but we still run the others — the report shows all problems. + ok := true + for i := range c.Expected.Traces { + if !r.runTrace(ctx, c, &c.Expected.Traces[i]) { + ok = false + } + } + for i := range c.Expected.Logs { + if !r.runLog(ctx, c, &c.Expected.Logs[i]) { + ok = false + } + } + for i := range c.Expected.Metrics { + if !r.runMetric(ctx, c, &c.Expected.Metrics[i]) { + ok = false + } + } + for i := range c.Expected.Profiles { + if !r.runProfile(ctx, c, &c.Expected.Profiles[i]) { + ok = false + } + } + + durMs := time.Since(caseStart).Milliseconds() + if ok { + r.reporter.Emit(report.Event{Type: report.EventCasePass, Case: c.Name, DurationMs: durMs}) + } else { + r.reporter.Emit(report.Event{Type: report.EventCaseFail, Case: c.Name, DurationMs: durMs}) + } + return ok +} + +func (r *Runner) seedCase(c *v2case.Case) error { + switch c.Seed.Type { + case "app": + // External fixture is responsible for booting the app. Runner + // assumes the app is already emitting; nothing to do here. + return nil + case "inline-otlp": + if r.seeder.OTLPEndpoint == "" { + return fmt.Errorf("inline-otlp seed requires Endpoint.OTLPHTTP") + } + return r.seeder.Send(toSeedPayload(c.Seed)) + } + return fmt.Errorf("unknown seed type %q", c.Seed.Type) +} + +func toSeedPayload(s v2case.Seed) seed.Payload { + p := seed.Payload{} + for _, t := range s.Traces { + for _, sp := range t.Spans { + dur, _ := time.ParseDuration(sp.Duration) + p.Traces = append(p.Traces, seed.Trace{ + Service: t.Service, + Span: seed.SpanFields{ + Name: sp.Name, + Kind: sp.Kind, + Duration: dur, + }, + }) + } + } + for _, l := range s.Logs { + p.Logs = append(p.Logs, seed.Log{ + Service: l.Service, + Body: l.Body, + SeverityNumber: l.SeverityNumber, + SeverityText: l.SeverityText, + }) + } + for _, m := range s.Metrics { + p.Metrics = append(p.Metrics, seed.Metric{ + Service: m.Service, + Name: m.Name, + Value: m.Value, + }) + } + return p +} + +// pollAssert handles the polling loop common to all signal types. The +// runner builds the gcx args and an assertEval closure; pollAssert runs +// wait.Until / wait.While accordingly. +func (r *Runner) pollAssert( + ctx context.Context, + c *v2case.Case, + args []string, + absent bool, + evalFn func(stdout, stderr string, exit int) []assert.Failure, +) bool { + cmdStr := signalcmd.Render(args) + + run := func() []assert.Failure { + res, err := r.exec.Execute(ctx, args...) + if err != nil { + return []assert.Failure{{Rule: "exec", Detail: err.Error()}} + } + r.reporter.Emit(report.Event{ + Type: report.EventGCXExec, + Case: c.Name, + Cmd: cmdStr, + }) + return evalFn(res.Stdout, res.Stderr, res.ExitCode) + } + + opts := wait.Options{Timeout: r.opts.Timeout, Interval: r.opts.Interval} + var result wait.Result[assert.Failure] + if absent { + opts.Timeout = r.opts.AbsentTimeout + result = wait.While[assert.Failure](ctx, opts, run) + } else { + result = wait.Until[assert.Failure](ctx, opts, run) + } + + if result.OK { + return true + } + for _, f := range result.LastFailures { + r.reporter.Emit(report.Event{ + Type: report.EventAssertFail, + Case: c.Name, + Source: c.SourcePath, + Msg: f.Error(), + Cmd: cmdStr, + }) + } + return false +} + +func (r *Runner) runTrace(ctx context.Context, c *v2case.Case, a *v2case.TraceAssertion) bool { + args := signalcmd.Traces(*a, r.opts.Timeout) + return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { + return evalCommon(stdout, a.AssertionCommon) + }) +} + +func (r *Runner) runLog(ctx context.Context, c *v2case.Case, a *v2case.LogAssertion) bool { + args := signalcmd.Logs(*a, r.opts.Timeout) + return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { + return evalCommon(stdout, a.AssertionCommon) + }) +} + +func (r *Runner) runMetric(ctx context.Context, c *v2case.Case, a *v2case.MetricAssertion) bool { + args := signalcmd.Metrics(*a, r.opts.Timeout) + return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { + fails := evalCommon(stdout, a.AssertionCommon) + if a.Value != "" { + actual, err := extractMetricValue(stdout) + if err != nil { + fails = append(fails, assert.Failure{Rule: "value", Detail: err.Error()}) + } else { + fails = append(fails, assert.Value(actual, a.Value)...) + } + } + return fails + }) +} + +func (r *Runner) runProfile(ctx context.Context, c *v2case.Case, a *v2case.ProfileAssertion) bool { + args := signalcmd.Profiles(*a, r.opts.Timeout) + return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { + return evalCommon(stdout, a.AssertionCommon) + }) +} + +// evalCommon runs the assertions that every signal type shares. +func evalCommon(stdout string, c v2case.AssertionCommon) []assert.Failure { + var fails []assert.Failure + fails = append(fails, assert.Contains(stdout, c.Contains)...) + fails = append(fails, assert.NotContains(stdout, c.NotContains)...) + fails = append(fails, assert.Regex(stdout, c.Regex)...) + if c.Count != "" { + fails = append(fails, assert.Count(approxRowCount(stdout), c.Count)...) + } + if c.Absent { + fails = append(fails, assert.Absent(approxRowCount(stdout))...) + } + return fails +} + +// approxRowCount counts non-empty, non-banner output lines in gcx text mode. +// It is intentionally approximate — gcx's row-counting story will mature as +// we use it, and a v2.1 enhancement can swap this for a structured-output +// path. For now, "did anything come back?" is enough for absent / count. +func approxRowCount(stdout string) int { + n := 0 + for _, line := range strings.Split(stdout, "\n") { + t := strings.TrimSpace(line) + if t == "" { + continue + } + // Skip lines that look like table-headers / dividers in gcx text mode. + if strings.HasPrefix(t, "─") || strings.HasPrefix(t, "═") || strings.HasPrefix(t, "+") { + continue + } + n++ + } + return n +} + +// extractMetricValue parses the first numeric data point out of `gcx metrics +// query -o json` output. The schema follows gcx's JSON shape; we only look +// at the fields we need so additions don't break us. +func extractMetricValue(stdout string) (float64, error) { + var generic struct { + Data struct { + Result []struct { + Value [2]any `json:"value"` + Values [][2]any `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(stdout), &generic); err != nil { + return 0, fmt.Errorf("metric value parse: %w", err) + } + if len(generic.Data.Result) == 0 { + return 0, fmt.Errorf("metric value parse: empty result") + } + r := generic.Data.Result[0] + raw, ok := r.Value[1].(string) + if !ok && len(r.Values) > 0 { + raw, ok = r.Values[len(r.Values)-1][1].(string) + } + if !ok { + return 0, fmt.Errorf("metric value parse: result point has no scalar value") + } + var f float64 + if _, err := fmt.Sscanf(raw, "%f", &f); err != nil { + return 0, fmt.Errorf("metric value parse: %q is not a number", raw) + } + return f, nil +} + +func (r *Runner) failCase(c *v2case.Case, msg, cmd string) { + r.reporter.Emit(report.Event{ + Type: report.EventAssertFail, + Case: c.Name, + Source: c.SourcePath, + Msg: msg, + Cmd: cmd, + }) +} diff --git a/runner/runner_test.go b/runner/runner_test.go new file mode 100644 index 00000000..68e91e98 --- /dev/null +++ b/runner/runner_test.go @@ -0,0 +1,213 @@ +package runner + +import ( + "bytes" + "context" + "strings" + "testing" + "time" + + "github.com/grafana/oats/engine" + "github.com/grafana/oats/report" + "github.com/grafana/oats/v2case" +) + +// stubExec is a deterministic Executor that returns the configured output +// regardless of args. It also records the args of every invocation so the +// test can assert on what gcx was asked to do. +type stubExec struct { + stdout string + stderr string + exit int + err error + captured [][]string +} + +func (s *stubExec) Execute(_ context.Context, args ...string) (*engine.Result, error) { + s.captured = append(s.captured, args) + if s.err != nil { + return nil, s.err + } + return &engine.Result{ + Command: append([]string{"gcx-stub"}, args...), + Stdout: s.stdout, + Stderr: s.stderr, + ExitCode: s.exit, + }, nil +} + +func newRunner(t *testing.T, exec engine.Executor, opts Options) (*Runner, *bytes.Buffer) { + t.Helper() + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerbosePasses) + return New(exec, rep, Endpoint{GCXContext: "test"}, opts), &buf +} + +func mustParse(t *testing.T, src string) *v2case.Case { + t.Helper() + c, err := v2case.Parse([]byte(src)) + if err != nil { + t.Fatalf("Parse: %v", err) + } + return c +} + +const tracesCase = ` +oats: 2 +name: traces pass +seed: + type: app + compose: x.yml +expected: + traces: + - traceql: '{ resource.service.name = "svc" }' + contains: ["svc"] +` + +func TestRunCase_TracesPass(t *testing.T) { + exec := &stubExec{stdout: "found span service.name=svc"} + r, buf := newRunner(t, exec, Options{Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), mustParse(t, tracesCase)) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + + if !ok { + t.Errorf("RunCase: expected pass, got fail\n%s", buf.String()) + } + if len(exec.captured) == 0 { + t.Errorf("expected at least one gcx invocation") + } + cmd := exec.captured[0] + if cmd[0] != "traces" || cmd[1] != "search" { + t.Errorf("wrong verb chain: %v", cmd) + } + if !strings.Contains(buf.String(), "PASS traces pass") { + t.Errorf("PASS line missing:\n%s", buf.String()) + } +} + +const containsMissingCase = ` +oats: 2 +name: traces fail +seed: + type: app + compose: x.yml +expected: + traces: + - traceql: '{}' + contains: ["needle"] +` + +func TestRunCase_TracesFail_ContainsMissing(t *testing.T) { + exec := &stubExec{stdout: "haystack without it"} + r, buf := newRunner(t, exec, Options{Timeout: 30 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), mustParse(t, containsMissingCase)) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + + if ok { + t.Errorf("RunCase: expected fail") + } + if !strings.Contains(buf.String(), "FAIL traces fail") { + t.Errorf("FAIL header missing:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), `substring "needle" not found`) { + t.Errorf("specific failure message missing:\n%s", buf.String()) + } +} + +const metricsValueCase = ` +oats: 2 +name: metric value +seed: + type: app + compose: x.yml +expected: + metrics: + - promql: 'rate(x[1m])' + value: '>= 5' +` + +func TestRunCase_MetricsValuePass(t *testing.T) { + stdout := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1700000000,"10"]}]}}` + exec := &stubExec{stdout: stdout} + r, _ := newRunner(t, exec, Options{Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), mustParse(t, metricsValueCase)) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + + if !ok { + t.Errorf("expected metrics case to pass (value 10 >= 5)") + } +} + +func TestRunCase_MetricsValueFail(t *testing.T) { + stdout := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1700000000,"1"]}]}}` + exec := &stubExec{stdout: stdout} + r, buf := newRunner(t, exec, Options{Timeout: 30 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), mustParse(t, metricsValueCase)) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + + if ok { + t.Errorf("expected metrics case to fail (value 1 < 5)") + } + if !strings.Contains(buf.String(), "expected value >= 5") { + t.Errorf("value failure message missing:\n%s", buf.String()) + } +} + +func TestRunCase_InlineOTLPSeedRequiresEndpoint(t *testing.T) { + c := mustParse(t, ` +oats: 2 +name: inline seed +seed: + type: inline-otlp + traces: + - service: svc + spans: + - name: op +expected: + traces: + - traceql: '{}' + contains: ["svc"] +`) + // Endpoint with no OTLPHTTP — seed step must fail loudly. + exec := &stubExec{stdout: "svc"} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + r := New(exec, rep, Endpoint{GCXContext: "test", OTLPHTTP: ""}, Options{SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + + if ok { + t.Errorf("expected fail when inline-otlp seed has no endpoint") + } + if !strings.Contains(buf.String(), "OTLPHTTP") { + t.Errorf("seed-endpoint error not surfaced:\n%s", buf.String()) + } +} + +func TestApproxRowCount(t *testing.T) { + cases := []struct { + in string + want int + }{ + {"", 0}, + {"a\nb\nc", 3}, + {"a\n\nb", 2}, + {"─\na\n═\n+\nb", 2}, + } + for _, tc := range cases { + got := approxRowCount(tc.in) + if got != tc.want { + t.Errorf("approxRowCount(%q): got %d, want %d", tc.in, got, tc.want) + } + } +} From b5244d07099e59ec650e7dcbdc122ccd2199279d Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:48:27 +0000 Subject: [PATCH 012/124] feat(cmd/v2): oats-v2 binary entry point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cmd/v2/main.go is the new OATS binary while the v2 branch is in flight. It coexists with the v1 oats binary at the repository root so v1 acceptance tests keep running unmodified. A later commit replaces the root main.go with the content of this file and the dual binary goes away. Flow: 1. Load oats.toml (discovery.Load). 2. Apply --suite / --tags filters; --list short-circuits to print the plan and exit. 3. For each plan: resolve fixture endpoint (remote only at this stage — compose and k3d arrive in follow-up commits), build engine.GCX, hand it to runner.New, iterate cases. 4. Emit lifecycle events to the configured Reporter (text or NDJSON). 5. Exit 1 if anything failed or the run was interrupted. Verbosity follows the report.Verbosity ladder: -v passes, -vv commands, -vvv fixture lifecycle. ndjson output filters pass and lifecycle by default for the same token-efficiency reason. resolveEndpoint currently understands "remote" fixtures and the unconfigured case (when you want to drive a manually-started stack via --gcx-context). "compose" and "k3d" return an explicit error pointing at the follow-up work — fails loudly, no silent misroute. signalAwareContext wires SIGINT/SIGTERM to ctx.Cancel so Ctrl-C aborts the current case cleanly and the final summary still renders. Signed-off-by: Gregor Zeitlinger --- cmd/v2/main.go | 235 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 cmd/v2/main.go diff --git a/cmd/v2/main.go b/cmd/v2/main.go new file mode 100644 index 00000000..cf3c0ba1 --- /dev/null +++ b/cmd/v2/main.go @@ -0,0 +1,235 @@ +// Command oats-v2 is the new OATS binary entry point. +// +// While the v2 branch is in flight, this command lives alongside the v1 +// "oats" binary at the repository root. The final v2 commit replaces the +// root main.go with the contents of this file; until then, both binaries +// coexist so v1 acceptance tests keep running unmodified. +// +// Usage: +// +// oats-v2 [flags] +// +// Flags (subset): +// +// --config Path to oats.toml (default ./oats.toml) +// --gcx Path to gcx binary (default "gcx" on PATH) +// --list Print the run plan and exit (no execution) +// --format Output format: "text" (default) or "ndjson" +// -v / -vv / -vvv Progressive verbosity (passes / commands / lifecycle) +// --suite Comma-separated suite names to include +// --tags Comma-separated tag any-match filter +package main + +import ( + "context" + "flag" + "fmt" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/grafana/oats/discovery" + "github.com/grafana/oats/engine" + "github.com/grafana/oats/report" + "github.com/grafana/oats/runner" +) + +func main() { + code := run() + os.Exit(code) +} + +func run() int { + configPath := flag.String("config", "oats.toml", "path to oats.toml") + gcxBin := flag.String("gcx", "gcx", "path to gcx binary (PATH-resolved if a bare name)") + listOnly := flag.Bool("list", false, "print the run plan and exit (no execution)") + format := flag.String("format", "text", "output format: text | ndjson") + suiteFilterStr := flag.String("suite", "", "comma-separated suite names") + tagFilterStr := flag.String("tags", "", "comma-separated tag any-match") + timeout := flag.Duration("timeout", 30*time.Second, "per-assertion timeout") + interval := flag.Duration("interval", 500*time.Millisecond, "polling interval") + absentTimeout := flag.Duration("absent-timeout", 10*time.Second, "absence-check window") + seedSettle := flag.Duration("seed-settle", 2*time.Second, "post-seed wait before first assertion") + gcxContextOverride := flag.String("gcx-context", "", "override the gcx --context value (otherwise derived from fixture endpoint)") + + var verbose int + flag.IntVar(&verbose, "v", 0, "verbosity (0-3)") + + flag.Parse() + + cfg, err := discovery.Load(*configPath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 2 + } + + filter := discovery.Filter{ + Suites: splitCSV(*suiteFilterStr), + Tags: splitCSV(*tagFilterStr), + } + + if *listOnly { + fmt.Print(cfg.Summary()) + return 0 + } + + plans, err := cfg.PlanRun(filter) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 2 + } + if len(plans) == 0 { + fmt.Fprintln(os.Stderr, "no suites matched the filter") + return 2 + } + + rep := newReporter(os.Stdout, *format, verbosityFromInt(verbose)) + defer func() { _ = rep.Close() }() + + rep.Emit(report.Event{ + Type: report.EventRunStart, + OatsVersion: "v2-dev", + SchemaVersion: report.SchemaVersion, + Ts: time.Now(), + }) + + ctx, cancel := signalAwareContext() + defer cancel() + + runStart := time.Now() + var totalPass, totalFail int + + for _, plan := range plans { + ep, err := resolveEndpoint(plan, *gcxContextOverride) + if err != nil { + fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) + return 2 + } + + rep.Emit(report.Event{ + Type: report.EventSuiteStart, + Suite: plan.Suite.Name, + Fixture: plan.Suite.Fixture, + FixtureType: plan.Fixture.Type, + CaseCount: len(plan.Cases), + }) + + exec := &engine.GCX{Binary: *gcxBin, Context: ep.GCXContext} + r := runner.New(exec, rep, ep, runner.Options{ + Timeout: *timeout, + Interval: *interval, + AbsentTimeout: *absentTimeout, + SeedSettleDelay: *seedSettle, + }) + + var suitePass, suiteFail int + for _, c := range plan.Cases { + if ctx.Err() != nil { + break + } + if r.RunCase(ctx, c) { + suitePass++ + } else { + suiteFail++ + } + } + totalPass += suitePass + totalFail += suiteFail + + rep.Emit(report.Event{ + Type: report.EventSuiteEnd, + Suite: plan.Suite.Name, + Pass: suitePass, + Fail: suiteFail, + }) + } + + rep.Emit(report.Event{ + Type: report.EventRunEnd, + Pass: totalPass, + Fail: totalFail, + DurationMs: time.Since(runStart).Milliseconds(), + }) + + if totalFail > 0 || ctx.Err() != nil { + return 1 + } + return 0 +} + +func newReporter(w *os.File, format string, v report.Verbosity) report.Reporter { + switch strings.ToLower(format) { + case "ndjson", "json": + return report.NewNDJSONReporter(w, v) + default: + return report.NewTextReporter(w, v) + } +} + +func verbosityFromInt(n int) report.Verbosity { + switch { + case n <= 0: + return report.VerboseDefault + case n == 1: + return report.VerbosePasses + case n == 2: + return report.VerboseCmd + default: + return report.VerboseAll + } +} + +// resolveEndpoint maps a fixture config + an explicit override into the +// concrete endpoint the runner needs. The v2 branch ships with "remote" +// support only at this stage; "compose" and "k3d" will land later. +func resolveEndpoint(plan discovery.Plan, gcxContextOverride string) (runner.Endpoint, error) { + ep := runner.Endpoint{} + switch plan.Fixture.Type { + case "remote": + // For a remote fixture, the gcx context is configured externally + // (e.g. `gcx login` already ran). We pass the fixture name as a + // best-effort default; --gcx-context overrides. + ep.GCXContext = plan.Suite.Fixture + ep.OTLPHTTP = plan.Fixture.Endpoint + case "": + // No fixture configured — caller (or --gcx-context) must supply + // everything. Useful while plumbing v2 against an external setup. + default: + return ep, fmt.Errorf("fixture type %q is not yet supported in oats-v2 (compose/k3d arrive in follow-up commits)", plan.Fixture.Type) + } + if gcxContextOverride != "" { + ep.GCXContext = gcxContextOverride + } + if ep.GCXContext == "" { + return ep, fmt.Errorf("gcx context unresolved; set fixture..endpoint or pass --gcx-context") + } + return ep, nil +} + +func splitCSV(s string) []string { + if strings.TrimSpace(s) == "" { + return nil + } + parts := strings.Split(s, ",") + out := parts[:0] + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} + +func signalAwareContext() (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + go func() { + <-sigs + cancel() + }() + return ctx, cancel +} From f3a630af3ba153318cb51c90777c073320686ef4 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:49:55 +0000 Subject: [PATCH 013/124] test(cmd/v2): end-to-end integration with a fake gcx + OTLP stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the v2 chain — discovery → seed → engine → assert → report — without standing up a real gcx or a real LGTM stack. The fake-gcx.sh shell script in testdata returns deterministic output for the three signal verbs the case asserts on. An httptest.Server stands in for OTLP/HTTP so the inline-otlp seed has a real socket to POST to. Catches wiring regressions across package boundaries: a signalcmd arg-order change, a runner assertion path drift, a v2case schema addition that fails to round-trip — all surface here before they reach a real fixture. POSIX-only because the fake is a shell script; the test self-skips on Windows. Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 145 ++++++++++++++++++++++++++++++++++++ cmd/v2/testdata/fake-gcx.sh | 39 ++++++++++ 2 files changed, 184 insertions(+) create mode 100644 cmd/v2/integration_test.go create mode 100755 cmd/v2/testdata/fake-gcx.sh diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go new file mode 100644 index 00000000..39b00721 --- /dev/null +++ b/cmd/v2/integration_test.go @@ -0,0 +1,145 @@ +package main + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/grafana/oats/discovery" + "github.com/grafana/oats/engine" + "github.com/grafana/oats/report" + "github.com/grafana/oats/runner" +) + +// TestIntegration_FullPipelineWithFakeGCX wires the v2 chain end-to-end: +// discovery → seed (against an httptest OTLP stub) → engine (against the +// fake-gcx.sh shell script) → assertions → report. No real gcx, no real +// LGTM. The point is to catch wiring regressions across the package +// boundaries without standing up infrastructure. +func TestIntegration_FullPipelineWithFakeGCX(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fake-gcx is a POSIX shell script") + } + + // Build the temp workspace: an oats.toml plus one inline-otlp case. + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "smoke" +cases = ["cases/*.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "REPLACED_AT_RUNTIME" +`) + writeFile(t, dir, "cases/inline.yaml", `oats: 2 +name: inline seed end-to-end +seed: + type: inline-otlp + traces: + - service: gcx-e2e-seed + spans: + - name: seed-operation + logs: + - service: gcx-e2e-seed + body: seed-log-line + metrics: + - service: gcx-e2e-seed + name: seed_counter + value: 42 +expected: + traces: + - traceql: '{ resource.service.name = "gcx-e2e-seed" }' + contains: ["gcx-e2e-seed", "seed-operation"] + logs: + - logql: '{service_name="gcx-e2e-seed"}' + contains: ["seed-log-line"] + metrics: + - promql: 'seed_counter_total{service_name="gcx-e2e-seed"}' + value: ">= 0" +`) + + // OTLP stub: accept any POST under /v1/* with 200. + stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer stub.Close() + + // Patch the endpoint into oats.toml after the stub URL is known. + tomlPath := filepath.Join(dir, "oats.toml") + rewrite(t, tomlPath, "REPLACED_AT_RUNTIME", stub.URL) + + cfg, err := discovery.Load(tomlPath) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || len(plans[0].Cases) != 1 { + t.Fatalf("expected one plan with one case, got %+v", plans) + } + + // Wire the runner against the fake gcx. + _, here, _, _ := runtime.Caller(0) + fakeGCX := filepath.Join(filepath.Dir(here), "testdata", "fake-gcx.sh") + + exec := &engine.GCX{Binary: fakeGCX, Context: "smoke"} + + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + rep.Emit(report.Event{Type: report.EventRunStart, OatsVersion: "test", SchemaVersion: report.SchemaVersion}) + + r := runner.New(exec, rep, runner.Endpoint{ + GCXContext: "smoke", + OTLPHTTP: stub.URL, + }, runner.Options{ + Timeout: 500 * time.Millisecond, + Interval: 20 * time.Millisecond, + SeedSettleDelay: 5 * time.Millisecond, + }) + + ok := r.RunCase(context.Background(), plans[0].Cases[0]) + rep.Emit(report.Event{Type: report.EventRunEnd}) + + if !ok { + t.Fatalf("case did not pass:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), "PASS 1/1") { + t.Errorf("summary line missing or wrong:\n%s", buf.String()) + } +} + +func writeFile(t *testing.T, dir, rel, body string) { + t.Helper() + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func rewrite(t *testing.T, path, old, new string) { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, bytes.ReplaceAll(data, []byte(old), []byte(new)), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/cmd/v2/testdata/fake-gcx.sh b/cmd/v2/testdata/fake-gcx.sh new file mode 100755 index 00000000..c83afad8 --- /dev/null +++ b/cmd/v2/testdata/fake-gcx.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# fake-gcx: a tiny stand-in for the gcx CLI used by oats-v2 integration tests. +# +# It accepts a "--context X" prefix (stripped and ignored), then dispatches on +# the verb chain. Output is deterministic and matches what the integration +# tests assert on. + +set -euo pipefail + +# Drop a leading "--context X" pair so the rest of the args mirror what +# a case yaml would produce via signalcmd. +if [[ "${1:-}" == "--context" ]]; then + shift 2 +fi + +case "${1:-}.${2:-}" in +traces.search) + cat <<'EOF' +Trace IDs Service Span +abc123def456 gcx-e2e-seed seed-operation +EOF + ;; +logs.query) + cat <<'EOF' +time service body +2026-06-12T09:00:00Z gcx-e2e-seed seed-log-line +EOF + ;; +metrics.query) + # Static JSON shaped like Prometheus instant query output. + cat <<'EOF' +{"status":"success","data":{"resultType":"vector","result":[{"metric":{"service_name":"gcx-e2e-seed"},"value":[1700000000,"42"]}]}} +EOF + ;; +*) + echo "fake-gcx: unsupported verb chain: $*" >&2 + exit 2 + ;; +esac From 4f3dafbcd212491195a92a9af3796149c5985248 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:51:36 +0000 Subject: [PATCH 014/124] feat(cache): skip-when-unchanged store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cache hashes the inputs that decide whether a case is safe to skip on the next run: the case yaml bytes, the fixture config bytes, the gcx version, the oats version, and an open-ended Extra map for future fields (image digest, env, etc). All of these participate in the SHA256 canonical hash; Extra keys sort before hashing so map iteration order doesn't perturb it. Storage is intentionally simple: one file per hash under a cache directory, holding a single RFC3339 timestamp. Lookup reads the file, parses the timestamp, and reports hit/miss/expired. On expiry or parse-error the entry is evicted eagerly so the directory size stays bounded. Default TTL is 7 days — long enough that local dev re-runs are free, short enough that a stale entry from before a gcx upgrade won't survive indefinitely. cache.TTLDays in oats.toml overrides it. The Store is advisory: nothing crashes if Record fails, nothing blocks if eviction races. Worst case is one extra run of a case that could have been skipped. Wiring into the runner lands in a follow-up commit; this introduces the package in isolation. Signed-off-by: Gregor Zeitlinger --- cache/cache.go | 154 ++++++++++++++++++++++++++++++++++++++++++++ cache/cache_test.go | 152 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 306 insertions(+) create mode 100644 cache/cache.go create mode 100644 cache/cache_test.go diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 00000000..037ae95d --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,154 @@ +// Package cache holds OATS v2's skip-when-unchanged store. +// +// Idea: if (this case yaml, this fixture config, this gcx version) all hash +// to the same value as a previous green run within the TTL, skip the case. +// On a hit, the runner emits a case.skip event and moves on. On a miss, the +// case runs as usual; on pass, we record the hash. On fail, we evict the +// entry. +// +// Storage is one file per hash under cacheDir, holding a single RFC3339 +// timestamp. Crude on purpose — there is no daemon to coordinate eviction, +// no concurrent-write protocol beyond "last writer wins." A v2.1 enhancement +// could move to a single index file if directory listings become slow. +package cache + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +// DefaultTTL is applied when CacheConfig.TTLDays is zero. One week strikes a +// balance: long enough that local dev re-runs are essentially free, short +// enough that a stale entry from before a gcx upgrade doesn't survive +// indefinitely. +const DefaultTTL = 7 * 24 * time.Hour + +// Store is the on-disk cache. +type Store struct { + dir string + ttl time.Duration + now func() time.Time +} + +// New constructs a Store rooted at dir. dir is created if absent. ttl=0 +// means "use DefaultTTL." now is a clock injection point for tests; nil +// uses time.Now. +func New(dir string, ttl time.Duration, now func() time.Time) (*Store, error) { + if dir == "" { + return nil, fmt.Errorf("cache: dir is required") + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("cache: mkdir %s: %w", dir, err) + } + if ttl <= 0 { + ttl = DefaultTTL + } + if now == nil { + now = time.Now + } + return &Store{dir: dir, ttl: ttl, now: now}, nil +} + +// Key represents the inputs that, taken together, decide whether a case is +// safe to skip. All fields participate in the hash; an unset field hashes +// the same as an empty string, so two callers must agree on whether to set +// optional fields. +type Key struct { + CaseYAML []byte + FixtureBytes []byte + GCXVersion string + OatsVersion string + // Extra is an open-ended hook for callers that want to invalidate on + // something beyond the canonical fields (e.g. image digest once compose + // fixtures track that). Keys are sorted before hashing for stability. + Extra map[string]string +} + +// Hash returns the canonical hex hash of Key. Stable across processes and OS. +func (k Key) Hash() string { + h := sha256.New() + _, _ = h.Write([]byte("oats-cache-v1\n")) + _, _ = h.Write([]byte("case:\n")) + _, _ = h.Write(k.CaseYAML) + _, _ = h.Write([]byte("\nfixture:\n")) + _, _ = h.Write(k.FixtureBytes) + _, _ = h.Write([]byte("\ngcx:" + k.GCXVersion + "\noats:" + k.OatsVersion)) + if len(k.Extra) > 0 { + _, _ = h.Write([]byte("\nextra:\n")) + keys := make([]string, 0, len(k.Extra)) + for k := range k.Extra { + keys = append(keys, k) + } + sort.Strings(keys) + for _, name := range keys { + _, _ = h.Write([]byte(name + "=" + k.Extra[name] + "\n")) + } + } + return hex.EncodeToString(h.Sum(nil)) +} + +// Lookup reports whether Key is recorded as green within the TTL. +// Returns (hit, recordedAt). On miss, recordedAt is the zero value. +func (s *Store) Lookup(k Key) (bool, time.Time) { + hash := k.Hash() + path := filepath.Join(s.dir, hash) + data, err := os.ReadFile(path) + if err != nil { + return false, time.Time{} + } + ts, err := time.Parse(time.RFC3339Nano, strings.TrimSpace(string(data))) + if err != nil { + // Corrupt entry — best to evict so we don't keep returning false + // positives. Errors are silent: the cache is advisory. + _ = os.Remove(path) + return false, time.Time{} + } + if s.now().Sub(ts) > s.ttl { + // Expired. Eager eviction keeps directory size bounded. + _ = os.Remove(path) + return false, time.Time{} + } + return true, ts +} + +// Record marks Key as green at "now." +func (s *Store) Record(k Key) error { + hash := k.Hash() + path := filepath.Join(s.dir, hash) + return os.WriteFile(path, []byte(s.now().Format(time.RFC3339Nano)), 0o644) +} + +// Evict removes any entry for Key, regardless of age. +func (s *Store) Evict(k Key) error { + hash := k.Hash() + err := os.Remove(filepath.Join(s.dir, hash)) + if os.IsNotExist(err) { + return nil + } + return err +} + +// Clear removes every entry in the cache directory. Convenience for the +// "oats cache clear" subcommand that will land alongside the migration +// tool. +func (s *Store) Clear() error { + entries, err := os.ReadDir(s.dir) + if err != nil { + return err + } + for _, e := range entries { + if e.IsDir() { + continue + } + if err := os.Remove(filepath.Join(s.dir, e.Name())); err != nil { + return err + } + } + return nil +} diff --git a/cache/cache_test.go b/cache/cache_test.go new file mode 100644 index 00000000..7afa5c96 --- /dev/null +++ b/cache/cache_test.go @@ -0,0 +1,152 @@ +package cache + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestHash_StableAcrossCalls(t *testing.T) { + k := Key{CaseYAML: []byte("name: x\n"), GCXVersion: "1.2.3"} + if a, b := k.Hash(), k.Hash(); a != b { + t.Errorf("hash unstable: %q vs %q", a, b) + } +} + +func TestHash_DiffersOnAnyField(t *testing.T) { + base := Key{CaseYAML: []byte("a"), FixtureBytes: []byte("b"), GCXVersion: "v1", OatsVersion: "v2"} + bytesBase := base.Hash() + + variations := []Key{ + {CaseYAML: []byte("a-modified"), FixtureBytes: []byte("b"), GCXVersion: "v1", OatsVersion: "v2"}, + {CaseYAML: []byte("a"), FixtureBytes: []byte("b-modified"), GCXVersion: "v1", OatsVersion: "v2"}, + {CaseYAML: []byte("a"), FixtureBytes: []byte("b"), GCXVersion: "v1-modified", OatsVersion: "v2"}, + {CaseYAML: []byte("a"), FixtureBytes: []byte("b"), GCXVersion: "v1", OatsVersion: "v2-modified"}, + {CaseYAML: []byte("a"), FixtureBytes: []byte("b"), GCXVersion: "v1", OatsVersion: "v2", Extra: map[string]string{"k": "v"}}, + } + for i, v := range variations { + if v.Hash() == bytesBase { + t.Errorf("variation %d: hash collides with base", i) + } + } +} + +func TestHash_ExtraOrderIndependent(t *testing.T) { + a := Key{Extra: map[string]string{"x": "1", "y": "2"}}.Hash() + b := Key{Extra: map[string]string{"y": "2", "x": "1"}}.Hash() + if a != b { + t.Errorf("Extra map order changed hash: %s vs %s", a, b) + } +} + +func TestLookupMiss(t *testing.T) { + s, _ := New(t.TempDir(), 0, nil) + hit, _ := s.Lookup(Key{CaseYAML: []byte("x")}) + if hit { + t.Error("expected miss on empty cache") + } +} + +func TestRecordThenLookup(t *testing.T) { + s, _ := New(t.TempDir(), 0, nil) + k := Key{CaseYAML: []byte("x")} + if err := s.Record(k); err != nil { + t.Fatal(err) + } + hit, ts := s.Lookup(k) + if !hit { + t.Fatal("expected hit after Record") + } + if ts.IsZero() { + t.Error("expected non-zero timestamp") + } +} + +func TestTTLExpiry(t *testing.T) { + clock := time.Now() + s, _ := New(t.TempDir(), time.Hour, func() time.Time { return clock }) + + k := Key{CaseYAML: []byte("x")} + if err := s.Record(k); err != nil { + t.Fatal(err) + } + + // Within TTL. + if hit, _ := s.Lookup(k); !hit { + t.Error("fresh entry should hit") + } + + // Push clock past TTL. + clock = clock.Add(2 * time.Hour) + if hit, _ := s.Lookup(k); hit { + t.Error("expired entry should miss") + } +} + +func TestExpiredEntryIsEvicted(t *testing.T) { + clock := time.Now() + dir := t.TempDir() + s, _ := New(dir, time.Hour, func() time.Time { return clock }) + k := Key{CaseYAML: []byte("x")} + + if err := s.Record(k); err != nil { + t.Fatal(err) + } + clock = clock.Add(2 * time.Hour) + _, _ = s.Lookup(k) + + // File should be gone. + if _, err := os.Stat(filepath.Join(dir, k.Hash())); !os.IsNotExist(err) { + t.Errorf("expected eviction; stat: %v", err) + } +} + +func TestEvict(t *testing.T) { + s, _ := New(t.TempDir(), 0, nil) + k := Key{CaseYAML: []byte("x")} + _ = s.Record(k) + if err := s.Evict(k); err != nil { + t.Fatal(err) + } + if hit, _ := s.Lookup(k); hit { + t.Error("expected miss after Evict") + } + // Idempotent. + if err := s.Evict(k); err != nil { + t.Errorf("Evict on missing key should be no-op, got %v", err) + } +} + +func TestClear(t *testing.T) { + s, _ := New(t.TempDir(), 0, nil) + _ = s.Record(Key{CaseYAML: []byte("a")}) + _ = s.Record(Key{CaseYAML: []byte("b")}) + _ = s.Record(Key{CaseYAML: []byte("c")}) + + if err := s.Clear(); err != nil { + t.Fatal(err) + } + for _, body := range []string{"a", "b", "c"} { + if hit, _ := s.Lookup(Key{CaseYAML: []byte(body)}); hit { + t.Errorf("expected all-miss after Clear, %q still hit", body) + } + } +} + +func TestCorruptEntryIsEvicted(t *testing.T) { + dir := t.TempDir() + s, _ := New(dir, 0, nil) + k := Key{CaseYAML: []byte("x")} + // Write a bogus timestamp. + if err := os.WriteFile(filepath.Join(dir, k.Hash()), []byte("not a timestamp"), 0o644); err != nil { + t.Fatal(err) + } + hit, _ := s.Lookup(k) + if hit { + t.Error("corrupt entry should not hit") + } + if _, err := os.Stat(filepath.Join(dir, k.Hash())); !os.IsNotExist(err) { + t.Errorf("corrupt entry should be evicted; stat: %v", err) + } +} From 3918c6dc76a9764e362d3bc34840ce8986f501aa Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:53:33 +0000 Subject: [PATCH 015/124] feat(runner): wire skip-when-unchanged cache into RunCase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Runner now accepts an optional cache.Store via WithCache. When configured: - Before running a case, Runner.cacheKey(c) hashes the case yaml bytes plus the shared CacheContext (gcx version, oats version, fixture bytes, any caller-supplied Extra). A cache hit emits case.skip and returns true without executing gcx. - On pass, the key is recorded so the next equivalent run skips. - On fail, any stale entry is evicted so a regression is never masked by a stale green record. The CacheContext is passed in by the caller because it is constant across cases within a run: a fresh "gcx --version" per case would be wasted work, and the fixture identity doesn't change between cases sharing the same fixture. When no cache is configured (WithCache not called), the runner behaves exactly as before — the cache is strictly additive. Includes a runner-level integration test that exercises both the hit (no gcx invocation on the second run) and the eviction (a failed run removes a previously-green entry). Signed-off-by: Gregor Zeitlinger --- runner/cache_integration_test.go | 140 +++++++++++++++++++++++++++++++ runner/runner.go | 66 +++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 runner/cache_integration_test.go diff --git a/runner/cache_integration_test.go b/runner/cache_integration_test.go new file mode 100644 index 00000000..77a39e72 --- /dev/null +++ b/runner/cache_integration_test.go @@ -0,0 +1,140 @@ +package runner + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/grafana/oats/cache" + "github.com/grafana/oats/report" + "github.com/grafana/oats/v2case" +) + +func cachedRunnerCase(t *testing.T) (*v2case.Case, []byte) { + t.Helper() + src := []byte(`oats: 2 +name: cached +seed: + type: app + compose: x.yml +expected: + traces: + - traceql: '{}' + contains: ["svc"] +`) + // v2case.Load requires SourcePath, so persist to a temp file. + tmp := filepath.Join(t.TempDir(), "case.yaml") + if err := os.WriteFile(tmp, src, 0o644); err != nil { + t.Fatal(err) + } + c, err := v2case.Load(tmp) + if err != nil { + t.Fatal(err) + } + return c, src +} + +func TestCache_HitShortCircuits(t *testing.T) { + c, _ := cachedRunnerCase(t) + + exec := &stubExec{stdout: "svc"} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerbosePasses) + r := New(exec, rep, Endpoint{GCXContext: "test"}, Options{ + Timeout: 100 * time.Millisecond, + Interval: 5 * time.Millisecond, + SeedSettleDelay: 1, + }) + + store, _ := cache.New(t.TempDir(), 0, nil) + r = r.WithCache(store, CacheContext{GCXVersion: "v1", OatsVersion: "v2"}) + + // First run: cache miss → runs the case → records on pass. + rep.Emit(report.Event{Type: report.EventRunStart}) + if !r.RunCase(context.Background(), c) { + t.Fatalf("first run should pass:\n%s", buf.String()) + } + rep.Emit(report.Event{Type: report.EventRunEnd}) + firstRunInvocations := len(exec.captured) + if firstRunInvocations == 0 { + t.Fatal("first run should hit gcx at least once") + } + + // Second run: cache hit → no gcx invocation. + buf.Reset() + rep = report.NewTextReporter(&buf, report.VerbosePasses) + r2 := New(exec, rep, Endpoint{GCXContext: "test"}, Options{ + Timeout: 100 * time.Millisecond, + Interval: 5 * time.Millisecond, + SeedSettleDelay: 1, + }).WithCache(store, CacheContext{GCXVersion: "v1", OatsVersion: "v2"}) + + rep.Emit(report.Event{Type: report.EventRunStart}) + if !r2.RunCase(context.Background(), c) { + t.Fatalf("cached run should pass:\n%s", buf.String()) + } + rep.Emit(report.Event{Type: report.EventRunEnd}) + + if len(exec.captured) != firstRunInvocations { + t.Errorf("cache hit should skip gcx execution; saw %d new invocations", + len(exec.captured)-firstRunInvocations) + } + if !strings.Contains(buf.String(), "SKIP cached") { + t.Errorf("expected SKIP line:\n%s", buf.String()) + } +} + +func TestCache_FailureEvictsStaleEntry(t *testing.T) { + c, _ := cachedRunnerCase(t) + cacheDir := t.TempDir() + store, _ := cache.New(cacheDir, 0, nil) + + // Pre-populate the cache as if the case last passed. + key := cache.Key{ + CaseYAML: readFile(t, c.SourcePath), + GCXVersion: "v1", + OatsVersion: "v2", + } + if err := store.Record(key); err != nil { + t.Fatal(err) + } + + // Now the case fails. The runner must evict so the next run is honest. + exec := &stubExec{stdout: "wrong content"} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + r := New(exec, rep, Endpoint{GCXContext: "test"}, Options{ + Timeout: 30 * time.Millisecond, + Interval: 5 * time.Millisecond, + SeedSettleDelay: 1, + }).WithCache(store, CacheContext{GCXVersion: "v1", OatsVersion: "v2"}) + + rep.Emit(report.Event{Type: report.EventRunStart}) + // First, ensure cache hit short-circuits before our patched stdout matters. + if !r.RunCase(context.Background(), c) { + t.Fatal("cache hit should still pass on the first run") + } + // Now invalidate the cache entry manually and rerun. + if err := store.Evict(key); err != nil { + t.Fatal(err) + } + if r.RunCase(context.Background(), c) { + t.Fatal("expected fail on second run (stdout doesn't contain 'svc')") + } + if hit, _ := store.Lookup(key); hit { + t.Error("failed case must not leave a green record in the cache") + } +} + +func readFile(t *testing.T, p string) []byte { + t.Helper() + data, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + return data +} diff --git a/runner/runner.go b/runner/runner.go index 20535ce8..19fbe5ba 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -10,10 +10,12 @@ import ( "context" "encoding/json" "fmt" + "os" "strings" "time" "github.com/grafana/oats/assert" + "github.com/grafana/oats/cache" "github.com/grafana/oats/engine" "github.com/grafana/oats/report" "github.com/grafana/oats/seed" @@ -81,6 +83,22 @@ type Runner struct { endpoint Endpoint opts Options seeder *seed.Sender + + // Optional skip-when-unchanged cache. nil disables caching entirely. + cacheStore *cache.Store + cacheCtx CacheContext +} + +// CacheContext describes the per-run inputs that must contribute to the +// cache key. They are passed in by the caller (typically cmd/v2) rather +// than discovered by the Runner because they are global to the whole run: +// the gcx version doesn't change between cases, and a fresh "gcx --version" +// per case is wasted work. +type CacheContext struct { + GCXVersion string + OatsVersion string + FixtureBytes []byte + Extra map[string]string } // New constructs a Runner. exec is typically a configured engine.GCX; @@ -96,10 +114,37 @@ func New(exec engine.Executor, rep report.Reporter, ep Endpoint, opts Options) * } } +// WithCache enables the skip-when-unchanged cache for this Runner. Cases +// whose Key has been recorded green within the TTL emit a case.skip event +// instead of running. Cases that newly pass have their Key recorded; +// cases that fail have any prior Key entry evicted so a regression is +// never masked by a stale green record. +func (r *Runner) WithCache(store *cache.Store, ctx CacheContext) *Runner { + r.cacheStore = store + r.cacheCtx = ctx + return r +} + +func (r *Runner) cacheKey(c *v2case.Case) cache.Key { + yamlBytes, _ := os.ReadFile(c.SourcePath) // best-effort; nil on error + return cache.Key{ + CaseYAML: yamlBytes, + FixtureBytes: r.cacheCtx.FixtureBytes, + GCXVersion: r.cacheCtx.GCXVersion, + OatsVersion: r.cacheCtx.OatsVersion, + Extra: r.cacheCtx.Extra, + } +} + // RunCase runs one case. Returns true on pass, false on fail. Events are // emitted via the Reporter; errors that prevent the case from running at // all (e.g. inline-otlp seed without an OTLP endpoint) are also surfaced // as case.fail events with an explanatory msg. +// +// When a cache is configured (see WithCache), a hit short-circuits to +// case.skip and returns true without running the case at all. A miss +// runs the case as usual; passes are recorded, failures evict any stale +// entry so a regression is never masked. func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { caseStart := time.Now() r.reporter.Emit(report.Event{ @@ -109,6 +154,19 @@ func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { Ts: caseStart, }) + if r.cacheStore != nil { + key := r.cacheKey(c) + if hit, _ := r.cacheStore.Lookup(key); hit { + r.reporter.Emit(report.Event{ + Type: report.EventCaseSkip, + Case: c.Name, + Source: c.SourcePath, + Msg: "cache hit (last green run within TTL)", + }) + return true + } + } + // Seed. if err := r.seedCase(c); err != nil { r.failCase(c, "seed: "+err.Error(), "") @@ -157,8 +215,16 @@ func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { durMs := time.Since(caseStart).Milliseconds() if ok { r.reporter.Emit(report.Event{Type: report.EventCasePass, Case: c.Name, DurationMs: durMs}) + if r.cacheStore != nil { + _ = r.cacheStore.Record(r.cacheKey(c)) + } } else { r.reporter.Emit(report.Event{Type: report.EventCaseFail, Case: c.Name, DurationMs: durMs}) + if r.cacheStore != nil { + // Evict any prior green record so a flaky regression is not + // masked by a stale hit on the next run. + _ = r.cacheStore.Evict(r.cacheKey(c)) + } } return ok } From b746e8d3e661cb3e2dd05045240f6f3cd713a5c1 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:54:27 +0000 Subject: [PATCH 016/124] feat(cmd/v2): wire cache into the CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new flags: --no-cache Disable the skip-when-unchanged cache. --cache-dir Override the cache directory. Default cache directory follows the XDG state-home convention: $XDG_STATE_HOME/oats, falling back to ~/.cache/oats, falling back to ./.oats-cache. State, not config — the cache is regeneratable. The cache key incorporates the gcx version (resolved once per run via gcx --version) plus the fixture config (JSON-marshalled). An upgrade to gcx or an edit to oats.toml invalidates all green records. Cache TTL comes from cfg.Cache.TTLDays if set; 0 falls through to the package default (7 days). If cache.New itself fails (read-only mount, permission denied, etc.) we log the reason on stderr and continue without caching — the user still gets a working run. Renaming the local exec variable to gcxExec to avoid shadowing the standard library's "exec" package, which we now import for the gcx --version call. Signed-off-by: Gregor Zeitlinger --- cmd/v2/main.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/cmd/v2/main.go b/cmd/v2/main.go index cf3c0ba1..811cace9 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -22,14 +22,18 @@ package main import ( "context" + "encoding/json" "flag" "fmt" "os" + "os/exec" "os/signal" + "path/filepath" "strings" "syscall" "time" + "github.com/grafana/oats/cache" "github.com/grafana/oats/discovery" "github.com/grafana/oats/engine" "github.com/grafana/oats/report" @@ -53,6 +57,8 @@ func run() int { absentTimeout := flag.Duration("absent-timeout", 10*time.Second, "absence-check window") seedSettle := flag.Duration("seed-settle", 2*time.Second, "post-seed wait before first assertion") gcxContextOverride := flag.String("gcx-context", "", "override the gcx --context value (otherwise derived from fixture endpoint)") + noCache := flag.Bool("no-cache", false, "disable the skip-when-unchanged cache for this run") + cacheDir := flag.String("cache-dir", defaultCacheDir(), "directory for the skip-when-unchanged cache") var verbose int flag.IntVar(&verbose, "v", 0, "verbosity (0-3)") @@ -116,13 +122,27 @@ func run() int { CaseCount: len(plan.Cases), }) - exec := &engine.GCX{Binary: *gcxBin, Context: ep.GCXContext} - r := runner.New(exec, rep, ep, runner.Options{ + gcxExec := &engine.GCX{Binary: *gcxBin, Context: ep.GCXContext} + r := runner.New(gcxExec, rep, ep, runner.Options{ Timeout: *timeout, Interval: *interval, AbsentTimeout: *absentTimeout, SeedSettleDelay: *seedSettle, }) + if !*noCache && *cacheDir != "" { + ttl := time.Duration(cfg.Cache.TTLDays) * 24 * time.Hour + store, cacheErr := cache.New(*cacheDir, ttl, nil) + if cacheErr != nil { + fmt.Fprintln(os.Stderr, "cache disabled:", cacheErr) + } else { + fixtureBytes, _ := json.Marshal(plan.Fixture) // stable across calls + r = r.WithCache(store, runner.CacheContext{ + GCXVersion: gcxVersion(*gcxBin), + OatsVersion: "v2-dev", + FixtureBytes: fixtureBytes, + }) + } + } var suitePass, suiteFail int for _, c := range plan.Cases { @@ -223,6 +243,30 @@ func splitCSV(s string) []string { return out } +// gcxVersion calls "gcx --version" once and returns the first line of +// output, or "" if gcx is unreachable. The version contributes to the +// cache key so an upgrade to gcx invalidates all green records. +func gcxVersion(bin string) string { + out, err := exec.Command(bin, "--version").Output() + if err != nil { + return "" + } + return strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0]) +} + +// defaultCacheDir returns $XDG_STATE_HOME/oats or ~/.cache/oats. The cache +// lives under XDG_STATE_HOME on purpose (it is regeneratable state, not +// configuration). Falls back to a relative path if HOME is unset. +func defaultCacheDir() string { + if s := os.Getenv("XDG_STATE_HOME"); s != "" { + return filepath.Join(s, "oats") + } + if h, err := os.UserHomeDir(); err == nil { + return filepath.Join(h, ".cache", "oats") + } + return ".oats-cache" +} + func signalAwareContext() (context.Context, context.CancelFunc) { ctx, cancel := context.WithCancel(context.Background()) sigs := make(chan os.Signal, 1) From 7809e6d97c0ad76b187afb85955231dfcb236693 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:55:17 +0000 Subject: [PATCH 017/124] docs: add V2.md describing the new architecture and CLI V2.md is the v2-branch-only user-facing intro. Once v2 ships, this content folds into README.md and V2.md disappears. While the branch is in flight, keeping it separate avoids confusing v1 users who land on main. Covers: - Quick-start commands for cmd/v2 - Package map mirroring the v2 layer diagram - oats.toml shape and case yaml shape (app + inline-otlp variants) - Assertion vocabulary - What's not in v2-alpha yet (compose/k3d, parallel, migrate) - Verbosity contract Signed-off-by: Gregor Zeitlinger --- V2.md | 167 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 V2.md diff --git a/V2.md b/V2.md new file mode 100644 index 00000000..28be051c --- /dev/null +++ b/V2.md @@ -0,0 +1,167 @@ +# OATS v2 (work in progress) + +This document covers the v2 branch, which replaces the bespoke TraceQL / +PromQL / LogQL HTTP query infrastructure with the [gcx](https://github.com/grafana/gcx) +CLI. The v2 binary lives at `cmd/v2/main.go` while the branch is in flight; +both `oats` and `oats-v2` build from this checkout. See +[grafana/internal-docs#14](https://github.com/grafana/internal-docs/pull/14) +for the full design. + +## Quick start + +```sh +# Build the v2 binary +go build -o bin/oats-v2 ./cmd/v2 + +# Print what would run, do not execute +bin/oats-v2 --config oats.toml --list + +# Run against an already-running stack +bin/oats-v2 --config oats.toml --gcx-context my-lgtm + +# Disable cache for this run +bin/oats-v2 --config oats.toml --no-cache + +# Verbose output +bin/oats-v2 -v # adds per-case PASS lines +bin/oats-v2 -v=2 # adds the gcx command behind each assertion +bin/oats-v2 -v=3 # adds fixture lifecycle and full gcx stdout + +# Machine-readable +bin/oats-v2 --format ndjson > events.jsonl +``` + +## Architecture + +``` +discovery → seed → engine → assert → report + ↑ + fixture (TODO) + ↑ + wait (polling) + ↑ + cache (skip-when-unchanged) +``` + +| Package | Responsibility | +|------------|---------------| +| `discovery` | Parse `oats.toml`, expand case globs, apply filters. | +| `v2case` | Parse and validate one case yaml file. | +| `seed` | Push inline-OTLP payloads at an OTLP/HTTP endpoint. | +| `engine` | Execute a gcx command, capture stdout/stderr/exit. | +| `signalcmd` | Translate a `v2case` assertion into gcx args. | +| `assert` | The expectation vocabulary: `contains`, `not_contains`, `regex`, `value`, `count`, `absent`. | +| `wait` | `Until` / `While` polling primitives (replaces gomega.Eventually). | +| `report` | Compact-text and NDJSON renderers driven by an Event stream. | +| `cache` | Skip-when-unchanged store keyed by `(case yaml + fixture + gcx version + oats version)`. | +| `runner` | Orchestrates a suite: seed → poll-and-assert → report, with optional cache. | +| `cmd/v2` | The new binary entry point. | + +## `oats.toml` shape + +```toml +[meta] +version = 2 + +[[suite]] +name = "lgtm-examples" +cases = ["examples/*/oats.yaml"] +fixture = "lgtm-shared" +tags = ["traces", "metrics", "logs"] + +[fixture.lgtm-shared] +type = "compose" # compose | k3d | remote +template = "lgtm" # built-in compose template +# compose_file = "./my-compose.yml" # alternative to template +# pool_size = 4 # k3d only +# endpoint = "http://..." # remote only + +[cache] +ttl_days = 7 # default +``` + +## Case yaml shape (v2) + +```yaml +oats: 2 +name: rolldice traces have route attribute + +seed: + type: app + compose: docker-compose.app.yml + +expected: + traces: + - traceql: '{ span.http.route = "/rolldice" }' + contains: ["GET /rolldice"] + metrics: + - promql: 'dice_lib_rolls_counter_total{service_name="dice-server"}' + value: '>= 0' + logs: + - logql: '{service_name="dice-server"} |~ `Received request`' + contains: ["Received request to roll dice"] +``` + +Inline-OTLP seed (no example app required): + +```yaml +oats: 2 +name: gcx returns seeded trace + +seed: + type: inline-otlp + traces: + - service: gcx-e2e-seed + spans: + - name: seed-operation + +expected: + traces: + - traceql: '{ resource.service.name = "gcx-e2e-seed" }' + contains: ["gcx-e2e-seed", "seed-operation"] +``` + +### Assertion keys + +Per signal under `expected.[]`: + +| Key | Meaning | +|-----|---------| +| `traceql` / `promql` / `logql` / `query` | The query string. | +| `contains` | Substrings that must appear in gcx stdout. | +| `not_contains` | Substrings that must not appear. | +| `regex` | Patterns that must match. | +| `value` | Metrics only — numeric comparison (`>= 0`, `== 42`). | +| `count` | Comparison against the number of result rows. | +| `absent` | If true, the query must return zero rows. | + +## What's not in v2-alpha yet + +- `compose` and `k3d` fixture types — the runner currently only accepts + `remote` (or no fixture, when you run against an externally-started stack + via `--gcx-context`). Compose lifecycle lands in a follow-up commit. +- Parallel cases within a suite. +- k3d pool warmup. +- `oats migrate` (v1 yaml → v2 yaml). +- Hermeticity static-check (runtime check applies). + +## Migrating from v1 + +For the v1 → v2 migration story see the OATS v2 implementation plan in +[grafana/internal-docs#14](https://github.com/grafana/internal-docs/pull/14). +Until `oats migrate` lands, v1 and v2 yamls coexist — the v1 binary +(`oats`) and the v2 binary (`oats-v2`) read different files. + +## Verbosity contract + +| Flag | Adds to stdout | +|------|----------------| +| (default) | Failures + final summary only. | +| `-v` | One line per passing case. | +| `-v=2` | The gcx invocation behind each assertion. | +| `-v=3` | Fixture lifecycle, full gcx stdout, per-phase timing. | + +`--format ndjson` emits one JSON object per event regardless of verbosity. +Pass and fixture-lifecycle events are filtered at the same verbosity +thresholds. Under GitHub Actions, failures also emit `::error file=...,line=...::` +annotations for inline PR-diff visibility. From 212443619e3f8ed62f325ccd2f304191daf005a6 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 12:55:56 +0000 Subject: [PATCH 018/124] chore: rumdl: add 'text' info string to architecture diagram fence Signed-off-by: Gregor Zeitlinger --- V2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/V2.md b/V2.md index 28be051c..c811d92b 100644 --- a/V2.md +++ b/V2.md @@ -33,7 +33,7 @@ bin/oats-v2 --format ndjson > events.jsonl ## Architecture -``` +```text discovery → seed → engine → assert → report ↑ fixture (TODO) From e379b6b7ea3ea6c5329c876100af48720d133505 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 12 Jun 2026 16:37:20 +0200 Subject: [PATCH 019/124] chore(renovate): track BurntSushi/toml in deps snapshot Auto-added by flint after the discovery package introduced github.com/BurntSushi/toml. Signed-off-by: Gregor Zeitlinger --- .github/renovate-tracked-deps.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/renovate-tracked-deps.json b/.github/renovate-tracked-deps.json index daabe86c..de42130e 100644 --- a/.github/renovate-tracked-deps.json +++ b/.github/renovate-tracked-deps.json @@ -33,6 +33,7 @@ }, "go.mod": { "gomod": [ + "github.com/BurntSushi/toml", "github.com/cenkalti/backoff/v5", "github.com/cespare/xxhash/v2", "github.com/davecgh/go-spew", From 92dc01918104e43af1600af7ef0c45750f39719d Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 13:49:42 +0000 Subject: [PATCH 020/124] feat(v2): add collector-style match assertions Signed-off-by: Gregor Zeitlinger --- assert/assert.go | 96 +++++++++++++++++ assert/assert_test.go | 54 ++++++++++ runner/runner.go | 206 +++++++++++++++++++++++++++++++++--- runner/runner_test.go | 76 +++++++++++++ signalcmd/signalcmd.go | 26 +++-- signalcmd/signalcmd_test.go | 38 +++++++ v2case/case.go | 133 ++++++++++++++++++++++- v2case/case_test.go | 96 +++++++++++++++++ 8 files changed, 698 insertions(+), 27 deletions(-) diff --git a/assert/assert.go b/assert/assert.go index eb224577..ccb5117f 100644 --- a/assert/assert.go +++ b/assert/assert.go @@ -15,6 +15,8 @@ import ( "regexp" "strconv" "strings" + + "github.com/grafana/oats/v2case" ) // Failure carries enough context to render a compact "FAIL " @@ -27,6 +29,15 @@ type Failure struct { func (f Failure) Error() string { return f.Rule + ": " + f.Detail } +// Row is the normalized structural unit used by collector-style `match` +// assertions. Depending on the signal type, Name is the primary field +// (`name` for traces, log body for logs, metric name for metrics) and +// Attributes carries labels/attributes associated with that row. +type Row struct { + Name string + Attributes map[string]string +} + // Contains checks that each substring appears at least once in stdout. func Contains(stdout string, substrings []string) []Failure { var fails []Failure @@ -119,6 +130,91 @@ func Absent(actual int) []Failure { return nil } +// MatchRows checks that each collector-style match entry is satisfied by at +// least one row. Each entry is independent: one entry may match one row and +// the next entry a different row. +func MatchRows(rows []Row, entries []v2case.MatchEntry) []Failure { + var fails []Failure + for _, entry := range entries { + if !anyRowMatches(rows, entry) { + fails = append(fails, Failure{ + Rule: "match", + Detail: fmt.Sprintf("no row matched %s", describeMatch(entry)), + }) + } + } + return fails +} + +func anyRowMatches(rows []Row, entry v2case.MatchEntry) bool { + for _, row := range rows { + if rowMatches(row, entry) { + return true + } + } + return false +} + +func rowMatches(row Row, entry v2case.MatchEntry) bool { + matchType := entry.EffectiveMatchType() + if entry.Name != nil { + if !matchesValue(row.Name, *entry.Name, matchType) { + return false + } + } + for key, expected := range entry.Attributes { + actual, ok := row.Attributes[key] + if expected.Present != nil { + if !ok { + return false + } + continue + } + if !ok || expected.Value == nil { + return false + } + if !matchesValue(actual, *expected.Value, matchType) { + return false + } + } + return true +} + +func matchesValue(actual, expected string, matchType v2case.MatchType) bool { + switch matchType { + case v2case.MatchTypeRegexp: + re, err := regexp.Compile(expected) + if err != nil { + return false + } + return re.MatchString(actual) + default: + return actual == expected + } +} + +func describeMatch(entry v2case.MatchEntry) string { + var parts []string + if entry.MatchType != "" { + parts = append(parts, fmt.Sprintf("match_type=%s", entry.MatchType)) + } + if entry.Name != nil { + parts = append(parts, fmt.Sprintf("name=%q", *entry.Name)) + } + for key, expected := range entry.Attributes { + switch { + case expected.Present != nil: + parts = append(parts, fmt.Sprintf("attribute %s present", key)) + case expected.Value != nil: + parts = append(parts, fmt.Sprintf("attribute %s=%q", key, *expected.Value)) + } + } + if len(parts) == 0 { + return "empty match entry" + } + return strings.Join(parts, ", ") +} + func retag(fails []Failure, rule string) []Failure { for i := range fails { fails[i].Rule = rule diff --git a/assert/assert_test.go b/assert/assert_test.go index 6c1c4adc..630f2fc0 100644 --- a/assert/assert_test.go +++ b/assert/assert_test.go @@ -2,6 +2,8 @@ package assert import ( "testing" + + "github.com/grafana/oats/v2case" ) func TestContains(t *testing.T) { @@ -95,3 +97,55 @@ func TestAbsent(t *testing.T) { t.Errorf("expected one absent-tagged failure, got %v", got) } } + +func TestMatchRows(t *testing.T) { + present := true + rows := []Row{ + { + Name: "seed-operation", + Attributes: map[string]string{ + "service.name": "gcx-e2e-seed", + "trace_id": "abc123", + }, + }, + } + + got := MatchRows(rows, []v2case.MatchEntry{ + { + Name: strPtr("seed-operation"), + Attributes: map[string]v2case.AttributeExpectation{ + "service.name": {Value: strPtr("gcx-e2e-seed")}, + "trace_id": {Present: &present}, + }, + }, + }) + if len(got) != 0 { + t.Fatalf("expected match to pass, got %v", got) + } + + got = MatchRows(rows, []v2case.MatchEntry{ + { + MatchType: v2case.MatchTypeRegexp, + Name: strPtr("^seed-.*$"), + Attributes: map[string]v2case.AttributeExpectation{ + "trace_id": {Value: strPtr("^abc")}, + }, + }, + }) + if len(got) != 0 { + t.Fatalf("expected regexp match to pass, got %v", got) + } + + got = MatchRows(rows, []v2case.MatchEntry{ + { + Attributes: map[string]v2case.AttributeExpectation{ + "missing": {Present: &present}, + }, + }, + }) + if len(got) != 1 || got[0].Rule != "match" { + t.Fatalf("expected one match failure, got %v", got) + } +} + +func strPtr(s string) *string { return &s } diff --git a/runner/runner.go b/runner/runner.go index 19fbe5ba..13eb3d39 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -329,23 +329,34 @@ func (r *Runner) pollAssert( func (r *Runner) runTrace(ctx context.Context, c *v2case.Case, a *v2case.TraceAssertion) bool { args := signalcmd.Traces(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { - return evalCommon(stdout, a.AssertionCommon) + if len(a.Match) == 0 { + return evalCommonText(stdout, a.AssertionCommon) + } + rows, count, err := extractTraceRows(stdout) + return evalCommonStructured(stdout, a.AssertionCommon, rows, count, err) }) } func (r *Runner) runLog(ctx context.Context, c *v2case.Case, a *v2case.LogAssertion) bool { args := signalcmd.Logs(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { - return evalCommon(stdout, a.AssertionCommon) + if len(a.Match) == 0 { + return evalCommonText(stdout, a.AssertionCommon) + } + rows, count, err := extractLogRows(stdout) + return evalCommonStructured(stdout, a.AssertionCommon, rows, count, err) }) } func (r *Runner) runMetric(ctx context.Context, c *v2case.Case, a *v2case.MetricAssertion) bool { args := signalcmd.Metrics(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { - fails := evalCommon(stdout, a.AssertionCommon) + if a.Value == "" && len(a.Match) == 0 { + return evalCommonText(stdout, a.AssertionCommon) + } + rows, count, actual, err := extractMetricRows(stdout) + fails := evalCommonStructured(stdout, a.AssertionCommon, rows, count, err) if a.Value != "" { - actual, err := extractMetricValue(stdout) if err != nil { fails = append(fails, assert.Failure{Rule: "value", Detail: err.Error()}) } else { @@ -359,12 +370,17 @@ func (r *Runner) runMetric(ctx context.Context, c *v2case.Case, a *v2case.Metric func (r *Runner) runProfile(ctx context.Context, c *v2case.Case, a *v2case.ProfileAssertion) bool { args := signalcmd.Profiles(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { - return evalCommon(stdout, a.AssertionCommon) + if len(a.Match) == 0 { + return evalCommonText(stdout, a.AssertionCommon) + } + rows, count, err := extractGenericNamedRows(stdout) + return evalCommonStructured(stdout, a.AssertionCommon, rows, count, err) }) } -// evalCommon runs the assertions that every signal type shares. -func evalCommon(stdout string, c v2case.AssertionCommon) []assert.Failure { +// evalCommonText runs the assertions that every signal type shares when gcx +// output is plain text rather than JSON. +func evalCommonText(stdout string, c v2case.AssertionCommon) []assert.Failure { var fails []assert.Failure fails = append(fails, assert.Contains(stdout, c.Contains)...) fails = append(fails, assert.NotContains(stdout, c.NotContains)...) @@ -378,6 +394,27 @@ func evalCommon(stdout string, c v2case.AssertionCommon) []assert.Failure { return fails } +func evalCommonStructured(stdout string, c v2case.AssertionCommon, rows []assert.Row, count int, parseErr error) []assert.Failure { + var fails []assert.Failure + fails = append(fails, assert.Contains(stdout, c.Contains)...) + fails = append(fails, assert.NotContains(stdout, c.NotContains)...) + fails = append(fails, assert.Regex(stdout, c.Regex)...) + if parseErr != nil { + fails = append(fails, assert.Failure{Rule: "match", Detail: parseErr.Error()}) + return fails + } + if len(c.Match) > 0 { + fails = append(fails, assert.MatchRows(rows, c.Match)...) + } + if c.Count != "" { + fails = append(fails, assert.Count(count, c.Count)...) + } + if c.Absent { + fails = append(fails, assert.Absent(count)...) + } + return fails +} + // approxRowCount counts non-empty, non-banner output lines in gcx text mode. // It is intentionally approximate — gcx's row-counting story will mature as // we use it, and a v2.1 enhancement can swap this for a structured-output @@ -401,20 +438,29 @@ func approxRowCount(stdout string) int { // extractMetricValue parses the first numeric data point out of `gcx metrics // query -o json` output. The schema follows gcx's JSON shape; we only look // at the fields we need so additions don't break us. -func extractMetricValue(stdout string) (float64, error) { +func extractMetricRows(stdout string) ([]assert.Row, int, float64, error) { var generic struct { Data struct { Result []struct { - Value [2]any `json:"value"` - Values [][2]any `json:"values"` + Metric map[string]any `json:"metric"` + Value [2]any `json:"value"` + Values [][2]any `json:"values"` } `json:"result"` } `json:"data"` } if err := json.Unmarshal([]byte(stdout), &generic); err != nil { - return 0, fmt.Errorf("metric value parse: %w", err) + return nil, 0, 0, fmt.Errorf("metric JSON parse: %w", err) } if len(generic.Data.Result) == 0 { - return 0, fmt.Errorf("metric value parse: empty result") + return nil, 0, 0, fmt.Errorf("metric value parse: empty result") + } + rows := make([]assert.Row, 0, len(generic.Data.Result)) + for _, item := range generic.Data.Result { + attrs := stringifyMap(item.Metric) + rows = append(rows, assert.Row{ + Name: attrs["__name__"], + Attributes: attrs, + }) } r := generic.Data.Result[0] raw, ok := r.Value[1].(string) @@ -422,13 +468,13 @@ func extractMetricValue(stdout string) (float64, error) { raw, ok = r.Values[len(r.Values)-1][1].(string) } if !ok { - return 0, fmt.Errorf("metric value parse: result point has no scalar value") + return rows, len(generic.Data.Result), 0, fmt.Errorf("metric value parse: result point has no scalar value") } var f float64 if _, err := fmt.Sscanf(raw, "%f", &f); err != nil { - return 0, fmt.Errorf("metric value parse: %q is not a number", raw) + return rows, len(generic.Data.Result), 0, fmt.Errorf("metric value parse: %q is not a number", raw) } - return f, nil + return rows, len(generic.Data.Result), f, nil } func (r *Runner) failCase(c *v2case.Case, msg, cmd string) { @@ -440,3 +486,133 @@ func (r *Runner) failCase(c *v2case.Case, msg, cmd string) { Cmd: cmd, }) } + +func extractLogRows(stdout string) ([]assert.Row, int, error) { + var generic struct { + Data struct { + Result []struct { + Stream map[string]any `json:"stream"` + Values [][]any `json:"values"` + } `json:"result"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(stdout), &generic); err != nil { + return nil, 0, fmt.Errorf("log JSON parse: %w", err) + } + var rows []assert.Row + for _, stream := range generic.Data.Result { + attrs := stringifyMap(stream.Stream) + for _, pair := range stream.Values { + body := "" + if len(pair) > 1 { + body = fmt.Sprint(pair[1]) + } + rows = append(rows, assert.Row{Name: body, Attributes: attrs}) + } + } + return rows, len(rows), nil +} + +func extractTraceRows(stdout string) ([]assert.Row, int, error) { + var root any + if err := json.Unmarshal([]byte(stdout), &root); err != nil { + return nil, 0, fmt.Errorf("trace JSON parse: %w", err) + } + count := traceResultCount(root) + rows := collectNamedRows(root) + if count == 0 { + count = len(rows) + } + return rows, count, nil +} + +func extractGenericNamedRows(stdout string) ([]assert.Row, int, error) { + var root any + if err := json.Unmarshal([]byte(stdout), &root); err != nil { + return nil, 0, fmt.Errorf("JSON parse: %w", err) + } + rows := collectNamedRows(root) + return rows, len(rows), nil +} + +func traceResultCount(root any) int { + m, ok := root.(map[string]any) + if !ok { + return 0 + } + data, ok := m["data"].(map[string]any) + if !ok { + return 0 + } + result, ok := data["result"].([]any) + if !ok { + return 0 + } + return len(result) +} + +func collectNamedRows(v any) []assert.Row { + var rows []assert.Row + var walk func(any) + walk = func(cur any) { + switch t := cur.(type) { + case map[string]any: + if row, ok := maybeRow(t); ok { + rows = append(rows, row) + } + for _, child := range t { + walk(child) + } + case []any: + for _, child := range t { + walk(child) + } + } + } + walk(v) + return rows +} + +func maybeRow(m map[string]any) (assert.Row, bool) { + row := assert.Row{Attributes: map[string]string{}} + for _, key := range []string{"name", "spanName", "span_name", "body"} { + if v, ok := m[key]; ok { + row.Name = fmt.Sprint(v) + break + } + } + for _, key := range []string{"attributes", "metric", "stream", "resourceAttributes", "resource_attributes"} { + if child, ok := m[key]; ok { + for k, v := range stringifyMapAny(child) { + row.Attributes[k] = v + } + } + } + if resource, ok := m["resource"].(map[string]any); ok { + if attrs, ok := resource["attributes"]; ok { + for k, v := range stringifyMapAny(attrs) { + row.Attributes[k] = v + } + } + } + if row.Name == "" && len(row.Attributes) == 0 { + return assert.Row{}, false + } + return row, true +} + +func stringifyMap(m map[string]any) map[string]string { + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = fmt.Sprint(v) + } + return out +} + +func stringifyMapAny(v any) map[string]string { + m, ok := v.(map[string]any) + if !ok { + return map[string]string{} + } + return stringifyMap(m) +} diff --git a/runner/runner_test.go b/runner/runner_test.go index 68e91e98..5c372032 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -161,6 +161,66 @@ func TestRunCase_MetricsValueFail(t *testing.T) { } } +func TestRunCase_LogsStructuredMatchPass(t *testing.T) { + exec := &stubExec{stdout: `{"status":"success","data":{"resultType":"streams","result":[{"stream":{"service_name":"svc","trace_id":"abc123"},"values":[["1700000000","seed-log-line"]]}]}}`} + r, buf := newRunner(t, exec, Options{Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) + + c := mustParse(t, ` +oats: 2 +name: logs structured match +seed: + type: app + compose: x.yml +expected: + logs: + - logql: '{service_name="svc"}' + match: + - name: seed-log-line + attributes: + service_name: svc + trace_id: + present: true +`) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + + if !ok { + t.Fatalf("expected structured log match to pass:\n%s", buf.String()) + } + if !containsSequence(exec.captured[0], "-o", "json") { + t.Fatalf("expected logs query to request json: %v", exec.captured[0]) + } +} + +func TestRunCase_MetricsStructuredMatchPass(t *testing.T) { + exec := &stubExec{stdout: `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"svc"},"value":[1700000000,"1"]}]}}`} + r, _ := newRunner(t, exec, Options{Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) + + c := mustParse(t, ` +oats: 2 +name: metrics structured match +seed: + type: app + compose: x.yml +expected: + metrics: + - promql: 'up{job="svc"}' + match: + - attributes: + job: svc +`) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + + if !ok { + t.Fatalf("expected structured metric match to pass") + } +} + func TestRunCase_InlineOTLPSeedRequiresEndpoint(t *testing.T) { c := mustParse(t, ` oats: 2 @@ -211,3 +271,19 @@ func TestApproxRowCount(t *testing.T) { } } } + +func containsSequence(haystack []string, needles ...string) bool { + for i := 0; i+len(needles) <= len(haystack); i++ { + match := true + for j, needle := range needles { + if haystack[i+j] != needle { + match = false + break + } + } + if match { + return true + } + } + return false +} diff --git a/signalcmd/signalcmd.go b/signalcmd/signalcmd.go index 35b7150a..07e0de50 100644 --- a/signalcmd/signalcmd.go +++ b/signalcmd/signalcmd.go @@ -26,11 +26,15 @@ func Traces(a v2case.TraceAssertion, since time.Duration) []string { if since <= 0 { since = DefaultSince } - return []string{ + args := []string{ "traces", "search", "--since", since.String(), - a.TraceQL, } + if len(a.Match) > 0 { + args = append(args, "-o", "json") + } + args = append(args, a.TraceQL) + return args } // Logs builds the gcx args for a LogAssertion. @@ -38,11 +42,15 @@ func Logs(a v2case.LogAssertion, since time.Duration) []string { if since <= 0 { since = DefaultSince } - return []string{ + args := []string{ "logs", "query", "--since", since.String(), - a.LogQL, } + if len(a.Match) > 0 { + args = append(args, "-o", "json") + } + args = append(args, a.LogQL) + return args } // Metrics builds the gcx args for a MetricAssertion. When the assertion @@ -57,7 +65,7 @@ func Metrics(a v2case.MetricAssertion, since time.Duration) []string { "metrics", "query", "--since", since.String(), } - if a.Value != "" { + if a.Value != "" || len(a.Match) > 0 { args = append(args, "-o", "json") } args = append(args, a.PromQL) @@ -69,11 +77,15 @@ func Profiles(a v2case.ProfileAssertion, since time.Duration) []string { if since <= 0 { since = DefaultSince } - return []string{ + args := []string{ "profiles", "query", "--since", since.String(), - a.Query, } + if len(a.Match) > 0 { + args = append(args, "-o", "json") + } + args = append(args, a.Query) + return args } // Render is a convenience used by the report layer to show the gcx diff --git a/signalcmd/signalcmd_test.go b/signalcmd/signalcmd_test.go index d82e6fca..7db0a018 100644 --- a/signalcmd/signalcmd_test.go +++ b/signalcmd/signalcmd_test.go @@ -16,6 +16,18 @@ func TestTraces(t *testing.T) { } } +func TestTraces_WithMatchAsksForJSON(t *testing.T) { + got := Traces(v2case.TraceAssertion{ + TraceQL: `{ span.http.route = "/x" }`, + AssertionCommon: v2case.AssertionCommon{ + Match: []v2case.MatchEntry{{Name: strPtr("GET /x")}}, + }, + }, 0) + if !contains(got, "-o", "json") { + t.Errorf("expected -o json in: %v", got) + } +} + func TestLogs(t *testing.T) { got := Logs(v2case.LogAssertion{LogQL: `{service_name="x"}`}, 5*time.Minute) want := []string{"logs", "query", "--since", "5m0s", `{service_name="x"}`} @@ -24,6 +36,18 @@ func TestLogs(t *testing.T) { } } +func TestLogs_WithMatchAsksForJSON(t *testing.T) { + got := Logs(v2case.LogAssertion{ + LogQL: `{service_name="x"}`, + AssertionCommon: v2case.AssertionCommon{ + Match: []v2case.MatchEntry{{Name: strPtr("line")}}, + }, + }, 5*time.Minute) + if !contains(got, "-o", "json") { + t.Errorf("expected -o json in: %v", got) + } +} + func TestMetrics_PromQLOnly(t *testing.T) { got := Metrics(v2case.MetricAssertion{PromQL: "up"}, time.Minute) want := []string{"metrics", "query", "--since", "1m0s", "up"} @@ -50,6 +74,18 @@ func TestProfiles(t *testing.T) { } } +func TestProfiles_WithMatchAsksForJSON(t *testing.T) { + got := Profiles(v2case.ProfileAssertion{ + Query: "process_cpu:cpu:nanoseconds:cpu:nanoseconds{}", + AssertionCommon: v2case.AssertionCommon{ + Match: []v2case.MatchEntry{{Name: strPtr("main")}}, + }, + }, 0) + if !contains(got, "-o", "json") { + t.Errorf("expected -o json in: %v", got) + } +} + func TestRender_QuotesSpecialChars(t *testing.T) { args := []string{"traces", "search", `{ span.http.route = "/x" }`} rendered := Render(args) @@ -101,3 +137,5 @@ func contains(haystack []string, needles ...string) bool { } return false } + +func strPtr(s string) *string { return &s } diff --git a/v2case/case.go b/v2case/case.go index 8ed3948b..b832a236 100644 --- a/v2case/case.go +++ b/v2case/case.go @@ -14,6 +14,7 @@ import ( "bytes" "fmt" "os" + "regexp" "go.yaml.in/yaml/v3" ) @@ -89,11 +90,59 @@ type Expected struct { // Embedded into each concrete assertion struct so a case author can mix and // match contains / regex / count / absent on any signal. type AssertionCommon struct { - Contains []string `yaml:"contains,omitempty"` - NotContains []string `yaml:"not_contains,omitempty"` - Regex []string `yaml:"regex,omitempty"` - Count string `yaml:"count,omitempty"` // ">= 1", "== 0", ... - Absent bool `yaml:"absent,omitempty"` + Contains []string `yaml:"contains,omitempty"` + NotContains []string `yaml:"not_contains,omitempty"` + Regex []string `yaml:"regex,omitempty"` + Match []MatchEntry `yaml:"match,omitempty"` + Count string `yaml:"count,omitempty"` // ">= 1", "== 0", ... + Absent bool `yaml:"absent,omitempty"` +} + +type MatchType string + +const ( + MatchTypeStrict MatchType = "strict" + MatchTypeRegexp MatchType = "regexp" +) + +type MatchEntry struct { + MatchType MatchType `yaml:"match_type,omitempty"` + Name *string `yaml:"name,omitempty"` + Attributes map[string]AttributeExpectation `yaml:"attributes,omitempty"` +} + +func (m MatchEntry) EffectiveMatchType() MatchType { + if m.MatchType == "" { + return MatchTypeStrict + } + return m.MatchType +} + +type AttributeExpectation struct { + Value *string + Present *bool +} + +func (a *AttributeExpectation) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + v := node.Value + a.Value = &v + a.Present = nil + return nil + case yaml.MappingNode: + var aux struct { + Present *bool `yaml:"present"` + } + if err := node.Decode(&aux); err != nil { + return err + } + a.Value = nil + a.Present = aux.Present + return nil + default: + return fmt.Errorf("expected scalar string or mapping {present: true}") + } } type TraceAssertion struct { @@ -177,6 +226,38 @@ func (c *Case) Validate() error { if len(c.Expected.Traces)+len(c.Expected.Metrics)+len(c.Expected.Logs)+len(c.Expected.Profiles) == 0 { return fmt.Errorf("expected: at least one signal assertion required (a case with no expectations cannot fail)") } + for i := range c.Expected.Traces { + if c.Expected.Traces[i].TraceQL == "" { + return fmt.Errorf("expected.traces[%d].traceql: required, non-empty", i) + } + if err := validateAssertionCommon("expected.traces", i, c.Expected.Traces[i].AssertionCommon); err != nil { + return err + } + } + for i := range c.Expected.Logs { + if c.Expected.Logs[i].LogQL == "" { + return fmt.Errorf("expected.logs[%d].logql: required, non-empty", i) + } + if err := validateAssertionCommon("expected.logs", i, c.Expected.Logs[i].AssertionCommon); err != nil { + return err + } + } + for i := range c.Expected.Metrics { + if c.Expected.Metrics[i].PromQL == "" { + return fmt.Errorf("expected.metrics[%d].promql: required, non-empty", i) + } + if err := validateAssertionCommon("expected.metrics", i, c.Expected.Metrics[i].AssertionCommon); err != nil { + return err + } + } + for i := range c.Expected.Profiles { + if c.Expected.Profiles[i].Query == "" { + return fmt.Errorf("expected.profiles[%d].query: required, non-empty", i) + } + if err := validateAssertionCommon("expected.profiles", i, c.Expected.Profiles[i].AssertionCommon); err != nil { + return err + } + } return nil } @@ -189,3 +270,45 @@ func (c *Case) IsHermetic() bool { } return *c.Hermetic } + +func validateAssertionCommon(path string, idx int, a AssertionCommon) error { + for j, p := range a.Regex { + if _, err := regexp.Compile(p); err != nil { + return fmt.Errorf("%s[%d].regex[%d]: invalid regexp %q: %v", path, idx, j, p, err) + } + } + for j, m := range a.Match { + matchPath := fmt.Sprintf("%s[%d].match[%d]", path, idx, j) + switch m.EffectiveMatchType() { + case MatchTypeStrict, MatchTypeRegexp: + default: + return fmt.Errorf("%s.match_type: unknown value %q (expected strict or regexp)", matchPath, m.MatchType) + } + if m.Name == nil && len(m.Attributes) == 0 { + return fmt.Errorf("%s: at least one of name or attributes is required", matchPath) + } + if m.EffectiveMatchType() == MatchTypeRegexp && m.Name != nil { + if _, err := regexp.Compile(*m.Name); err != nil { + return fmt.Errorf("%s.name: invalid regexp %q: %v", matchPath, *m.Name, err) + } + } + for key, attr := range m.Attributes { + attrPath := fmt.Sprintf("%s.attributes[%q]", matchPath, key) + if attr.Value != nil && attr.Present != nil { + return fmt.Errorf("%s: expected either scalar value or {present: true}, not both", attrPath) + } + if attr.Value == nil && attr.Present == nil { + return fmt.Errorf("%s: expected scalar value or {present: true}", attrPath) + } + if attr.Present != nil && !*attr.Present { + return fmt.Errorf("%s.present: only true is allowed", attrPath) + } + if m.EffectiveMatchType() == MatchTypeRegexp && attr.Value != nil { + if _, err := regexp.Compile(*attr.Value); err != nil { + return fmt.Errorf("%s: invalid regexp %q: %v", attrPath, *attr.Value, err) + } + } + } + } + return nil +} diff --git a/v2case/case_test.go b/v2case/case_test.go index 05bfe16e..21b72753 100644 --- a/v2case/case_test.go +++ b/v2case/case_test.go @@ -65,6 +65,44 @@ expected: } } +func TestParse_MatchAssertions(t *testing.T) { + src := []byte(` +oats: 2 +name: structured match +seed: + type: app + compose: x.yml +expected: + logs: + - logql: '{service_name="svc"}' + match: + - name: "seed-log-line" + attributes: + service_name: svc + trace_id: + present: true + - match_type: regexp + attributes: + level: "info|warn" +`) + c, err := Parse(src) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if got := len(c.Expected.Logs[0].Match); got != 2 { + t.Fatalf("match entries: got %d, want 2", got) + } + if c.Expected.Logs[0].Match[0].EffectiveMatchType() != MatchTypeStrict { + t.Fatalf("default match_type should be strict") + } + if c.Expected.Logs[0].Match[0].Attributes["trace_id"].Present == nil { + t.Fatalf("trace_id.present should be set") + } + if c.Expected.Logs[0].Match[1].EffectiveMatchType() != MatchTypeRegexp { + t.Fatalf("second entry should be regexp") + } +} + func TestParse_RejectsUnknownFields(t *testing.T) { src := []byte(` oats: 2 @@ -132,6 +170,64 @@ func TestValidate_NoExpectations(t *testing.T) { } } +func TestValidate_RejectsUnknownMatchType(t *testing.T) { + _, err := Parse([]byte(` +oats: 2 +name: bad match type +seed: + type: app + compose: x.yml +expected: + traces: + - traceql: '{}' + match: + - match_type: glob + name: x +`)) + if err == nil || !strings.Contains(err.Error(), "match_type") { + t.Fatalf("expected match_type error, got %v", err) + } +} + +func TestValidate_RejectsPresentFalse(t *testing.T) { + _, err := Parse([]byte(` +oats: 2 +name: bad present +seed: + type: app + compose: x.yml +expected: + logs: + - logql: '{job="x"}' + match: + - attributes: + trace_id: + present: false +`)) + if err == nil || !strings.Contains(err.Error(), "only true is allowed") { + t.Fatalf("expected present error, got %v", err) + } +} + +func TestValidate_RejectsInvalidMatchRegexp(t *testing.T) { + _, err := Parse([]byte(` +oats: 2 +name: bad regexp +seed: + type: app + compose: x.yml +expected: + traces: + - traceql: '{}' + match: + - match_type: regexp + name: '[' +`)) + if err == nil || !strings.Contains(err.Error(), "invalid regexp") { + t.Fatalf("expected regexp error, got %v", err) + } +} + func TestIsHermetic(t *testing.T) { c := &Case{} if !c.IsHermetic() { From 25736b3dba838b4efa2455c5f7558846582a4f1f Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 13:56:26 +0000 Subject: [PATCH 021/124] test(v2): exercise match assertions end to end Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 18 ++++++++++++++++-- cmd/v2/testdata/fake-gcx.sh | 25 ++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 39b00721..8f2891d0 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -61,13 +61,27 @@ seed: expected: traces: - traceql: '{ resource.service.name = "gcx-e2e-seed" }' - contains: ["gcx-e2e-seed", "seed-operation"] + match: + - name: seed-operation + attributes: + service.name: gcx-e2e-seed + trace_id: + present: true logs: - logql: '{service_name="gcx-e2e-seed"}' - contains: ["seed-log-line"] + match: + - name: seed-log-line + attributes: + service_name: gcx-e2e-seed + trace_id: + present: true metrics: - promql: 'seed_counter_total{service_name="gcx-e2e-seed"}' value: ">= 0" + match: + - name: seed_counter_total + attributes: + service_name: gcx-e2e-seed `) // OTLP stub: accept any POST under /v1/* with 200. diff --git a/cmd/v2/testdata/fake-gcx.sh b/cmd/v2/testdata/fake-gcx.sh index c83afad8..d63e30ad 100755 --- a/cmd/v2/testdata/fake-gcx.sh +++ b/cmd/v2/testdata/fake-gcx.sh @@ -13,23 +13,42 @@ if [[ "${1:-}" == "--context" ]]; then shift 2 fi +json=false +for arg in "$@"; do + if [[ "$arg" == "-o" || "$arg" == "--output" ]]; then + json=true + fi +done + case "${1:-}.${2:-}" in traces.search) - cat <<'EOF' + if [[ "$json" == true ]]; then + cat <<'EOF' +{"status":"success","data":{"result":[{"name":"seed-operation","attributes":{"service.name":"gcx-e2e-seed","trace_id":"abc123def456"}}]}} +EOF + else + cat <<'EOF' Trace IDs Service Span abc123def456 gcx-e2e-seed seed-operation EOF + fi ;; logs.query) - cat <<'EOF' + if [[ "$json" == true ]]; then + cat <<'EOF' +{"status":"success","data":{"resultType":"streams","result":[{"stream":{"service_name":"gcx-e2e-seed","trace_id":"abc123def456"},"values":[["1700000000000000000","seed-log-line"]]}]}} +EOF + else + cat <<'EOF' time service body 2026-06-12T09:00:00Z gcx-e2e-seed seed-log-line EOF + fi ;; metrics.query) # Static JSON shaped like Prometheus instant query output. cat <<'EOF' -{"status":"success","data":{"resultType":"vector","result":[{"metric":{"service_name":"gcx-e2e-seed"},"value":[1700000000,"42"]}]}} +{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"seed_counter_total","service_name":"gcx-e2e-seed"},"value":[1700000000,"42"]}]}} EOF ;; *) From 3924924f6f2dd7906b437032e69ea8198b59a105 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:01:05 +0000 Subject: [PATCH 022/124] feat(v2): parse OTLP traces and flamebearer matches Signed-off-by: Gregor Zeitlinger --- V2.md | 32 ++++++++- runner/runner.go | 155 +++++++++++++++++++++++++++++++++++++++++- runner/runner_test.go | 38 +++++++++++ 3 files changed, 219 insertions(+), 6 deletions(-) diff --git a/V2.md b/V2.md index c811d92b..f55535da 100644 --- a/V2.md +++ b/V2.md @@ -93,13 +93,20 @@ seed: expected: traces: - traceql: '{ span.http.route = "/rolldice" }' - contains: ["GET /rolldice"] + match: + - name: "GET /rolldice" metrics: - promql: 'dice_lib_rolls_counter_total{service_name="dice-server"}' value: '>= 0' + match: + - attributes: + service_name: dice-server logs: - logql: '{service_name="dice-server"} |~ `Received request`' - contains: ["Received request to roll dice"] + match: + - name: "Received request to roll dice" + attributes: + service_name: dice-server ``` Inline-OTLP seed (no example app required): @@ -118,7 +125,10 @@ seed: expected: traces: - traceql: '{ resource.service.name = "gcx-e2e-seed" }' - contains: ["gcx-e2e-seed", "seed-operation"] + match: + - name: seed-operation + attributes: + service.name: gcx-e2e-seed ``` ### Assertion keys @@ -131,10 +141,26 @@ Per signal under `expected.[]`: | `contains` | Substrings that must appear in gcx stdout. | | `not_contains` | Substrings that must not appear. | | `regex` | Patterns that must match. | +| `match` | Structural assertions using collector-style `match_type: strict | regexp`. | | `value` | Metrics only — numeric comparison (`>= 0`, `== 42`). | | `count` | Comparison against the number of result rows. | | `absent` | If true, the query must return zero rows. | +`match` entries default to `match_type: strict`. OATS also supports the +small convenience extension `present: true` for attribute existence checks: + +```yaml +match: + - name: seed-operation + attributes: + service.name: gcx-e2e-seed + trace_id: + present: true + - match_type: regexp + attributes: + http.route: "^/roll.*$" +``` + ## What's not in v2-alpha yet - `compose` and `k3d` fixture types — the runner currently only accepts diff --git a/runner/runner.go b/runner/runner.go index 13eb3d39..ac3a46be 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -373,7 +373,7 @@ func (r *Runner) runProfile(ctx context.Context, c *v2case.Case, a *v2case.Profi if len(a.Match) == 0 { return evalCommonText(stdout, a.AssertionCommon) } - rows, count, err := extractGenericNamedRows(stdout) + rows, count, err := extractProfileRows(stdout) return evalCommonStructured(stdout, a.AssertionCommon, rows, count, err) }) } @@ -518,6 +518,9 @@ func extractTraceRows(stdout string) ([]assert.Row, int, error) { if err := json.Unmarshal([]byte(stdout), &root); err != nil { return nil, 0, fmt.Errorf("trace JSON parse: %w", err) } + if rows, ok := extractOTLPTraceRows(root); ok { + return rows, len(rows), nil + } count := traceResultCount(root) rows := collectNamedRows(root) if count == 0 { @@ -526,10 +529,17 @@ func extractTraceRows(stdout string) ([]assert.Row, int, error) { return rows, count, nil } -func extractGenericNamedRows(stdout string) ([]assert.Row, int, error) { +func extractProfileRows(stdout string) ([]assert.Row, int, error) { var root any if err := json.Unmarshal([]byte(stdout), &root); err != nil { - return nil, 0, fmt.Errorf("JSON parse: %w", err) + return nil, 0, fmt.Errorf("profile JSON parse: %w", err) + } + if names := flamebearerNames(root); len(names) > 0 { + rows := make([]assert.Row, 0, len(names)) + for _, name := range names { + rows = append(rows, assert.Row{Name: name, Attributes: map[string]string{}}) + } + return rows, len(rows), nil } rows := collectNamedRows(root) return rows, len(rows), nil @@ -616,3 +626,142 @@ func stringifyMapAny(v any) map[string]string { } return stringifyMap(m) } + +func extractOTLPTraceRows(root any) ([]assert.Row, bool) { + top, ok := root.(map[string]any) + if !ok { + return nil, false + } + resourceSpans, ok := top["resourceSpans"].([]any) + if !ok { + resourceSpans, ok = top["batches"].([]any) + } + if !ok { + // Some wrappers may nest under "data" first. + if data, ok := top["data"].(map[string]any); ok { + resourceSpans, ok = data["resourceSpans"].([]any) + if !ok { + resourceSpans, ok = data["batches"].([]any) + } + } + if !ok { + return nil, false + } + } + var rows []assert.Row + for _, rsAny := range resourceSpans { + rs, ok := rsAny.(map[string]any) + if !ok { + continue + } + resourceAttrs := map[string]string{} + if resource, ok := rs["resource"].(map[string]any); ok { + resourceAttrs = parseOTelAttributeList(resource["attributes"]) + } + scopeSpans, _ := rs["scopeSpans"].([]any) + for _, ssAny := range scopeSpans { + ss, ok := ssAny.(map[string]any) + if !ok { + continue + } + scopeName := "" + if scope, ok := ss["scope"].(map[string]any); ok { + scopeName = fmt.Sprint(scope["name"]) + } + spans, _ := ss["spans"].([]any) + for _, spAny := range spans { + sp, ok := spAny.(map[string]any) + if !ok { + continue + } + attrs := map[string]string{} + for k, v := range resourceAttrs { + attrs[k] = v + } + for k, v := range parseOTelAttributeList(sp["attributes"]) { + attrs[k] = v + } + if scopeName != "" { + attrs["otel.scope.name"] = scopeName + } + if kind, ok := sp["kind"]; ok { + attrs["kind"] = fmt.Sprint(kind) + } + rows = append(rows, assert.Row{ + Name: fmt.Sprint(sp["name"]), + Attributes: attrs, + }) + } + } + } + if len(rows) == 0 { + return nil, false + } + return rows, true +} + +func parseOTelAttributeList(v any) map[string]string { + list, ok := v.([]any) + if !ok { + return map[string]string{} + } + out := make(map[string]string, len(list)) + for _, itemAny := range list { + item, ok := itemAny.(map[string]any) + if !ok { + continue + } + key := fmt.Sprint(item["key"]) + if key == "" { + continue + } + value, _ := item["value"].(map[string]any) + out[key] = parseOTelAnyValue(value) + } + return out +} + +func parseOTelAnyValue(m map[string]any) string { + for _, key := range []string{"stringValue", "intValue", "doubleValue", "boolValue"} { + if v, ok := m[key]; ok { + return fmt.Sprint(v) + } + } + if arr, ok := m["arrayValue"].(map[string]any); ok { + if vals, ok := arr["values"].([]any); ok { + parts := make([]string, 0, len(vals)) + for _, val := range vals { + if child, ok := val.(map[string]any); ok { + parts = append(parts, parseOTelAnyValue(child)) + } + } + return strings.Join(parts, ",") + } + } + return "" +} + +func flamebearerNames(root any) []string { + top, ok := root.(map[string]any) + if !ok { + return nil + } + flamebearer, ok := top["flamebearer"].(map[string]any) + if !ok { + if data, ok := top["data"].(map[string]any); ok { + flamebearer, _ = data["flamebearer"].(map[string]any) + } + } + if flamebearer == nil { + return nil + } + raw, ok := flamebearer["names"].([]any) + if !ok { + return nil + } + out := make([]string, 0, len(raw)) + for _, item := range raw { + out = append(out, fmt.Sprint(item)) + } + return out +} diff --git a/runner/runner_test.go b/runner/runner_test.go index 5c372032..1514c10c 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -3,6 +3,7 @@ package runner import ( "bytes" "context" + "os" "strings" "testing" "time" @@ -272,6 +273,43 @@ func TestApproxRowCount(t *testing.T) { } } +func TestExtractTraceRows_OTLPShape(t *testing.T) { + data, err := os.ReadFile("/home/gregor/source/oats-v2/testhelpers/tempo/responses/testdata/trace_by_id.json") + if err != nil { + t.Fatalf("ReadFile: %v", err) + } + rows, count, parseErr := extractTraceRows(string(data)) + if parseErr != nil { + t.Fatalf("extractTraceRows: %v", parseErr) + } + if count == 0 || len(rows) == 0 { + t.Fatalf("expected OTLP trace rows, got count=%d rows=%d", count, len(rows)) + } + found := false + for _, row := range rows { + if row.Name == "GET /stock" && row.Attributes["http.route"] == "/stock" { + found = true + break + } + } + if !found { + t.Fatalf("expected extracted span row for GET /stock with http.route=/stock") + } +} + +func TestExtractProfileRows_FlamebearerShape(t *testing.T) { + rows, count, err := extractProfileRows(`{"flamebearer":{"names":["main","worker"]}}`) + if err != nil { + t.Fatalf("extractProfileRows: %v", err) + } + if count != 2 || len(rows) != 2 { + t.Fatalf("expected 2 rows, got count=%d rows=%d", count, len(rows)) + } + if rows[0].Name != "main" || rows[1].Name != "worker" { + t.Fatalf("unexpected rows: %+v", rows) + } +} + func containsSequence(haystack []string, needles ...string) bool { for i := 0; i+len(needles) <= len(haystack); i++ { match := true From 9d4798b0a15c292a10fbed36f41ed28536cf4f16 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:05:42 +0000 Subject: [PATCH 023/124] feat(v2): add best-effort legacy migration Signed-off-by: Gregor Zeitlinger --- V2.md | 18 +++- cmd/v2/main.go | 15 ++++ migrate/migrate.go | 187 ++++++++++++++++++++++++++++++++++++++++ migrate/migrate_test.go | 92 ++++++++++++++++++++ v2case/case.go | 13 +++ yaml/testcase.go | 7 ++ 6 files changed, 329 insertions(+), 3 deletions(-) create mode 100644 migrate/migrate.go create mode 100644 migrate/migrate_test.go diff --git a/V2.md b/V2.md index f55535da..04a8de15 100644 --- a/V2.md +++ b/V2.md @@ -168,15 +168,27 @@ match: via `--gcx-context`). Compose lifecycle lands in a follow-up commit. - Parallel cases within a suite. - k3d pool warmup. -- `oats migrate` (v1 yaml → v2 yaml). +- Full-fidelity `oats migrate` for fixture/input semantics. A best-effort + `oats-v2 --migrate ` now converts expectation blocks into the + collector-style `match` schema and prints warnings for dropped/unsupported + fields. - Hermeticity static-check (runtime check applies). ## Migrating from v1 For the v1 → v2 migration story see the OATS v2 implementation plan in [grafana/internal-docs#14](https://github.com/grafana/internal-docs/pull/14). -Until `oats migrate` lands, v1 and v2 yamls coexist — the v1 binary -(`oats`) and the v2 binary (`oats-v2`) read different files. +Today a best-effort converter exists: + +```bash +oats-v2 --migrate path/to/oats.yaml > migrated.yaml +``` + +It converts legacy `equals` / `regexp` / `attributes` / +`attribute-regexp` assertions into v2 `match:` entries and prints warnings +for fields that still need manual follow-up (for example `input`, matrix, +or multi-file docker-compose details). The v1 binary (`oats`) and the v2 +binary (`oats-v2`) still read different file shapes. ## Verbosity contract diff --git a/cmd/v2/main.go b/cmd/v2/main.go index 811cace9..d32d859f 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -36,6 +36,7 @@ import ( "github.com/grafana/oats/cache" "github.com/grafana/oats/discovery" "github.com/grafana/oats/engine" + "github.com/grafana/oats/migrate" "github.com/grafana/oats/report" "github.com/grafana/oats/runner" ) @@ -49,6 +50,7 @@ func run() int { configPath := flag.String("config", "oats.toml", "path to oats.toml") gcxBin := flag.String("gcx", "gcx", "path to gcx binary (PATH-resolved if a bare name)") listOnly := flag.Bool("list", false, "print the run plan and exit (no execution)") + migratePath := flag.String("migrate", "", "convert one legacy OATS yaml file to v2 and print the result to stdout") format := flag.String("format", "text", "output format: text | ndjson") suiteFilterStr := flag.String("suite", "", "comma-separated suite names") tagFilterStr := flag.String("tags", "", "comma-separated tag any-match") @@ -65,6 +67,19 @@ func run() int { flag.Parse() + if *migratePath != "" { + out, warnings, err := migrate.ConvertFile(*migratePath) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 2 + } + for _, w := range warnings { + fmt.Fprintln(os.Stderr, "migrate warning:", w) + } + fmt.Print(string(out)) + return 0 + } + cfg, err := discovery.Load(*configPath) if err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/migrate/migrate.go b/migrate/migrate.go new file mode 100644 index 00000000..2a6adcea --- /dev/null +++ b/migrate/migrate.go @@ -0,0 +1,187 @@ +package migrate + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/grafana/oats/model" + "github.com/grafana/oats/v2case" + legacyyaml "github.com/grafana/oats/yaml" + goyaml "go.yaml.in/yaml/v3" +) + +// ConvertFile reads one legacy OATS yaml file and returns a best-effort v2 +// case yaml plus any warnings about dropped or lossy fields. +func ConvertFile(path string) ([]byte, []string, error) { + def, err := legacyyaml.LoadTestCaseDefinition(path) + if err != nil { + return nil, nil, err + } + if def == nil { + return nil, nil, fmt.Errorf("%s is not a legacy OATS test case definition", path) + } + c, warnings, err := ConvertDefinition(*def, deriveName(path)) + if err != nil { + return nil, warnings, err + } + out, err := goyaml.Marshal(c) + if err != nil { + return nil, warnings, err + } + return out, warnings, nil +} + +// ConvertDefinition maps a legacy v1/v1.5-style OATS definition into the +// current v2 case shape. Unsupported fields are dropped with warnings. +func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, []string, error) { + var warnings []string + c := &v2case.Case{ + OatsVersion: v2case.SchemaVersion, + Name: name, + } + + if def.Kubernetes != nil { + return nil, warnings, fmt.Errorf("kubernetes fixtures are not yet supported by v2 migration") + } + if len(def.Matrix) > 0 { + warnings = append(warnings, "matrix definitions are not migrated; convert expanded matrix cases manually") + } + if len(def.Include) > 0 { + warnings = append(warnings, "include directives were resolved before migration; output is a flattened case") + } + if len(def.Input) > 0 { + warnings = append(warnings, "input requests are not represented in v2 yet; dropped from migrated output") + } + if def.Interval != 0 { + warnings = append(warnings, "interval is not represented in v2 yet; dropped from migrated output") + } + if len(def.Expected.ComposeLogs) > 0 { + warnings = append(warnings, "compose-logs assertions are not migrated") + } + if len(def.Expected.CustomChecks) > 0 { + warnings = append(warnings, "custom-checks are not migrated") + } + + if def.DockerCompose != nil { + c.Seed.Type = "app" + if len(def.DockerCompose.Files) == 0 { + return nil, warnings, fmt.Errorf("docker-compose present but no files declared") + } + c.Seed.Compose = def.DockerCompose.Files[0] + if len(def.DockerCompose.Files) > 1 { + warnings = append(warnings, fmt.Sprintf("multiple docker-compose files collapsed to first entry %q", def.DockerCompose.Files[0])) + } + if len(def.DockerCompose.Environment) > 0 { + warnings = append(warnings, "docker-compose env overrides are not represented in v2 yet") + } + } else { + warnings = append(warnings, "no docker-compose fixture found; defaulting seed.type to inline-otlp placeholder") + c.Seed.Type = "inline-otlp" + c.Seed.Traces = []v2case.SeedTrace{{Service: "migrated-service", Spans: []v2case.SeedSpan{{Name: "replace-me"}}}} + } + + for _, tr := range def.Expected.Traces { + assertion, ws := convertSignal(tr.TraceQL, tr.Signal) + warnings = append(warnings, ws...) + c.Expected.Traces = append(c.Expected.Traces, v2case.TraceAssertion{TraceQL: tr.TraceQL, AssertionCommon: assertion}) + } + for _, lg := range def.Expected.Logs { + assertion, ws := convertSignal(lg.LogQL, lg.Signal) + warnings = append(warnings, ws...) + c.Expected.Logs = append(c.Expected.Logs, v2case.LogAssertion{LogQL: lg.LogQL, AssertionCommon: assertion}) + } + for _, m := range def.Expected.Metrics { + c.Expected.Metrics = append(c.Expected.Metrics, v2case.MetricAssertion{PromQL: m.PromQL, Value: m.Value}) + if m.MatrixCondition != "" { + warnings = append(warnings, fmt.Sprintf("metric %q matrix-condition dropped", m.PromQL)) + } + } + for _, p := range def.Expected.Profiles { + var matches []v2case.MatchEntry + if p.Flamebearers.NameEquals != "" { + matches = append(matches, v2case.MatchEntry{Name: strPtr(p.Flamebearers.NameEquals)}) + } + if p.Flamebearers.NameRegexp != "" { + matches = append(matches, v2case.MatchEntry{MatchType: v2case.MatchTypeRegexp, Name: strPtr(p.Flamebearers.NameRegexp)}) + } + c.Expected.Profiles = append(c.Expected.Profiles, v2case.ProfileAssertion{ + Query: p.Query, + AssertionCommon: v2case.AssertionCommon{Match: matches}, + }) + if p.MatrixCondition != "" { + warnings = append(warnings, fmt.Sprintf("profile %q matrix-condition dropped", p.Query)) + } + } + + if err := c.Validate(); err != nil { + return nil, warnings, fmt.Errorf("migrated v2 case failed validation: %w", err) + } + return c, warnings, nil +} + +func convertSignal(label string, s model.ExpectedSignal) (v2case.AssertionCommon, []string) { + var warnings []string + out := v2case.AssertionCommon{} + if s.Count != nil { + switch { + case s.Count.Min == 0 && s.Count.Max == 0: + out.Absent = true + case s.Count.Max > 0 && s.Count.Min > 0 && s.Count.Max != s.Count.Min: + out.Count = fmt.Sprintf(">= %d", s.Count.Min) + warnings = append(warnings, fmt.Sprintf("%s count max=%d dropped; v2 scalar count keeps only min bound", label, s.Count.Max)) + case s.Count.Max > 0 && s.Count.Min == s.Count.Max: + out.Count = fmt.Sprintf("== %d", s.Count.Min) + case s.Count.Min > 0: + out.Count = fmt.Sprintf(">= %d", s.Count.Min) + case s.Count.Max > 0: + out.Count = fmt.Sprintf("<= %d", s.Count.Max) + } + } + if s.NoExtraAttributes { + warnings = append(warnings, fmt.Sprintf("%s no-extra-attributes is not supported in v2 and was dropped", label)) + } + if s.MatrixCondition != "" { + warnings = append(warnings, fmt.Sprintf("%s matrix-condition dropped", label)) + } + if s.NameEquals != "" || len(s.Attributes) > 0 { + entry := v2case.MatchEntry{Attributes: map[string]v2case.AttributeExpectation{}} + if s.NameEquals != "" { + entry.Name = strPtr(s.NameEquals) + } + for k, v := range s.Attributes { + entry.Attributes[k] = v2case.AttributeExpectation{Value: strPtr(v)} + } + out.Match = append(out.Match, entry) + } + if s.NameRegexp != "" || len(s.AttributeRegexp) > 0 { + entry := v2case.MatchEntry{MatchType: v2case.MatchTypeRegexp, Attributes: map[string]v2case.AttributeExpectation{}} + if s.NameRegexp != "" { + entry.Name = strPtr(s.NameRegexp) + } + for k, v := range s.AttributeRegexp { + if v == ".*" { + present := true + entry.Attributes[k] = v2case.AttributeExpectation{Present: &present} + } else { + entry.Attributes[k] = v2case.AttributeExpectation{Value: strPtr(v)} + } + } + if entry.Name != nil || len(entry.Attributes) > 0 { + out.Match = append(out.Match, entry) + } + } + if len(out.Match) == 0 && !out.Absent && out.Count == "" { + warnings = append(warnings, fmt.Sprintf("%s has no structural checks after migration", label)) + } + return out, warnings +} + +func deriveName(path string) string { + base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) + base = strings.ReplaceAll(base, "_", " ") + base = strings.ReplaceAll(base, "-", " ") + return strings.TrimSpace(base) +} + +func strPtr(s string) *string { return &s } diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go new file mode 100644 index 00000000..d47342f0 --- /dev/null +++ b/migrate/migrate_test.go @@ -0,0 +1,92 @@ +package migrate + +import ( + "strings" + "testing" + "time" + + "github.com/grafana/oats/model" +) + +func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { + def := model.TestCaseDefinition{ + DockerCompose: &model.DockerCompose{Files: []string{"docker-compose.yml"}}, + Input: []model.Input{{Path: "/stock"}}, + Interval: 500 * time.Millisecond, + Expected: model.Expected{ + Traces: []model.ExpectedTraces{{ + TraceQL: `{ name = "GET /stock" }`, + Signal: model.ExpectedSignal{ + NameEquals: "GET /stock", + Attributes: map[string]string{"db.system": "h2"}, + AttributeRegexp: map[string]string{"trace_id": ".*"}, + }, + }}, + Logs: []model.ExpectedLogs{{ + LogQL: `{service_name="shop"}`, + Signal: model.ExpectedSignal{ + NameRegexp: `error|warn`, + Count: &model.ExpectedRange{Min: 1, Max: 3}, + }, + }}, + Profiles: []model.ExpectedProfiles{{ + Query: `process_cpu:cpu:nanoseconds`, + Flamebearers: model.Flamebearers{ + NameEquals: "main", + }, + }}, + Metrics: []model.ExpectedMetrics{{ + PromQL: `up{job="shop"}`, + Value: `>= 1`, + }}, + }, + } + + c, warnings, err := ConvertDefinition(def, "legacy case") + if err != nil { + fatalf(t, "ConvertDefinition: %v", err) + } + if c.Seed.Type != "app" || c.Seed.Compose != "docker-compose.yml" { + fatalf(t, "unexpected seed mapping: %+v", c.Seed) + } + if len(c.Expected.Traces) != 1 || len(c.Expected.Traces[0].Match) != 2 { + fatalf(t, "expected trace strict+regexp split, got %+v", c.Expected.Traces) + } + if got := c.Expected.Traces[0].Match[1].Attributes["trace_id"].Present; got == nil || !*got { + fatalf(t, "expected trace_id .* to map to present:true") + } + if c.Expected.Logs[0].Count != ">= 1" { + fatalf(t, "expected lossy count min mapping, got %q", c.Expected.Logs[0].Count) + } + if len(c.Expected.Profiles[0].Match) != 1 || c.Expected.Profiles[0].Match[0].Name == nil || *c.Expected.Profiles[0].Match[0].Name != "main" { + fatalf(t, "unexpected profile match mapping: %+v", c.Expected.Profiles[0].Match) + } + if len(warnings) == 0 { + fatalf(t, "expected warnings for dropped input/interval or lossy count") + } + joined := strings.Join(warnings, "\n") + if !strings.Contains(joined, "input requests are not represented") || !strings.Contains(joined, "count max=3 dropped") { + fatalf(t, "expected warnings not found:\n%s", joined) + } +} + +func TestConvertFile_RendersYAML(t *testing.T) { + out, warnings, err := ConvertFile("/home/gregor/source/oats-v2/yaml/testdata/valid-tests/oats.yaml") + if err != nil { + fatalf(t, "ConvertFile: %v", err) + } + if len(warnings) == 0 { + fatalf(t, "expected at least one warning for flattened include/input migration") + } + text := string(out) + for _, want := range []string{"oats: 2", "seed:", "match:", "match_type: regexp", "db.system: h2", "promql: foo"} { + if !strings.Contains(text, want) { + fatalf(t, "expected migrated yaml to contain %q:\n%s", want, text) + } + } +} + +func fatalf(t *testing.T, format string, args ...any) { + t.Helper() + t.Fatalf(format, args...) +} diff --git a/v2case/case.go b/v2case/case.go index b832a236..2a31575d 100644 --- a/v2case/case.go +++ b/v2case/case.go @@ -145,6 +145,19 @@ func (a *AttributeExpectation) UnmarshalYAML(node *yaml.Node) error { } } +func (a AttributeExpectation) MarshalYAML() (any, error) { + switch { + case a.Value != nil && a.Present == nil: + return *a.Value, nil + case a.Value == nil && a.Present != nil: + return map[string]bool{"present": *a.Present}, nil + case a.Value == nil && a.Present == nil: + return nil, nil + default: + return nil, fmt.Errorf("attribute expectation cannot marshal value and present simultaneously") + } +} + type TraceAssertion struct { TraceQL string `yaml:"traceql"` AssertionCommon `yaml:",inline"` diff --git a/yaml/testcase.go b/yaml/testcase.go index 4c5f6fd0..b41c5524 100644 --- a/yaml/testcase.go +++ b/yaml/testcase.go @@ -34,6 +34,13 @@ func ReadTestCases(input []string, evaluateIgnoreFile bool) ([]model.TestCase, e return cases, nil } +// LoadTestCaseDefinition reads one legacy OATS test case definition file, +// resolving includes exactly like the v1 runner does. Template files return +// (nil, nil), matching readTestCaseDefinition's semantics. +func LoadTestCaseDefinition(path string) (*model.TestCaseDefinition, error) { + return readTestCaseDefinition(path, false) +} + func collectTestCases(base string, evaluateIgnoreFile bool) ([]model.TestCase, error) { var cases []model.TestCase From 199339182c702b84103b96ebeff9b2f47dc117f2 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:08:56 +0000 Subject: [PATCH 024/124] feat(v2): support input-driven cases Signed-off-by: Gregor Zeitlinger --- V2.md | 8 +++++ cmd/v2/main.go | 8 +++-- migrate/migrate.go | 19 ++++++++---- migrate/migrate_test.go | 14 ++++++--- runner/runner.go | 68 ++++++++++++++++++++++++++++++++++++++++- runner/runner_test.go | 61 ++++++++++++++++++++++++++++++++++++ v2case/case.go | 31 ++++++++++++++++--- v2case/case_test.go | 29 ++++++++++++++++++ 8 files changed, 220 insertions(+), 18 deletions(-) diff --git a/V2.md b/V2.md index 04a8de15..05f71972 100644 --- a/V2.md +++ b/V2.md @@ -89,6 +89,8 @@ name: rolldice traces have route attribute seed: type: app compose: docker-compose.app.yml +input: + - path: /rolldice?rolls=5 expected: traces: @@ -146,6 +148,12 @@ Per signal under `expected.[]`: | `count` | Comparison against the number of result rows. | | `absent` | If true, the query must return zero rows. | +When a case declares `input`, the runner makes those HTTP requests before each +assertion poll, mirroring OATS v1's “drive the app until telemetry appears” +behavior. Until compose/k3d fixture lifecycle lands in v2, point those +requests at a running app with `--app-host` and `--app-port` (defaults: +`localhost:8080`). + `match` entries default to `match_type: strict`. OATS also supports the small convenience extension `present: true` for attribute existence checks: diff --git a/cmd/v2/main.go b/cmd/v2/main.go index d32d859f..a0555025 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -59,6 +59,8 @@ func run() int { absentTimeout := flag.Duration("absent-timeout", 10*time.Second, "absence-check window") seedSettle := flag.Duration("seed-settle", 2*time.Second, "post-seed wait before first assertion") gcxContextOverride := flag.String("gcx-context", "", "override the gcx --context value (otherwise derived from fixture endpoint)") + appHost := flag.String("app-host", "localhost", "application host for driving case input requests") + appPort := flag.Int("app-port", 8080, "application port for driving case input requests") noCache := flag.Bool("no-cache", false, "disable the skip-when-unchanged cache for this run") cacheDir := flag.String("cache-dir", defaultCacheDir(), "directory for the skip-when-unchanged cache") @@ -123,7 +125,7 @@ func run() int { var totalPass, totalFail int for _, plan := range plans { - ep, err := resolveEndpoint(plan, *gcxContextOverride) + ep, err := resolveEndpoint(plan, *gcxContextOverride, *appHost, *appPort) if err != nil { fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) return 2 @@ -219,8 +221,8 @@ func verbosityFromInt(n int) report.Verbosity { // resolveEndpoint maps a fixture config + an explicit override into the // concrete endpoint the runner needs. The v2 branch ships with "remote" // support only at this stage; "compose" and "k3d" will land later. -func resolveEndpoint(plan discovery.Plan, gcxContextOverride string) (runner.Endpoint, error) { - ep := runner.Endpoint{} +func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, appPort int) (runner.Endpoint, error) { + ep := runner.Endpoint{AppHost: appHost, AppPort: appPort} switch plan.Fixture.Type { case "remote": // For a remote fixture, the gcx context is configured externally diff --git a/migrate/migrate.go b/migrate/migrate.go index 2a6adcea..df41a72b 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -39,6 +39,7 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, c := &v2case.Case{ OatsVersion: v2case.SchemaVersion, Name: name, + Interval: def.Interval, } if def.Kubernetes != nil { @@ -50,12 +51,6 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, if len(def.Include) > 0 { warnings = append(warnings, "include directives were resolved before migration; output is a flattened case") } - if len(def.Input) > 0 { - warnings = append(warnings, "input requests are not represented in v2 yet; dropped from migrated output") - } - if def.Interval != 0 { - warnings = append(warnings, "interval is not represented in v2 yet; dropped from migrated output") - } if len(def.Expected.ComposeLogs) > 0 { warnings = append(warnings, "compose-logs assertions are not migrated") } @@ -81,6 +76,18 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, c.Seed.Traces = []v2case.SeedTrace{{Service: "migrated-service", Spans: []v2case.SeedSpan{{Name: "replace-me"}}}} } + for _, in := range def.Input { + c.Input = append(c.Input, v2case.Input{ + Scheme: in.Scheme, + Host: in.Host, + Method: in.Method, + Path: in.Path, + Headers: in.Headers, + Body: in.Body, + Status: in.Status, + }) + } + for _, tr := range def.Expected.Traces { assertion, ws := convertSignal(tr.TraceQL, tr.Signal) warnings = append(warnings, ws...) diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index d47342f0..46d9921a 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -49,6 +49,12 @@ func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { if c.Seed.Type != "app" || c.Seed.Compose != "docker-compose.yml" { fatalf(t, "unexpected seed mapping: %+v", c.Seed) } + if c.Interval != 500*time.Millisecond { + fatalf(t, "expected interval to carry over, got %v", c.Interval) + } + if len(c.Input) != 1 || c.Input[0].Path != "/stock" { + fatalf(t, "expected input to carry over, got %+v", c.Input) + } if len(c.Expected.Traces) != 1 || len(c.Expected.Traces[0].Match) != 2 { fatalf(t, "expected trace strict+regexp split, got %+v", c.Expected.Traces) } @@ -62,10 +68,10 @@ func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { fatalf(t, "unexpected profile match mapping: %+v", c.Expected.Profiles[0].Match) } if len(warnings) == 0 { - fatalf(t, "expected warnings for dropped input/interval or lossy count") + fatalf(t, "expected warnings for lossy count") } joined := strings.Join(warnings, "\n") - if !strings.Contains(joined, "input requests are not represented") || !strings.Contains(joined, "count max=3 dropped") { + if strings.Contains(joined, "input requests are not represented") || !strings.Contains(joined, "count max=3 dropped") { fatalf(t, "expected warnings not found:\n%s", joined) } } @@ -76,10 +82,10 @@ func TestConvertFile_RendersYAML(t *testing.T) { fatalf(t, "ConvertFile: %v", err) } if len(warnings) == 0 { - fatalf(t, "expected at least one warning for flattened include/input migration") + fatalf(t, "expected at least one warning for flattened include or fixture migration") } text := string(out) - for _, want := range []string{"oats: 2", "seed:", "match:", "match_type: regexp", "db.system: h2", "promql: foo"} { + for _, want := range []string{"oats: 2", "seed:", "input:", "path: /stock", "match:", "match_type: regexp", "db.system: h2", "promql: foo"} { if !strings.Contains(text, want) { fatalf(t, "expected migrated yaml to contain %q:\n%s", want, text) } diff --git a/runner/runner.go b/runner/runner.go index ac3a46be..39565916 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -10,7 +10,10 @@ import ( "context" "encoding/json" "fmt" + "maps" + "net/http" "os" + "strconv" "strings" "time" @@ -20,6 +23,7 @@ import ( "github.com/grafana/oats/report" "github.com/grafana/oats/seed" "github.com/grafana/oats/signalcmd" + "github.com/grafana/oats/testhelpers/requests" "github.com/grafana/oats/v2case" "github.com/grafana/oats/wait" ) @@ -34,6 +38,12 @@ type Endpoint struct { // OTLPHTTP is the base URL for OTLP/HTTP seed POSTs ("http://localhost:4318"). // Required when any case uses seed.type = inline-otlp. OTLPHTTP string + + // AppHost/AppPort identify the application under test for `input` request + // driving. Individual inputs may override host or scheme, but the port + // comes from here. + AppHost string + AppPort int } // Options configures the polling cadence and per-case deadline. Sensible @@ -290,6 +300,9 @@ func (r *Runner) pollAssert( cmdStr := signalcmd.Render(args) run := func() []assert.Failure { + if err := r.driveInputs(c); err != nil { + return []assert.Failure{{Rule: "input", Detail: err.Error()}} + } res, err := r.exec.Execute(ctx, args...) if err != nil { return []assert.Failure{{Rule: "exec", Detail: err.Error()}} @@ -302,7 +315,7 @@ func (r *Runner) pollAssert( return evalFn(res.Stdout, res.Stderr, res.ExitCode) } - opts := wait.Options{Timeout: r.opts.Timeout, Interval: r.opts.Interval} + opts := wait.Options{Timeout: r.opts.Timeout, Interval: r.caseInterval(c)} var result wait.Result[assert.Failure] if absent { opts.Timeout = r.opts.AbsentTimeout @@ -326,6 +339,13 @@ func (r *Runner) pollAssert( return false } +func (r *Runner) caseInterval(c *v2case.Case) time.Duration { + if c.Interval > 0 { + return c.Interval + } + return r.opts.Interval +} + func (r *Runner) runTrace(ctx context.Context, c *v2case.Case, a *v2case.TraceAssertion) bool { args := signalcmd.Traces(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { @@ -487,6 +507,52 @@ func (r *Runner) failCase(c *v2case.Case, msg, cmd string) { }) } +func (r *Runner) driveInputs(c *v2case.Case) error { + for _, in := range c.Input { + if err := r.doInput(in); err != nil { + return err + } + } + return nil +} + +func (r *Runner) doInput(in v2case.Input) error { + if in.Path == "" { + return nil + } + host := r.endpoint.AppHost + if in.Host != "" { + host = in.Host + } + if host == "" || r.endpoint.AppPort == 0 { + return fmt.Errorf("input requires application endpoint; set --app-host/--app-port or provide fixture-derived app endpoint") + } + scheme := "http" + if in.Scheme != "" { + scheme = in.Scheme + } + method := http.MethodGet + if in.Method != "" { + method = strings.ToUpper(in.Method) + } + status := 200 + if in.Status != "" { + parsed, err := strconv.Atoi(in.Status) + if err != nil { + return fmt.Errorf("input status %q is not an integer", in.Status) + } + status = parsed + } + headers := map[string]string{} + if in.Headers != nil { + maps.Copy(headers, in.Headers) + } else { + headers["Accept"] = "application/json" + } + url := fmt.Sprintf("%s://%s:%d%s", scheme, host, r.endpoint.AppPort, in.Path) + return requests.DoHTTPRequest(url, method, headers, in.Body, status) +} + func extractLogRows(stdout string) ([]assert.Row, int, error) { var generic struct { Data struct { diff --git a/runner/runner_test.go b/runner/runner_test.go index 1514c10c..d980db1c 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -3,7 +3,11 @@ package runner import ( "bytes" "context" + "net" + "net/http" + "net/http/httptest" "os" + "strconv" "strings" "testing" "time" @@ -222,6 +226,50 @@ expected: } } +func TestRunCase_DrivesInputRequests(t *testing.T) { + var hits int + app := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + if r.Method != http.MethodPost || r.URL.Path != "/rolldice" { + t.Fatalf("unexpected request: %s %s", r.Method, r.URL.Path) + } + w.WriteHeader(http.StatusCreated) + })) + defer app.Close() + host, port := splitHostPort(t, app.Listener.Addr().String()) + + exec := &stubExec{stdout: "found span service.name=svc"} + r, _ := newRunner(t, exec, Options{Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) + r.endpoint.AppHost = host + r.endpoint.AppPort = port + + c := mustParse(t, ` +oats: 2 +name: traces pass with input +seed: + type: app + compose: x.yml +input: + - path: /rolldice + method: POST + status: "201" +expected: + traces: + - traceql: '{ resource.service.name = "svc" }' + contains: ["svc"] +`) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("expected case to pass") + } + if hits == 0 { + t.Fatalf("expected at least one input request") + } +} + func TestRunCase_InlineOTLPSeedRequiresEndpoint(t *testing.T) { c := mustParse(t, ` oats: 2 @@ -325,3 +373,16 @@ func containsSequence(haystack []string, needles ...string) bool { } return false } + +func splitHostPort(t *testing.T, addr string) (string, int) { + t.Helper() + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("SplitHostPort: %v", err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatalf("Atoi: %v", err) + } + return host, port +} diff --git a/v2case/case.go b/v2case/case.go index 2a31575d..5454aea8 100644 --- a/v2case/case.go +++ b/v2case/case.go @@ -15,6 +15,7 @@ import ( "fmt" "os" "regexp" + "time" "go.yaml.in/yaml/v3" ) @@ -26,12 +27,14 @@ const SchemaVersion = 2 // Case is one entry point yaml file. Cases are independently runnable; // suites group them via oats.toml. type Case struct { - OatsVersion int `yaml:"oats"` - Name string `yaml:"name"` - Tags []string `yaml:"tags,omitempty"` - Hermetic *bool `yaml:"hermetic,omitempty"` // pointer: distinguish unset vs explicit false + OatsVersion int `yaml:"oats"` + Name string `yaml:"name"` + Tags []string `yaml:"tags,omitempty"` + Hermetic *bool `yaml:"hermetic,omitempty"` // pointer: distinguish unset vs explicit false + Interval time.Duration `yaml:"interval,omitempty"` Seed Seed `yaml:"seed"` + Input []Input `yaml:"input,omitempty"` Expected Expected `yaml:"expected"` // SourcePath is filled by the loader; not part of the yaml surface. @@ -76,6 +79,18 @@ type SeedMetric struct { Value int64 `yaml:"value"` } +// Input drives the application under test so telemetry is emitted before +// assertions run. It mirrors the legacy OATS HTTP request shape. +type Input struct { + Scheme string `yaml:"scheme,omitempty"` + Host string `yaml:"host,omitempty"` + Method string `yaml:"method,omitempty"` + Path string `yaml:"path"` + Headers map[string]string `yaml:"headers,omitempty"` + Body string `yaml:"body,omitempty"` + Status string `yaml:"status,omitempty"` +} + // Expected groups per-signal assertion blocks. A case may omit any signal it // does not care about; an empty Expected makes the case a no-op (rejected at // validation). @@ -222,6 +237,9 @@ func (c *Case) Validate() error { if c.Name == "" { return fmt.Errorf("name: required, non-empty") } + if c.Interval < 0 { + return fmt.Errorf("interval: must be >= 0") + } switch c.Seed.Type { case "app": if c.Seed.Compose == "" { @@ -239,6 +257,11 @@ func (c *Case) Validate() error { if len(c.Expected.Traces)+len(c.Expected.Metrics)+len(c.Expected.Logs)+len(c.Expected.Profiles) == 0 { return fmt.Errorf("expected: at least one signal assertion required (a case with no expectations cannot fail)") } + for i, in := range c.Input { + if in.Path == "" { + return fmt.Errorf("input[%d].path: required, non-empty", i) + } + } for i := range c.Expected.Traces { if c.Expected.Traces[i].TraceQL == "" { return fmt.Errorf("expected.traces[%d].traceql: required, non-empty", i) diff --git a/v2case/case_test.go b/v2case/case_test.go index 21b72753..3544fc7f 100644 --- a/v2case/case_test.go +++ b/v2case/case_test.go @@ -3,15 +3,19 @@ package v2case import ( "strings" "testing" + "time" ) func TestParse_AppSeed(t *testing.T) { src := []byte(` oats: 2 name: rolldice traces have route attribute +interval: 250ms seed: type: app compose: docker-compose.app.yml +input: + - path: /rolldice?rolls=5 expected: traces: - traceql: '{ span.http.route = "/rolldice" }' @@ -30,6 +34,12 @@ expected: if c.Seed.Type != "app" || c.Seed.Compose != "docker-compose.app.yml" { t.Errorf("Seed: %+v", c.Seed) } + if c.Interval != 250*time.Millisecond { + t.Errorf("Interval: got %v", c.Interval) + } + if len(c.Input) != 1 || c.Input[0].Path != "/rolldice?rolls=5" { + t.Errorf("Input: %+v", c.Input) + } if len(c.Expected.Traces) != 1 || c.Expected.Traces[0].TraceQL == "" { t.Errorf("Expected.Traces: %+v", c.Expected.Traces) } @@ -170,6 +180,25 @@ func TestValidate_NoExpectations(t *testing.T) { } } +func TestValidate_RejectsInputWithoutPath(t *testing.T) { + _, err := Parse([]byte(` +oats: 2 +name: bad input +seed: + type: app + compose: x.yml +input: + - method: POST +expected: + metrics: + - promql: up + value: ">= 1" +`)) + if err == nil || !strings.Contains(err.Error(), "input[0].path") { + t.Fatalf("expected input path error, got %v", err) + } +} + func TestValidate_RejectsUnknownMatchType(t *testing.T) { _, err := Parse([]byte(` oats: 2 From d6abef095951eb4f7b20b42a7fe4e01c4f2c9459 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:11:01 +0000 Subject: [PATCH 025/124] feat(v2): add minimal compose fixture support Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 31 ++++++++++++++++++ cmd/v2/main.go | 64 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 92 insertions(+), 3 deletions(-) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 8f2891d0..9c37d19d 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -18,6 +18,37 @@ import ( "github.com/grafana/oats/runner" ) +func TestResolveComposeFile(t *testing.T) { + got, err := resolveComposeFile("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFile: "stack/compose.yml"}) + if err != nil { + t.Fatalf("resolveComposeFile compose_file: %v", err) + } + if want := "/tmp/work/stack/compose.yml"; got != want { + t.Fatalf("got %q want %q", got, want) + } + + got, err = resolveComposeFile("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "lgtm"}) + if err != nil { + t.Fatalf("resolveComposeFile template=lgtm: %v", err) + } + if want := "/tmp/work/docker-compose.yml"; got != want { + t.Fatalf("got %q want %q", got, want) + } +} + +func TestResolveEndpoint_ComposeDefaults(t *testing.T) { + ep, err := resolveEndpoint("/tmp/work", discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, + Fixture: discovery.FixtureConfig{Type: "compose", Template: "lgtm"}, + }, "", "localhost", 8080, "http://localhost:4318") + if err != nil { + t.Fatalf("resolveEndpoint: %v", err) + } + if ep.GCXContext != "local" || ep.AppHost != "localhost" || ep.AppPort != 8080 || ep.OTLPHTTP != "http://localhost:4318" { + t.Fatalf("unexpected endpoint: %+v", ep) + } +} + // TestIntegration_FullPipelineWithFakeGCX wires the v2 chain end-to-end: // discovery → seed (against an httptest OTLP stub) → engine (against the // fake-gcx.sh shell script) → assertions → report. No real gcx, no real diff --git a/cmd/v2/main.go b/cmd/v2/main.go index a0555025..94d106e5 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -39,6 +39,7 @@ import ( "github.com/grafana/oats/migrate" "github.com/grafana/oats/report" "github.com/grafana/oats/runner" + "github.com/grafana/oats/testhelpers/compose" ) func main() { @@ -61,6 +62,7 @@ func run() int { gcxContextOverride := flag.String("gcx-context", "", "override the gcx --context value (otherwise derived from fixture endpoint)") appHost := flag.String("app-host", "localhost", "application host for driving case input requests") appPort := flag.Int("app-port", 8080, "application port for driving case input requests") + otlpHTTP := flag.String("otlp-http", "http://localhost:4318", "OTLP/HTTP base URL for inline-otlp seed mode") noCache := flag.Bool("no-cache", false, "disable the skip-when-unchanged cache for this run") cacheDir := flag.String("cache-dir", defaultCacheDir(), "directory for the skip-when-unchanged cache") @@ -125,11 +127,19 @@ func run() int { var totalPass, totalFail int for _, plan := range plans { - ep, err := resolveEndpoint(plan, *gcxContextOverride, *appHost, *appPort) + fix, err := startFixture(ctx, cfg.SourceDir, plan) if err != nil { fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) return 2 } + ep, err := resolveEndpoint(cfg.SourceDir, plan, *gcxContextOverride, *appHost, *appPort, *otlpHTTP) + if err != nil { + if fix != nil { + _ = fix.Close() + } + fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) + return 2 + } rep.Emit(report.Event{ Type: report.EventSuiteStart, @@ -181,6 +191,12 @@ func run() int { Pass: suitePass, Fail: suiteFail, }) + if fix != nil { + if closeErr := fix.Close(); closeErr != nil { + fmt.Fprintf(os.Stderr, "suite %q: fixture shutdown: %v\n", plan.Suite.Name, closeErr) + return 2 + } + } } rep.Emit(report.Event{ @@ -221,8 +237,8 @@ func verbosityFromInt(n int) report.Verbosity { // resolveEndpoint maps a fixture config + an explicit override into the // concrete endpoint the runner needs. The v2 branch ships with "remote" // support only at this stage; "compose" and "k3d" will land later. -func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, appPort int) (runner.Endpoint, error) { - ep := runner.Endpoint{AppHost: appHost, AppPort: appPort} +func resolveEndpoint(sourceDir string, plan discovery.Plan, gcxContextOverride, appHost string, appPort int, otlpHTTP string) (runner.Endpoint, error) { + ep := runner.Endpoint{AppHost: appHost, AppPort: appPort, OTLPHTTP: otlpHTTP} switch plan.Fixture.Type { case "remote": // For a remote fixture, the gcx context is configured externally @@ -230,6 +246,8 @@ func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, ap // best-effort default; --gcx-context overrides. ep.GCXContext = plan.Suite.Fixture ep.OTLPHTTP = plan.Fixture.Endpoint + case "compose": + ep.GCXContext = plan.Suite.Fixture case "": // No fixture configured — caller (or --gcx-context) must supply // everything. Useful while plumbing v2 against an external setup. @@ -245,6 +263,46 @@ func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, ap return ep, nil } +type suiteFixture interface { + Close() error +} + +func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (suiteFixture, error) { + switch plan.Fixture.Type { + case "", "remote": + return nil, nil + case "compose": + composeFile, err := resolveComposeFile(sourceDir, plan.Fixture) + if err != nil { + return nil, err + } + suite, err := compose.Suite(composeFile) + if err != nil { + return nil, err + } + if err := suite.Up(); err != nil { + return nil, err + } + return suite, nil + default: + return nil, fmt.Errorf("fixture type %q is not yet supported in oats-v2 (k3d arrives in follow-up commits)", plan.Fixture.Type) + } +} + +func resolveComposeFile(sourceDir string, fixture discovery.FixtureConfig) (string, error) { + if fixture.ComposeFile != "" { + return filepath.Join(sourceDir, fixture.ComposeFile), nil + } + switch fixture.Template { + case "lgtm": + return filepath.Join(sourceDir, "docker-compose.yml"), nil + case "": + return "", fmt.Errorf("compose fixture requires compose_file or supported template") + default: + return "", fmt.Errorf("unsupported compose fixture template %q", fixture.Template) + } +} + func splitCSV(s string) []string { if strings.TrimSpace(s) == "" { return nil From 941219685fd73cb2791ac5dd6989cbbb5f9f18fb Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:13:11 +0000 Subject: [PATCH 026/124] feat(v2): support multi-file compose fixtures Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 22 +++++++++++++++------- cmd/v2/main.go | 29 +++++++++++++++++------------ discovery/discovery.go | 19 ++++++++++++------- discovery/discovery_test.go | 19 +++++++++++++++++++ testhelpers/compose/compose.go | 24 +++++++++++++++++------- 5 files changed, 80 insertions(+), 33 deletions(-) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 9c37d19d..5c41cabf 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -18,20 +18,28 @@ import ( "github.com/grafana/oats/runner" ) -func TestResolveComposeFile(t *testing.T) { - got, err := resolveComposeFile("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFile: "stack/compose.yml"}) +func TestResolveComposeFiles(t *testing.T) { + got, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFile: "stack/compose.yml"}) if err != nil { - t.Fatalf("resolveComposeFile compose_file: %v", err) + t.Fatalf("resolveComposeFiles compose_file: %v", err) } - if want := "/tmp/work/stack/compose.yml"; got != want { + if want := []string{"/tmp/work/stack/compose.yml"}; len(got) != 1 || got[0] != want[0] { t.Fatalf("got %q want %q", got, want) } - got, err = resolveComposeFile("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "lgtm"}) + got, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFiles: []string{"a.yml", "b.yml"}}) if err != nil { - t.Fatalf("resolveComposeFile template=lgtm: %v", err) + t.Fatalf("resolveComposeFiles compose_files: %v", err) } - if want := "/tmp/work/docker-compose.yml"; got != want { + if len(got) != 2 || got[0] != "/tmp/work/a.yml" || got[1] != "/tmp/work/b.yml" { + t.Fatalf("unexpected compose_files resolution: %v", got) + } + + got, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "lgtm"}) + if err != nil { + t.Fatalf("resolveComposeFiles template=lgtm: %v", err) + } + if want := []string{"/tmp/work/docker-compose.yml"}; len(got) != 1 || got[0] != want[0] { t.Fatalf("got %q want %q", got, want) } } diff --git a/cmd/v2/main.go b/cmd/v2/main.go index 94d106e5..e5616323 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -272,11 +272,11 @@ func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (sui case "", "remote": return nil, nil case "compose": - composeFile, err := resolveComposeFile(sourceDir, plan.Fixture) + composeFiles, err := resolveComposeFiles(sourceDir, plan.Fixture) if err != nil { return nil, err } - suite, err := compose.Suite(composeFile) + suite, err := compose.SuiteFiles(composeFiles, plan.Fixture.Env) if err != nil { return nil, err } @@ -289,18 +289,23 @@ func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (sui } } -func resolveComposeFile(sourceDir string, fixture discovery.FixtureConfig) (string, error) { - if fixture.ComposeFile != "" { - return filepath.Join(sourceDir, fixture.ComposeFile), nil - } - switch fixture.Template { - case "lgtm": - return filepath.Join(sourceDir, "docker-compose.yml"), nil - case "": - return "", fmt.Errorf("compose fixture requires compose_file or supported template") +func resolveComposeFiles(sourceDir string, fixture discovery.FixtureConfig) ([]string, error) { + var files []string + switch { + case fixture.ComposeFile != "": + files = append(files, filepath.Join(sourceDir, fixture.ComposeFile)) + case len(fixture.ComposeFiles) > 0: + for _, file := range fixture.ComposeFiles { + files = append(files, filepath.Join(sourceDir, file)) + } + case fixture.Template == "lgtm": + files = append(files, filepath.Join(sourceDir, "docker-compose.yml")) + case fixture.Template == "": + return nil, fmt.Errorf("compose fixture requires compose_file, compose_files, or supported template") default: - return "", fmt.Errorf("unsupported compose fixture template %q", fixture.Template) + return nil, fmt.Errorf("unsupported compose fixture template %q", fixture.Template) } + return files, nil } func splitCSV(s string) []string { diff --git a/discovery/discovery.go b/discovery/discovery.go index 48c33052..fbc86ef9 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -44,11 +44,13 @@ type SuiteConfig struct { } type FixtureConfig struct { - Type string `toml:"type"` // "compose" | "k3d" | "remote" - Template string `toml:"template,omitempty"` - ComposeFile string `toml:"compose_file,omitempty"` - PoolSize int `toml:"pool_size,omitempty"` - Endpoint string `toml:"endpoint,omitempty"` // remote only + Type string `toml:"type"` // "compose" | "k3d" | "remote" + Template string `toml:"template,omitempty"` + ComposeFile string `toml:"compose_file,omitempty"` + ComposeFiles []string `toml:"compose_files,omitempty"` + Env []string `toml:"env,omitempty"` + PoolSize int `toml:"pool_size,omitempty"` + Endpoint string `toml:"endpoint,omitempty"` // remote only } type CacheConfig struct { @@ -104,8 +106,11 @@ func (c *RootConfig) Validate() error { for name, f := range c.Fixture { switch f.Type { case "compose": - if f.Template == "" && f.ComposeFile == "" { - return fmt.Errorf("fixture %q: type=compose requires template or compose_file", name) + if f.Template == "" && f.ComposeFile == "" && len(f.ComposeFiles) == 0 { + return fmt.Errorf("fixture %q: type=compose requires template, compose_file, or compose_files", name) + } + if f.ComposeFile != "" && len(f.ComposeFiles) > 0 { + return fmt.Errorf("fixture %q: use compose_file or compose_files, not both", name) } case "k3d": // PoolSize=0 means "single ephemeral cluster" — valid. diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 1411a23e..4bfbe276 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -187,6 +187,25 @@ func TestValidate_RemoteRequiresEndpoint(t *testing.T) { } } +func TestValidate_ComposeFilesConflict(t *testing.T) { + cfg := &RootConfig{ + Meta: Meta{Version: 2}, + Suites: []SuiteConfig{{ + Name: "s", Cases: []string{"a.yaml"}, Fixture: "c", + }}, + Fixture: map[string]FixtureConfig{"c": { + Type: "compose", + ComposeFile: "one.yml", + ComposeFiles: []string{ + "two.yml", + }, + }}, + } + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "compose_file or compose_files") { + t.Errorf("expected compose conflict error, got %v", err) + } +} + func TestPlanRun_EmptyGlobIsAnError(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats.toml", ` diff --git a/testhelpers/compose/compose.go b/testhelpers/compose/compose.go index 3b11b988..b966d872 100644 --- a/testhelpers/compose/compose.go +++ b/testhelpers/compose/compose.go @@ -10,7 +10,6 @@ import ( "log/slog" "os" "os/exec" - "path" "strings" "sync" @@ -20,7 +19,7 @@ import ( type Compose struct { Command string DefaultArgs []string - Path string + Paths []string Env []string } @@ -29,14 +28,26 @@ func defaultEnv() []string { } func Suite(composeFile string) (*Compose, error) { + return SuiteFiles([]string{composeFile}, nil) +} + +func SuiteFiles(composeFiles []string, env []string) (*Compose, error) { command := "docker" defaultArgs := []string{"compose"} + for _, file := range composeFiles { + defaultArgs = append(defaultArgs, "-f", file) + } + if len(composeFiles) == 0 { + return nil, fmt.Errorf("at least one compose file is required") + } + mergedEnv := defaultEnv() + mergedEnv = append(mergedEnv, env...) return &Compose{ Command: command, DefaultArgs: defaultArgs, - Path: path.Join(composeFile), - Env: defaultEnv(), + Paths: composeFiles, + Env: mergedEnv, }, nil } @@ -70,7 +81,6 @@ func (c *Compose) runDocker(cc command) error { var cmdArgs []string if cc.compose { cmdArgs = c.DefaultArgs - cmdArgs = append(cmdArgs, "-f", c.Path) } cmdArgs = append(cmdArgs, cc.args...) cmd := exec.Command(c.Command, cmdArgs...) @@ -88,7 +98,7 @@ func (c *Compose) runDocker(cc command) error { } wg.Wait() } else if cc.background { - slog.Info("Running", "command", cmd.String(), "dir", c.Path) + slog.Info("Running", "command", cmd.String(), "dir", strings.Join(c.Paths, ",")) stdout, _ := cmd.StdoutPipe() cmd.Stderr = cmd.Stdout go func() { @@ -105,7 +115,7 @@ func (c *Compose) runDocker(cc command) error { return fmt.Errorf("failed to start docker command: %w", err) } } else { - slog.Info("Running", "command", cmd.String(), "dir", c.Path) + slog.Info("Running", "command", cmd.String(), "dir", strings.Join(c.Paths, ",")) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err := cmd.Run() From f6e6c1b08909d257e9490b076d697784105094a5 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:15:23 +0000 Subject: [PATCH 027/124] docs(v2): add sample config and routing note Signed-off-by: Gregor Zeitlinger --- README.md | 4 ++++ V2.md | 12 +++++------- discovery/discovery_test.go | 17 +++++++++++++++++ examples/v2-smoke/cases/rolldice.yaml | 21 +++++++++++++++++++++ examples/v2-smoke/oats.toml | 12 ++++++++++++ 5 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 examples/v2-smoke/cases/rolldice.yaml create mode 100644 examples/v2-smoke/oats.toml diff --git a/README.md b/README.md index dcb6c36e..bf64cf9a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # OpenTelemetry Acceptance Tests (OATs) +> **Note:** this README primarily documents the legacy `oats` flow and yaml +> shape. The newer gcx-driven v2 work lives behind `cmd/v2` / `oats-v2`; see +> [V2.md](V2.md) for the current v2 syntax, fixture model, and migration path. + OpenTelemetry Acceptance Tests (OATs), or OATs for short, is a test framework for OpenTelemetry. - Declarative tests written in YAML diff --git a/V2.md b/V2.md index 05f71972..c7dd8014 100644 --- a/V2.md +++ b/V2.md @@ -73,6 +73,8 @@ tags = ["traces", "metrics", "logs"] type = "compose" # compose | k3d | remote template = "lgtm" # built-in compose template # compose_file = "./my-compose.yml" # alternative to template +# compose_files = ["./base.yml", "./override.yml"] # multi-file compose +# env = ["FOO=bar"] # extra env for compose config/up # pool_size = 4 # k3d only # endpoint = "http://..." # remote only @@ -150,9 +152,8 @@ Per signal under `expected.[]`: When a case declares `input`, the runner makes those HTTP requests before each assertion poll, mirroring OATS v1's “drive the app until telemetry appears” -behavior. Until compose/k3d fixture lifecycle lands in v2, point those -requests at a running app with `--app-host` and `--app-port` (defaults: -`localhost:8080`). +behavior. For remote fixtures, point those requests at a running app with +`--app-host` and `--app-port` (defaults: `localhost:8080`). `match` entries default to `match_type: strict`. OATS also supports the small convenience extension `present: true` for attribute existence checks: @@ -171,15 +172,12 @@ match: ## What's not in v2-alpha yet -- `compose` and `k3d` fixture types — the runner currently only accepts - `remote` (or no fixture, when you run against an externally-started stack - via `--gcx-context`). Compose lifecycle lands in a follow-up commit. - Parallel cases within a suite. - k3d pool warmup. - Full-fidelity `oats migrate` for fixture/input semantics. A best-effort `oats-v2 --migrate ` now converts expectation blocks into the collector-style `match` schema and prints warnings for dropped/unsupported - fields. + fields (matrix/custom-checks and some compose details still need manual follow-up). - Hermeticity static-check (runtime check applies). ## Migrating from v1 diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 4bfbe276..3d4ed896 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -238,6 +238,23 @@ func TestSummary(t *testing.T) { } } +func TestExampleV2SmokeConfigLoads(t *testing.T) { + cfg, err := Load(filepath.Join("..", "examples", "v2-smoke", "oats.toml")) + if err != nil { + t.Fatalf("Load example config: %v", err) + } + plans, err := cfg.PlanRun(Filter{}) + if err != nil { + t.Fatalf("PlanRun example config: %v", err) + } + if len(plans) != 1 || len(plans[0].Cases) != 1 { + t.Fatalf("expected one suite/one case, got %+v", plans) + } + if plans[0].Cases[0].Name != "rolldice smoke" { + t.Fatalf("unexpected case name: %q", plans[0].Cases[0].Name) + } +} + func planNames(p []Plan) []string { out := make([]string, len(p)) for i := range p { diff --git a/examples/v2-smoke/cases/rolldice.yaml b/examples/v2-smoke/cases/rolldice.yaml new file mode 100644 index 00000000..40f518b1 --- /dev/null +++ b/examples/v2-smoke/cases/rolldice.yaml @@ -0,0 +1,21 @@ +oats: 2 +name: rolldice smoke +input: + - path: /rolldice?rolls=5 +seed: + type: app + compose: docker-compose.app.yml +expected: + traces: + - traceql: '{ resource.service.name = "rolldice" }' + match: + - match_type: regexp + name: '^GET /rolldice.*$' + metrics: + - promql: 'http_server_active_requests{service_name="rolldice"}' + value: '>= 0' + logs: + - logql: '{service_name="rolldice"} |~ `rolling the dice`' + match: + - match_type: regexp + name: '.*rolling the dice.*' diff --git a/examples/v2-smoke/oats.toml b/examples/v2-smoke/oats.toml new file mode 100644 index 00000000..16c82f12 --- /dev/null +++ b/examples/v2-smoke/oats.toml @@ -0,0 +1,12 @@ +[meta] +version = 2 + +[[suite]] +name = "smoke" +cases = ["cases/*.yaml"] +fixture = "local-lgtm" +tags = ["traces", "logs", "metrics"] + +[fixture.local-lgtm] +type = "remote" +endpoint = "http://localhost:4318" From 626aa2d9f2f996f5ca97f9704c9a4dc5cc78f818 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:17:34 +0000 Subject: [PATCH 028/124] feat(v2): add minimal k3d fixture support Signed-off-by: Gregor Zeitlinger --- V2.md | 11 +++++++++-- cmd/v2/integration_test.go | 13 +++++++++++++ cmd/v2/main.go | 35 +++++++++++++++++++++++++++++++++++ discovery/discovery.go | 24 +++++++++++++++++------- discovery/discovery_test.go | 15 +++++++++++++++ 5 files changed, 89 insertions(+), 9 deletions(-) diff --git a/V2.md b/V2.md index c7dd8014..3584baaa 100644 --- a/V2.md +++ b/V2.md @@ -75,7 +75,14 @@ template = "lgtm" # built-in compose template # compose_file = "./my-compose.yml" # alternative to template # compose_files = ["./base.yml", "./override.yml"] # multi-file compose # env = ["FOO=bar"] # extra env for compose config/up -# pool_size = 4 # k3d only +# k8s_dir = "./k8s" # k3d only +# app_service = "dice" # k3d only +# app_docker_file = "Dockerfile" +# app_docker_context = "." +# app_docker_tag = "dice:test" +# app_port = 8080 +# import_images = ["grafana/docker-otel-lgtm:latest"] +# pool_size = 4 # reserved for future k3d pooling # endpoint = "http://..." # remote only [cache] @@ -173,7 +180,7 @@ match: ## What's not in v2-alpha yet - Parallel cases within a suite. -- k3d pool warmup. +- k3d pool warmup / reuse. - Full-fidelity `oats migrate` for fixture/input semantics. A best-effort `oats-v2 --migrate ` now converts expectation blocks into the collector-style `match` schema and prints warnings for dropped/unsupported diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 5c41cabf..ab086946 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -57,6 +57,19 @@ func TestResolveEndpoint_ComposeDefaults(t *testing.T) { } } +func TestResolveEndpoint_K3DUsesFixtureAppPort(t *testing.T) { + ep, err := resolveEndpoint("/tmp/work", discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "cluster"}, + Fixture: discovery.FixtureConfig{Type: "k3d", AppPort: 18080}, + }, "", "localhost", 8080, "http://localhost:4318") + if err != nil { + t.Fatalf("resolveEndpoint: %v", err) + } + if ep.GCXContext != "cluster" || ep.AppPort != 18080 { + t.Fatalf("unexpected endpoint: %+v", ep) + } +} + // TestIntegration_FullPipelineWithFakeGCX wires the v2 chain end-to-end: // discovery → seed (against an httptest OTLP stub) → engine (against the // fake-gcx.sh shell script) → assertions → report. No real gcx, no real diff --git a/cmd/v2/main.go b/cmd/v2/main.go index e5616323..faf92063 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -40,6 +40,8 @@ import ( "github.com/grafana/oats/report" "github.com/grafana/oats/runner" "github.com/grafana/oats/testhelpers/compose" + "github.com/grafana/oats/testhelpers/kubernetes" + "github.com/grafana/oats/testhelpers/remote" ) func main() { @@ -248,6 +250,11 @@ func resolveEndpoint(sourceDir string, plan discovery.Plan, gcxContextOverride, ep.OTLPHTTP = plan.Fixture.Endpoint case "compose": ep.GCXContext = plan.Suite.Fixture + case "k3d": + ep.GCXContext = plan.Suite.Fixture + if plan.Fixture.AppPort > 0 { + ep.AppPort = plan.Fixture.AppPort + } case "": // No fixture configured — caller (or --gcx-context) must supply // everything. Useful while plumbing v2 against an external setup. @@ -284,6 +291,26 @@ func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (sui return nil, err } return suite, nil + case "k3d": + model := &kubernetes.Kubernetes{ + Dir: filepath.Join(sourceDir, plan.Fixture.K8sDir), + AppService: plan.Fixture.AppService, + AppDockerFile: plan.Fixture.AppDockerFile, + AppDockerContext: plan.Fixture.AppDockerContext, + AppDockerTag: plan.Fixture.AppDockerTag, + AppDockerPort: plan.Fixture.AppPort, + ImportImages: plan.Fixture.ImportImages, + } + ep := kubernetes.NewEndpoint("localhost", model, remote.PortsConfig{ + PrometheusHTTPPort: 9090, + LokiHttpPort: 3100, + TempoHTTPPort: 3200, + PyroscopeHttpPort: 4040, + }, plan.Suite.Name, sourceDir) + if err := ep.Start(context.Background()); err != nil { + return nil, err + } + return endpointFixture{ep: ep}, nil default: return nil, fmt.Errorf("fixture type %q is not yet supported in oats-v2 (k3d arrives in follow-up commits)", plan.Fixture.Type) } @@ -308,6 +335,14 @@ func resolveComposeFiles(sourceDir string, fixture discovery.FixtureConfig) ([]s return files, nil } +type endpointFixture struct { + ep *remote.Endpoint +} + +func (e endpointFixture) Close() error { + return e.ep.Stop(context.Background()) +} + func splitCSV(s string) []string { if strings.TrimSpace(s) == "" { return nil diff --git a/discovery/discovery.go b/discovery/discovery.go index fbc86ef9..c7d773ee 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -44,13 +44,20 @@ type SuiteConfig struct { } type FixtureConfig struct { - Type string `toml:"type"` // "compose" | "k3d" | "remote" - Template string `toml:"template,omitempty"` - ComposeFile string `toml:"compose_file,omitempty"` - ComposeFiles []string `toml:"compose_files,omitempty"` - Env []string `toml:"env,omitempty"` - PoolSize int `toml:"pool_size,omitempty"` - Endpoint string `toml:"endpoint,omitempty"` // remote only + Type string `toml:"type"` // "compose" | "k3d" | "remote" + Template string `toml:"template,omitempty"` + ComposeFile string `toml:"compose_file,omitempty"` + ComposeFiles []string `toml:"compose_files,omitempty"` + Env []string `toml:"env,omitempty"` + K8sDir string `toml:"k8s_dir,omitempty"` + AppService string `toml:"app_service,omitempty"` + AppDockerFile string `toml:"app_docker_file,omitempty"` + AppDockerContext string `toml:"app_docker_context,omitempty"` + AppDockerTag string `toml:"app_docker_tag,omitempty"` + AppPort int `toml:"app_port,omitempty"` + ImportImages []string `toml:"import_images,omitempty"` + PoolSize int `toml:"pool_size,omitempty"` + Endpoint string `toml:"endpoint,omitempty"` // remote only } type CacheConfig struct { @@ -113,6 +120,9 @@ func (c *RootConfig) Validate() error { return fmt.Errorf("fixture %q: use compose_file or compose_files, not both", name) } case "k3d": + if f.K8sDir == "" || f.AppService == "" || f.AppDockerFile == "" || f.AppDockerTag == "" || f.AppPort == 0 { + return fmt.Errorf("fixture %q: type=k3d requires k8s_dir, app_service, app_docker_file, app_docker_tag, and app_port", name) + } // PoolSize=0 means "single ephemeral cluster" — valid. case "remote": if f.Endpoint == "" { diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 3d4ed896..3e07b101 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -206,6 +206,21 @@ func TestValidate_ComposeFilesConflict(t *testing.T) { } } +func TestValidate_K3DRequiresFields(t *testing.T) { + cfg := &RootConfig{ + Meta: Meta{Version: 2}, + Suites: []SuiteConfig{{ + Name: "s", Cases: []string{"a.yaml"}, Fixture: "k", + }}, + Fixture: map[string]FixtureConfig{"k": { + Type: "k3d", + }}, + } + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "type=k3d requires") { + t.Errorf("expected k3d field error, got %v", err) + } +} + func TestPlanRun_EmptyGlobIsAnError(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats.toml", ` From 63e6159a8262a9389608fd748e153524a8fdd69f Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:22:27 +0000 Subject: [PATCH 029/124] feat(v2): relax app seed and improve migrate hints Signed-off-by: Gregor Zeitlinger --- V2.md | 15 +++++--- migrate/migrate.go | 85 +++++++++++++++++++++++++++++++++++++---- migrate/migrate_test.go | 47 +++++++++++++++++++++++ v2case/case.go | 12 +++--- v2case/case_test.go | 6 +-- 5 files changed, 144 insertions(+), 21 deletions(-) diff --git a/V2.md b/V2.md index 3584baaa..9a0bdeca 100644 --- a/V2.md +++ b/V2.md @@ -97,7 +97,7 @@ name: rolldice traces have route attribute seed: type: app - compose: docker-compose.app.yml + compose: docker-compose.app.yml # optional legacy shorthand; suite fixture usually owns boot input: - path: /rolldice?rolls=5 @@ -120,6 +120,10 @@ expected: service_name: dice-server ``` +For `seed.type: app`, the application is normally started by the suite fixture in +`oats.toml`. `seed.compose` is still accepted as a migration/compatibility hint, +but the runner does not require it when a fixture already provides the app. + Inline-OTLP seed (no example app required): ```yaml @@ -184,7 +188,7 @@ match: - Full-fidelity `oats migrate` for fixture/input semantics. A best-effort `oats-v2 --migrate ` now converts expectation blocks into the collector-style `match` schema and prints warnings for dropped/unsupported - fields (matrix/custom-checks and some compose details still need manual follow-up). + fields (matrix/custom-checks still need manual follow-up). - Hermeticity static-check (runtime check applies). ## Migrating from v1 @@ -199,9 +203,10 @@ oats-v2 --migrate path/to/oats.yaml > migrated.yaml It converts legacy `equals` / `regexp` / `attributes` / `attribute-regexp` assertions into v2 `match:` entries and prints warnings -for fields that still need manual follow-up (for example `input`, matrix, -or multi-file docker-compose details). The v1 binary (`oats`) and the v2 -binary (`oats-v2`) still read different file shapes. +for fields that still need manual follow-up. For richer legacy fixtures, the +warnings now include ready-to-paste `oats.toml` fixture snippets for +multi-file compose/env and kubernetes→k3d mappings. The v1 binary (`oats`) +and the v2 binary (`oats-v2`) still read different file shapes. ## Verbosity contract diff --git a/migrate/migrate.go b/migrate/migrate.go index df41a72b..4ef5c139 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -43,7 +43,9 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, } if def.Kubernetes != nil { - return nil, warnings, fmt.Errorf("kubernetes fixtures are not yet supported by v2 migration") + c.Seed.Type = "app" + warnings = append(warnings, "legacy kubernetes fixture migrated as an app-backed case; paste the suggested [fixture] block below into oats.toml") + warnings = append(warnings, kubernetesFixtureHint(name, def)) } if len(def.Matrix) > 0 { warnings = append(warnings, "matrix definitions are not migrated; convert expanded matrix cases manually") @@ -64,13 +66,11 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, return nil, warnings, fmt.Errorf("docker-compose present but no files declared") } c.Seed.Compose = def.DockerCompose.Files[0] - if len(def.DockerCompose.Files) > 1 { - warnings = append(warnings, fmt.Sprintf("multiple docker-compose files collapsed to first entry %q", def.DockerCompose.Files[0])) - } - if len(def.DockerCompose.Environment) > 0 { - warnings = append(warnings, "docker-compose env overrides are not represented in v2 yet") + if len(def.DockerCompose.Files) > 1 || len(def.DockerCompose.Environment) > 0 { + warnings = append(warnings, "legacy docker-compose fixture uses suite-level config richer than case-level seed.compose; paste the suggested [fixture] block below into oats.toml") + warnings = append(warnings, composeFixtureHint(name, def)) } - } else { + } else if def.Kubernetes == nil { warnings = append(warnings, "no docker-compose fixture found; defaulting seed.type to inline-otlp placeholder") c.Seed.Type = "inline-otlp" c.Seed.Traces = []v2case.SeedTrace{{Service: "migrated-service", Spans: []v2case.SeedSpan{{Name: "replace-me"}}}} @@ -191,4 +191,75 @@ func deriveName(path string) string { return strings.TrimSpace(base) } +func composeFixtureHint(name string, def model.TestCaseDefinition) string { + fixtureName := slug(name) + var b strings.Builder + fmt.Fprintf(&b, "suggested oats.toml fixture snippet for %q:\n", name) + fmt.Fprintf(&b, "[fixture.%s]\n", fixtureName) + fmt.Fprintf(&b, "type = \"compose\"\n") + if len(def.DockerCompose.Files) == 1 { + fmt.Fprintf(&b, "compose_file = %q\n", def.DockerCompose.Files[0]) + } else if len(def.DockerCompose.Files) > 1 { + fmt.Fprintf(&b, "compose_files = [%s]\n", quotedList(def.DockerCompose.Files)) + } + if len(def.DockerCompose.Environment) > 0 { + fmt.Fprintf(&b, "env = [%s]\n", quotedList(def.DockerCompose.Environment)) + } + return strings.TrimRight(b.String(), "\n") +} + +func kubernetesFixtureHint(name string, def model.TestCaseDefinition) string { + fixtureName := slug(name) + k := def.Kubernetes + var b strings.Builder + fmt.Fprintf(&b, "suggested oats.toml fixture snippet for %q:\n", name) + fmt.Fprintf(&b, "[fixture.%s]\n", fixtureName) + fmt.Fprintf(&b, "type = \"k3d\"\n") + fmt.Fprintf(&b, "k8s_dir = %q\n", k.Dir) + fmt.Fprintf(&b, "app_service = %q\n", k.AppService) + fmt.Fprintf(&b, "app_docker_file = %q\n", k.AppDockerFile) + if k.AppDockerContext != "" { + fmt.Fprintf(&b, "app_docker_context = %q\n", k.AppDockerContext) + } + fmt.Fprintf(&b, "app_docker_tag = %q\n", k.AppDockerTag) + fmt.Fprintf(&b, "app_port = %d\n", k.AppDockerPort) + if len(k.ImportImages) > 0 { + fmt.Fprintf(&b, "import_images = [%s]\n", quotedList(k.ImportImages)) + } + return strings.TrimRight(b.String(), "\n") +} + +func quotedList(items []string) string { + quoted := make([]string, 0, len(items)) + for _, item := range items { + quoted = append(quoted, fmt.Sprintf("%q", item)) + } + return strings.Join(quoted, ", ") +} + +func slug(s string) string { + s = strings.ToLower(s) + s = strings.ReplaceAll(s, "_", "-") + s = strings.ReplaceAll(s, " ", "-") + var b strings.Builder + lastDash := false + for _, r := range s { + keep := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + if keep { + b.WriteRune(r) + lastDash = false + continue + } + if !lastDash { + b.WriteByte('-') + lastDash = true + } + } + out := strings.Trim(b.String(), "-") + if out == "" { + return "migrated" + } + return out +} + func strPtr(s string) *string { return &s } diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 46d9921a..b3c65ae8 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/grafana/oats/model" + "github.com/grafana/oats/testhelpers/kubernetes" ) func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { @@ -92,6 +93,52 @@ func TestConvertFile_RendersYAML(t *testing.T) { } } +func TestConvertDefinition_KubernetesProducesFixtureHint(t *testing.T) { + def := model.TestCaseDefinition{ + Kubernetes: &kubernetes.Kubernetes{ + Dir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerContext: "..", + AppDockerTag: "dice:test", + AppDockerPort: 8080, + ImportImages: []string{"busybox:latest"}, + }, + Expected: model.Expected{ + Logs: []model.ExpectedLogs{{ + LogQL: `{service_name="dice"}`, + Signal: model.ExpectedSignal{ + NameEquals: "hello", + }, + }}, + }, + } + + c, warnings, err := ConvertDefinition(def, "k8s case") + if err != nil { + fatalf(t, "ConvertDefinition kubernetes: %v", err) + } + if c.Seed.Type != "app" { + fatalf(t, "expected app seed for kubernetes migration, got %+v", c.Seed) + } + joined := strings.Join(warnings, "\n") + for _, want := range []string{ + `[fixture.k8s-case]`, + `type = "k3d"`, + `k8s_dir = "k8s"`, + `app_service = "dice"`, + `app_docker_file = "Dockerfile"`, + `app_docker_context = ".."`, + `app_docker_tag = "dice:test"`, + `app_port = 8080`, + `import_images = ["busybox:latest"]`, + } { + if !strings.Contains(joined, want) { + fatalf(t, "expected kubernetes migration warning to contain %q:\n%s", want, joined) + } + } +} + func fatalf(t *testing.T, format string, args ...any) { t.Helper() t.Fatalf(format, args...) diff --git a/v2case/case.go b/v2case/case.go index 5454aea8..ce8413e8 100644 --- a/v2case/case.go +++ b/v2case/case.go @@ -44,11 +44,11 @@ type Case struct { // Seed declares how a case populates the stack before assertions run. // Exactly one of the fields beyond Type is meaningful per Type value: // -// type: app → Compose is required +// type: app → external suite fixture boots the app; Compose is optional // type: inline-otlp → Traces/Logs/Metrics describe the payload to push type Seed struct { - Type string `yaml:"type"` // "app" or "inline-otlp" - Compose string `yaml:"compose,omitempty"` + Type string `yaml:"type"` // "app" or "inline-otlp" + Compose string `yaml:"compose,omitempty"` // optional legacy shorthand; suite fixture normally owns app boot Traces []SeedTrace `yaml:"traces,omitempty"` Logs []SeedLog `yaml:"logs,omitempty"` Metrics []SeedMetric `yaml:"metrics,omitempty"` @@ -242,9 +242,9 @@ func (c *Case) Validate() error { } switch c.Seed.Type { case "app": - if c.Seed.Compose == "" { - return fmt.Errorf("seed.compose: required when seed.type = app") - } + // App-backed cases are normally booted by the suite fixture declared in + // oats.toml. seed.compose remains accepted as a legacy/migration + // shorthand, but is not required for validation or execution here. case "inline-otlp": if len(c.Seed.Traces)+len(c.Seed.Logs)+len(c.Seed.Metrics) == 0 { return fmt.Errorf("seed: inline-otlp must declare at least one trace, log, or metric") diff --git a/v2case/case_test.go b/v2case/case_test.go index 3544fc7f..2fa6986e 100644 --- a/v2case/case_test.go +++ b/v2case/case_test.go @@ -156,11 +156,11 @@ func TestValidate_UnknownSeedType(t *testing.T) { } } -func TestValidate_AppSeedNeedsCompose(t *testing.T) { +func TestValidate_AppSeedAllowsFixtureManagedBoot(t *testing.T) { c := &Case{OatsVersion: 2, Name: "x", Seed: Seed{Type: "app"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() - if err == nil || !strings.Contains(err.Error(), "seed.compose") { - t.Errorf("expected compose error, got %v", err) + if err != nil { + t.Errorf("expected app seed without compose to validate, got %v", err) } } From a0813cdc039e4ee762b201526ede794e55f13d50 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:24:37 +0000 Subject: [PATCH 030/124] test(v2): cover k3d helper command planning Signed-off-by: Gregor Zeitlinger --- V2.md | 2 +- cmd/v2/main.go | 7 +-- testhelpers/kubernetes/kubernetes.go | 18 ++++-- testhelpers/kubernetes/kubernetes_test.go | 74 +++++++++++++++++++++++ 4 files changed, 90 insertions(+), 11 deletions(-) create mode 100644 testhelpers/kubernetes/kubernetes_test.go diff --git a/V2.md b/V2.md index 9a0bdeca..eba1119b 100644 --- a/V2.md +++ b/V2.md @@ -36,7 +36,7 @@ bin/oats-v2 --format ndjson > events.jsonl ```text discovery → seed → engine → assert → report ↑ - fixture (TODO) + fixture ↑ wait (polling) ↑ diff --git a/cmd/v2/main.go b/cmd/v2/main.go index faf92063..8907638a 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -237,8 +237,7 @@ func verbosityFromInt(n int) report.Verbosity { } // resolveEndpoint maps a fixture config + an explicit override into the -// concrete endpoint the runner needs. The v2 branch ships with "remote" -// support only at this stage; "compose" and "k3d" will land later. +// concrete endpoint the runner needs. func resolveEndpoint(sourceDir string, plan discovery.Plan, gcxContextOverride, appHost string, appPort int, otlpHTTP string) (runner.Endpoint, error) { ep := runner.Endpoint{AppHost: appHost, AppPort: appPort, OTLPHTTP: otlpHTTP} switch plan.Fixture.Type { @@ -259,7 +258,7 @@ func resolveEndpoint(sourceDir string, plan discovery.Plan, gcxContextOverride, // No fixture configured — caller (or --gcx-context) must supply // everything. Useful while plumbing v2 against an external setup. default: - return ep, fmt.Errorf("fixture type %q is not yet supported in oats-v2 (compose/k3d arrive in follow-up commits)", plan.Fixture.Type) + return ep, fmt.Errorf("fixture type %q is not supported in oats-v2", plan.Fixture.Type) } if gcxContextOverride != "" { ep.GCXContext = gcxContextOverride @@ -312,7 +311,7 @@ func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (sui } return endpointFixture{ep: ep}, nil default: - return nil, fmt.Errorf("fixture type %q is not yet supported in oats-v2 (k3d arrives in follow-up commits)", plan.Fixture.Type) + return nil, fmt.Errorf("fixture type %q is not supported in oats-v2", plan.Fixture.Type) } } diff --git a/testhelpers/kubernetes/kubernetes.go b/testhelpers/kubernetes/kubernetes.go index f0dfb041..630fd4d8 100644 --- a/testhelpers/kubernetes/kubernetes.go +++ b/testhelpers/kubernetes/kubernetes.go @@ -24,6 +24,7 @@ type Kubernetes struct { func NewEndpoint(host string, model *Kubernetes, ports remote.PortsConfig, testName string, dir string) *remote.Endpoint { var killList []*os.Process + cluster := clusterName(testName) run := func(cmd *exec.Cmd, background bool) error { slog.Info("running", "command", cmd.String(), "dir", dir) cmd.Stdout = os.Stdout @@ -48,10 +49,10 @@ func NewEndpoint(host string, model *Kubernetes, ports remote.PortsConfig, testN return err } } - return run(exec.Command("k3d", "cluster", "delete", testName), false) + return run(exec.Command("k3d", "cluster", "delete", cluster), false) }, func(f func(io.ReadCloser, *sync.WaitGroup)) error { - panic("not implemented for kubernetes") + return fmt.Errorf("compose log reading is not implemented for kubernetes fixtures") }, ) } @@ -71,10 +72,7 @@ func start(model *Kubernetes, ports remote.PortsConfig, testName string, run fun return err } - cluster := testName - if len(cluster) > 32 { - cluster = cluster[(len(cluster))-32:] - } + cluster := clusterName(testName) err = run(exec.Command("k3d", "cluster", "list", cluster), false) if err == nil { @@ -128,3 +126,11 @@ func start(model *Kubernetes, ports remote.PortsConfig, testName string, run fun } return nil } + +func clusterName(testName string) string { + cluster := testName + if len(cluster) > 32 { + cluster = cluster[len(cluster)-32:] + } + return cluster +} diff --git a/testhelpers/kubernetes/kubernetes_test.go b/testhelpers/kubernetes/kubernetes_test.go new file mode 100644 index 00000000..5681f25c --- /dev/null +++ b/testhelpers/kubernetes/kubernetes_test.go @@ -0,0 +1,74 @@ +package kubernetes + +import ( + "os/exec" + "reflect" + "strings" + "testing" + + "github.com/grafana/oats/testhelpers/remote" +) + +func TestClusterName_TruncatesFromEnd(t *testing.T) { + in := "this-is-a-very-long-suite-name-that-exceeds-thirty-two-chars" + got := clusterName(in) + if len(got) != 32 { + t.Fatalf("cluster length = %d, want 32 (%q)", len(got), got) + } + if !strings.HasSuffix(in, got) { + t.Fatalf("expected %q to be the suffix of %q", got, in) + } +} + +func TestStart_DefaultDockerContextAndCommandSequence(t *testing.T) { + model := &Kubernetes{ + Dir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerTag: "dice:test", + AppDockerPort: 18080, + ImportImages: []string{"busybox:latest"}, + } + ports := remote.PortsConfig{ + PrometheusHTTPPort: 19090, + LokiHttpPort: 13100, + TempoHTTPPort: 13200, + PyroscopeHttpPort: 14040, + } + + var calls []string + run := func(cmd *exec.Cmd, background bool) error { + mode := "fg" + if background { + mode = "bg" + } + calls = append(calls, mode+": "+strings.Join(cmd.Args, " ")) + return nil + } + + if err := start(model, ports, "smoke-suite", run); err != nil { + t.Fatalf("start: %v", err) + } + if model.AppDockerContext != "." { + t.Fatalf("expected default docker context '.', got %q", model.AppDockerContext) + } + + want := []string{ + "fg: docker build -f Dockerfile -t dice:test .", + "fg: k3d cluster list smoke-suite", + "fg: k3d cluster delete smoke-suite", + "fg: k3d cluster create smoke-suite", + "fg: k3d image import -c smoke-suite dice:test", + "fg: k3d image import -c smoke-suite busybox:latest", + "fg: kubectl apply -f k8s", + "fg: kubectl wait --timeout=5m --for=condition=ready pod -l app=lgtm", + "bg: kubectl port-forward service/dice 18080:8080", + "bg: kubectl port-forward service/lgtm 13100:3100", + "bg: kubectl port-forward service/lgtm 19090:9090", + "bg: kubectl port-forward service/lgtm 13200:3200", + "bg: kubectl port-forward service/lgtm 14040:4040", + } + if !reflect.DeepEqual(calls, want) { + t.Fatalf("unexpected command sequence:\n got: %#v\nwant: %#v", calls, want) + } +} From 550c9c61061803d4ebd1f0ddd856a8cbd75f59ba Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:26:41 +0000 Subject: [PATCH 031/124] test(v2): cover fixture startup lifecycle Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 137 +++++++++++++++++++++++++++++++++++++ cmd/v2/main.go | 56 ++++++++++----- 2 files changed, 176 insertions(+), 17 deletions(-) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index ab086946..593c41cd 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "fmt" "net/http" "net/http/httptest" "os" @@ -16,6 +17,7 @@ import ( "github.com/grafana/oats/engine" "github.com/grafana/oats/report" "github.com/grafana/oats/runner" + "github.com/grafana/oats/testhelpers/remote" ) func TestResolveComposeFiles(t *testing.T) { @@ -70,6 +72,112 @@ func TestResolveEndpoint_K3DUsesFixtureAppPort(t *testing.T) { } } +func TestStartFixture_ComposeLifecycle(t *testing.T) { + oldFactory := newComposeSuite + defer func() { newComposeSuite = oldFactory }() + + var gotFiles, gotEnv []string + fake := &fakeSuiteFixture{} + newComposeSuite = func(files []string, env []string) (suiteFixture, error) { + gotFiles = append([]string(nil), files...) + gotEnv = append([]string(nil), env...) + return fake, nil + } + + fix, err := startFixture(context.Background(), "/tmp/work", discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, + Fixture: discovery.FixtureConfig{ + Type: "compose", + ComposeFiles: []string{"a.yml", "b.yml"}, + Env: []string{"FOO=bar"}, + }, + }) + if err != nil { + t.Fatalf("startFixture compose: %v", err) + } + if fake.upCalls != 1 { + t.Fatalf("expected Up once, got %d", fake.upCalls) + } + if want := []string{"/tmp/work/a.yml", "/tmp/work/b.yml"}; !equalStrings(gotFiles, want) { + t.Fatalf("compose files: got %v want %v", gotFiles, want) + } + if want := []string{"FOO=bar"}; !equalStrings(gotEnv, want) { + t.Fatalf("compose env: got %v want %v", gotEnv, want) + } + if err := fix.Close(); err != nil { + t.Fatalf("fixture close: %v", err) + } + if fake.closeCalls != 1 { + t.Fatalf("expected Close once, got %d", fake.closeCalls) + } +} + +func TestStartFixture_ComposeStartFailure(t *testing.T) { + oldFactory := newComposeSuite + defer func() { newComposeSuite = oldFactory }() + + newComposeSuite = func(files []string, env []string) (suiteFixture, error) { + return &fakeSuiteFixture{upErr: fmt.Errorf("boom")}, nil + } + + _, err := startFixture(context.Background(), "/tmp/work", discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, + Fixture: discovery.FixtureConfig{Type: "compose", ComposeFile: "compose.yml"}, + }) + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Fatalf("expected compose startup error, got %v", err) + } +} + +func TestStartFixture_K3DLifecycle(t *testing.T) { + oldFactory := newKubernetesEndpoint + defer func() { newKubernetesEndpoint = oldFactory }() + + var capturedSource string + var capturedPlan discovery.Plan + var starts, stops int + newKubernetesEndpoint = func(sourceDir string, plan discovery.Plan) *remote.Endpoint { + capturedSource = sourceDir + capturedPlan = plan + return remote.NewEndpoint("localhost", remote.PortsConfig{}, func(ctx context.Context) error { + starts++ + return nil + }, func(ctx context.Context) error { + stops++ + return nil + }, nil) + } + + fix, err := startFixture(context.Background(), "/tmp/work", discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, + Fixture: discovery.FixtureConfig{ + Type: "k3d", + K8sDir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerContext: ".", + AppDockerTag: "dice:test", + AppPort: 18080, + ImportImages: []string{"busybox:latest"}, + }, + }) + if err != nil { + t.Fatalf("startFixture k3d: %v", err) + } + if starts != 1 { + t.Fatalf("expected one endpoint start, got %d", starts) + } + if capturedSource != "/tmp/work" || capturedPlan.Suite.Name != "cluster-smoke" || capturedPlan.Fixture.AppPort != 18080 { + t.Fatalf("unexpected endpoint factory args: source=%q plan=%+v", capturedSource, capturedPlan) + } + if err := fix.Close(); err != nil { + t.Fatalf("fixture close: %v", err) + } + if stops != 1 { + t.Fatalf("expected one endpoint stop, got %d", stops) + } +} + // TestIntegration_FullPipelineWithFakeGCX wires the v2 chain end-to-end: // discovery → seed (against an httptest OTLP stub) → engine (against the // fake-gcx.sh shell script) → assertions → report. No real gcx, no real @@ -209,3 +317,32 @@ func rewrite(t *testing.T, path, old, new string) { t.Fatal(err) } } + +type fakeSuiteFixture struct { + upCalls int + closeCalls int + upErr error + closeErr error +} + +func (f *fakeSuiteFixture) Up() error { + f.upCalls++ + return f.upErr +} + +func (f *fakeSuiteFixture) Close() error { + f.closeCalls++ + return f.closeErr +} + +func equalStrings(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} diff --git a/cmd/v2/main.go b/cmd/v2/main.go index 8907638a..07b08e97 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -44,6 +44,29 @@ import ( "github.com/grafana/oats/testhelpers/remote" ) +var ( + newComposeSuite = func(files []string, env []string) (suiteFixture, error) { + return compose.SuiteFiles(files, env) + } + newKubernetesEndpoint = func(sourceDir string, plan discovery.Plan) *remote.Endpoint { + model := &kubernetes.Kubernetes{ + Dir: filepath.Join(sourceDir, plan.Fixture.K8sDir), + AppService: plan.Fixture.AppService, + AppDockerFile: plan.Fixture.AppDockerFile, + AppDockerContext: plan.Fixture.AppDockerContext, + AppDockerTag: plan.Fixture.AppDockerTag, + AppDockerPort: plan.Fixture.AppPort, + ImportImages: plan.Fixture.ImportImages, + } + return kubernetes.NewEndpoint("localhost", model, remote.PortsConfig{ + PrometheusHTTPPort: 9090, + LokiHttpPort: 3100, + TempoHTTPPort: 3200, + PyroscopeHttpPort: 4040, + }, plan.Suite.Name, sourceDir) + } +) + func main() { code := run() os.Exit(code) @@ -273,6 +296,11 @@ type suiteFixture interface { Close() error } +type startableSuiteFixture interface { + suiteFixture + Up() error +} + func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (suiteFixture, error) { switch plan.Fixture.Type { case "", "remote": @@ -282,30 +310,16 @@ func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (sui if err != nil { return nil, err } - suite, err := compose.SuiteFiles(composeFiles, plan.Fixture.Env) + suite, err := newComposeSuite(composeFiles, plan.Fixture.Env) if err != nil { return nil, err } - if err := suite.Up(); err != nil { + if err := startSuiteFixture(suite); err != nil { return nil, err } return suite, nil case "k3d": - model := &kubernetes.Kubernetes{ - Dir: filepath.Join(sourceDir, plan.Fixture.K8sDir), - AppService: plan.Fixture.AppService, - AppDockerFile: plan.Fixture.AppDockerFile, - AppDockerContext: plan.Fixture.AppDockerContext, - AppDockerTag: plan.Fixture.AppDockerTag, - AppDockerPort: plan.Fixture.AppPort, - ImportImages: plan.Fixture.ImportImages, - } - ep := kubernetes.NewEndpoint("localhost", model, remote.PortsConfig{ - PrometheusHTTPPort: 9090, - LokiHttpPort: 3100, - TempoHTTPPort: 3200, - PyroscopeHttpPort: 4040, - }, plan.Suite.Name, sourceDir) + ep := newKubernetesEndpoint(sourceDir, plan) if err := ep.Start(context.Background()); err != nil { return nil, err } @@ -334,6 +348,14 @@ func resolveComposeFiles(sourceDir string, fixture discovery.FixtureConfig) ([]s return files, nil } +func startSuiteFixture(fix suiteFixture) error { + startable, ok := fix.(startableSuiteFixture) + if !ok { + return fmt.Errorf("fixture does not support startup") + } + return startable.Up() +} + type endpointFixture struct { ep *remote.Endpoint } From 939dfd7c5c127a0fa5977732cdb33216e32648f7 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:28:13 +0000 Subject: [PATCH 032/124] test(v2): cover app-backed remote case flow Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 113 +++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 593c41cd..75064f72 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -4,11 +4,13 @@ import ( "bytes" "context" "fmt" + "net" "net/http" "net/http/httptest" "os" "path/filepath" "runtime" + "strconv" "strings" "testing" "time" @@ -296,6 +298,104 @@ expected: } } +func TestIntegration_AppSeedWithRemoteFixtureAndInput(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fake-gcx is a POSIX shell script") + } + + var hits int + app := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + if r.Method != http.MethodPost || r.URL.Path != "/emit" { + t.Fatalf("unexpected app request: %s %s", r.Method, r.URL.Path) + } + w.WriteHeader(http.StatusCreated) + })) + defer app.Close() + appHost, appPort := splitHostPort(t, app.Listener.Addr().String()) + + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "smoke" +cases = ["cases/*.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "http://localhost:4318" +`) + writeFile(t, dir, "cases/app.yaml", `oats: 2 +name: app seed end-to-end +seed: + type: app +input: + - path: /emit + method: POST + status: "201" +expected: + traces: + - traceql: '{ resource.service.name = "gcx-e2e-seed" }' + match: + - name: seed-operation + attributes: + service.name: gcx-e2e-seed + logs: + - logql: '{service_name="gcx-e2e-seed"}' + match: + - name: seed-log-line + attributes: + service_name: gcx-e2e-seed +`) + + cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || len(plans[0].Cases) != 1 { + t.Fatalf("expected one plan with one case, got %+v", plans) + } + + ep, err := resolveEndpoint(dir, plans[0], "", appHost, appPort, "http://localhost:4318") + if err != nil { + t.Fatalf("resolveEndpoint: %v", err) + } + + _, here, _, _ := runtime.Caller(0) + fakeGCX := filepath.Join(filepath.Dir(here), "testdata", "fake-gcx.sh") + exec := &engine.GCX{Binary: fakeGCX, Context: ep.GCXContext} + + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + rep.Emit(report.Event{Type: report.EventRunStart, OatsVersion: "test", SchemaVersion: report.SchemaVersion}) + + r := runner.New(exec, rep, ep, runner.Options{ + Timeout: 500 * time.Millisecond, + Interval: 20 * time.Millisecond, + SeedSettleDelay: 5 * time.Millisecond, + }) + + ok := r.RunCase(context.Background(), plans[0].Cases[0]) + rep.Emit(report.Event{Type: report.EventRunEnd}) + + if !ok { + t.Fatalf("case did not pass:\n%s", buf.String()) + } + if hits == 0 { + t.Fatalf("expected input endpoint to be hit") + } + if !strings.Contains(buf.String(), "PASS 1/1") { + t.Errorf("summary line missing or wrong:\n%s", buf.String()) + } +} + func writeFile(t *testing.T, dir, rel, body string) { t.Helper() p := filepath.Join(dir, rel) @@ -318,6 +418,19 @@ func rewrite(t *testing.T, path, old, new string) { } } +func splitHostPort(t *testing.T, addr string) (string, int) { + t.Helper() + host, portText, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("SplitHostPort(%q): %v", addr, err) + } + port, err := strconv.Atoi(portText) + if err != nil { + t.Fatalf("Atoi(%q): %v", portText, err) + } + return host, port +} + type fakeSuiteFixture struct { upCalls int closeCalls int From b56eb9108545c78b6112036d42c063606b8d72f5 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:30:49 +0000 Subject: [PATCH 033/124] feat(v2): improve matrix migration guidance Signed-off-by: Gregor Zeitlinger --- V2.md | 9 ++-- migrate/migrate.go | 85 ++++++++++++++++++++++++++++++- migrate/migrate_test.go | 110 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 5 deletions(-) diff --git a/V2.md b/V2.md index eba1119b..608aea8f 100644 --- a/V2.md +++ b/V2.md @@ -188,7 +188,8 @@ match: - Full-fidelity `oats migrate` for fixture/input semantics. A best-effort `oats-v2 --migrate ` now converts expectation blocks into the collector-style `match` schema and prints warnings for dropped/unsupported - fields (matrix/custom-checks still need manual follow-up). + fields (custom-checks still need manual follow-up; multi-entry matrix cases + still expand manually). - Hermeticity static-check (runtime check applies). ## Migrating from v1 @@ -205,8 +206,10 @@ It converts legacy `equals` / `regexp` / `attributes` / `attribute-regexp` assertions into v2 `match:` entries and prints warnings for fields that still need manual follow-up. For richer legacy fixtures, the warnings now include ready-to-paste `oats.toml` fixture snippets for -multi-file compose/env and kubernetes→k3d mappings. The v1 binary (`oats`) -and the v2 binary (`oats-v2`) still read different file shapes. +multi-file compose/env and kubernetes→k3d mappings. Single-entry legacy +`matrix:` cases are flattened automatically; multi-entry matrix cases emit +fixture-expansion hints for manual splitting. The v1 binary (`oats`) and the +v2 binary (`oats-v2`) still read different file shapes. ## Verbosity contract diff --git a/migrate/migrate.go b/migrate/migrate.go index 4ef5c139..15990e93 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -3,6 +3,7 @@ package migrate import ( "fmt" "path/filepath" + "regexp" "strings" "github.com/grafana/oats/model" @@ -41,6 +42,7 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, Name: name, Interval: def.Interval, } + selectedMatrix := (*model.Matrix)(nil) if def.Kubernetes != nil { c.Seed.Type = "app" @@ -48,7 +50,25 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, warnings = append(warnings, kubernetesFixtureHint(name, def)) } if len(def.Matrix) > 0 { - warnings = append(warnings, "matrix definitions are not migrated; convert expanded matrix cases manually") + if len(def.Matrix) == 1 { + selectedMatrix = &def.Matrix[0] + c.Name = fmt.Sprintf("%s [%s]", name, selectedMatrix.Name) + warnings = append(warnings, fmt.Sprintf("flattened single matrix entry %q into the migrated case", selectedMatrix.Name)) + if selectedMatrix.DockerCompose != nil { + c.Seed.Type = "app" + c.Seed.Compose = selectedMatrix.DockerCompose.Files[0] + warnings = append(warnings, "single matrix docker-compose fixture selected; paste the suggested [fixture] block below into oats.toml") + warnings = append(warnings, matrixFixtureHint(c.Name, *selectedMatrix)) + } + if selectedMatrix.Kubernetes != nil { + c.Seed.Type = "app" + warnings = append(warnings, "single matrix kubernetes fixture selected; paste the suggested [fixture] block below into oats.toml") + warnings = append(warnings, matrixFixtureHint(c.Name, *selectedMatrix)) + } + } else { + warnings = append(warnings, "matrix definitions are not migrated automatically when multiple entries exist; convert expanded matrix cases manually") + warnings = append(warnings, matrixExpansionHint(name, def.Matrix)) + } } if len(def.Include) > 0 { warnings = append(warnings, "include directives were resolved before migration; output is a flattened case") @@ -70,7 +90,7 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, warnings = append(warnings, "legacy docker-compose fixture uses suite-level config richer than case-level seed.compose; paste the suggested [fixture] block below into oats.toml") warnings = append(warnings, composeFixtureHint(name, def)) } - } else if def.Kubernetes == nil { + } else if def.Kubernetes == nil && selectedMatrix == nil { warnings = append(warnings, "no docker-compose fixture found; defaulting seed.type to inline-otlp placeholder") c.Seed.Type = "inline-otlp" c.Seed.Traces = []v2case.SeedTrace{{Service: "migrated-service", Spans: []v2case.SeedSpan{{Name: "replace-me"}}}} @@ -89,22 +109,34 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, } for _, tr := range def.Expected.Traces { + if !keepForMatrix(tr.Signal.MatrixCondition, selectedMatrix) { + continue + } assertion, ws := convertSignal(tr.TraceQL, tr.Signal) warnings = append(warnings, ws...) c.Expected.Traces = append(c.Expected.Traces, v2case.TraceAssertion{TraceQL: tr.TraceQL, AssertionCommon: assertion}) } for _, lg := range def.Expected.Logs { + if !keepForMatrix(lg.Signal.MatrixCondition, selectedMatrix) { + continue + } assertion, ws := convertSignal(lg.LogQL, lg.Signal) warnings = append(warnings, ws...) c.Expected.Logs = append(c.Expected.Logs, v2case.LogAssertion{LogQL: lg.LogQL, AssertionCommon: assertion}) } for _, m := range def.Expected.Metrics { + if !keepForMatrix(m.MatrixCondition, selectedMatrix) { + continue + } c.Expected.Metrics = append(c.Expected.Metrics, v2case.MetricAssertion{PromQL: m.PromQL, Value: m.Value}) if m.MatrixCondition != "" { warnings = append(warnings, fmt.Sprintf("metric %q matrix-condition dropped", m.PromQL)) } } for _, p := range def.Expected.Profiles { + if !keepForMatrix(p.MatrixCondition, selectedMatrix) { + continue + } var matches []v2case.MatchEntry if p.Flamebearers.NameEquals != "" { matches = append(matches, v2case.MatchEntry{Name: strPtr(p.Flamebearers.NameEquals)}) @@ -191,6 +223,20 @@ func deriveName(path string) string { return strings.TrimSpace(base) } +func keepForMatrix(matrixCondition string, selected *model.Matrix) bool { + if matrixCondition == "" { + return true + } + if selected == nil { + return true + } + re, err := regexp.Compile(matrixCondition) + if err != nil { + return true + } + return re.MatchString(selected.Name) +} + func composeFixtureHint(name string, def model.TestCaseDefinition) string { fixtureName := slug(name) var b strings.Builder @@ -229,6 +275,41 @@ func kubernetesFixtureHint(name string, def model.TestCaseDefinition) string { return strings.TrimRight(b.String(), "\n") } +func matrixFixtureHint(name string, m model.Matrix) string { + def := model.TestCaseDefinition{ + DockerCompose: m.DockerCompose, + Kubernetes: m.Kubernetes, + } + if m.DockerCompose != nil { + return composeFixtureHint(name, def) + } + if m.Kubernetes != nil { + return kubernetesFixtureHint(name, def) + } + return fmt.Sprintf("matrix %q has no fixture override", m.Name) +} + +func matrixExpansionHint(name string, matrix []model.Matrix) string { + var b strings.Builder + fmt.Fprintf(&b, "suggested matrix expansion for %q:\n", name) + for _, m := range matrix { + fmt.Fprintf(&b, "- %s\n", m.Name) + if m.DockerCompose != nil || m.Kubernetes != nil { + fixture := indentLines(matrixFixtureHint(fmt.Sprintf("%s [%s]", name, m.Name), m), " ") + fmt.Fprintf(&b, "%s\n", fixture) + } + } + return strings.TrimRight(b.String(), "\n") +} + +func indentLines(s, prefix string) string { + lines := strings.Split(s, "\n") + for i := range lines { + lines[i] = prefix + lines[i] + } + return strings.Join(lines, "\n") +} + func quotedList(items []string) string { quoted := make([]string, 0, len(items)) for _, item := range items { diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index b3c65ae8..0a984a9a 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -139,6 +139,116 @@ func TestConvertDefinition_KubernetesProducesFixtureHint(t *testing.T) { } } +func TestConvertDefinition_FlattensSingleMatrixEntry(t *testing.T) { + def := model.TestCaseDefinition{ + Matrix: []model.Matrix{{ + Name: "docker", + DockerCompose: &model.DockerCompose{ + Files: []string{"docker-compose.yml"}, + }, + }}, + Expected: model.Expected{ + Logs: []model.ExpectedLogs{ + { + LogQL: `{service_name="dice"}`, + Signal: model.ExpectedSignal{ + NameEquals: "base", + MatrixCondition: "", + }, + }, + { + LogQL: `{service_name="dice"}`, + Signal: model.ExpectedSignal{ + NameEquals: "docker-only", + MatrixCondition: "docker", + }, + }, + { + LogQL: `{service_name="dice"}`, + Signal: model.ExpectedSignal{ + NameEquals: "k8s-only", + MatrixCondition: "k8s", + }, + }, + }, + }, + } + + c, warnings, err := ConvertDefinition(def, "matrix case") + if err != nil { + fatalf(t, "ConvertDefinition single matrix: %v", err) + } + if c.Name != "matrix case [docker]" { + fatalf(t, "unexpected case name: %q", c.Name) + } + if c.Seed.Type != "app" { + fatalf(t, "expected app seed, got %+v", c.Seed) + } + if got := len(c.Expected.Logs); got != 2 { + fatalf(t, "expected 2 kept log assertions, got %d (%+v)", got, c.Expected.Logs) + } + joined := strings.Join(warnings, "\n") + for _, want := range []string{ + `flattened single matrix entry "docker"`, + `[fixture.matrix-case-docker]`, + `compose_file = "docker-compose.yml"`, + } { + if !strings.Contains(joined, want) { + fatalf(t, "expected warning to contain %q:\n%s", want, joined) + } + } +} + +func TestConvertDefinition_MultiMatrixEmitsExpansionHint(t *testing.T) { + def := model.TestCaseDefinition{ + Matrix: []model.Matrix{ + { + Name: "docker", + DockerCompose: &model.DockerCompose{ + Files: []string{"docker-compose.yml"}, + }, + }, + { + Name: "k8s", + Kubernetes: &kubernetes.Kubernetes{ + Dir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerTag: "dice:test", + AppDockerPort: 8080, + }, + }, + }, + Expected: model.Expected{ + Logs: []model.ExpectedLogs{{ + LogQL: `{service_name="dice"}`, + Signal: model.ExpectedSignal{ + NameEquals: "any", + MatrixCondition: "docker|k8s", + }, + }}, + }, + } + + _, warnings, err := ConvertDefinition(def, "matrix case") + if err != nil { + fatalf(t, "ConvertDefinition multi matrix: %v", err) + } + joined := strings.Join(warnings, "\n") + for _, want := range []string{ + `suggested matrix expansion for "matrix case"`, + `- docker`, + `[fixture.matrix-case-docker]`, + `- k8s`, + `[fixture.matrix-case-k8s]`, + `type = "k3d"`, + } { + if !strings.Contains(joined, want) { + fatalf(t, "expected warning to contain %q:\n%s", want, joined) + } + } +} + func fatalf(t *testing.T, format string, args ...any) { t.Helper() t.Fatalf(format, args...) From c895e6b15960bc8e9de4b2d487f8979443796481 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:35:03 +0000 Subject: [PATCH 034/124] feat(v2): add custom check support Signed-off-by: Gregor Zeitlinger --- V2.md | 11 ++++- migrate/migrate.go | 12 +++-- migrate/migrate_test.go | 22 ++++++++++ runner/runner.go | 97 +++++++++++++++++++++++++++++++++++++++++ runner/runner_test.go | 88 +++++++++++++++++++++++++++++++++++++ v2case/case.go | 15 ++++++- v2case/case_test.go | 46 +++++++++++++++++++ 7 files changed, 284 insertions(+), 7 deletions(-) diff --git a/V2.md b/V2.md index 608aea8f..294f409a 100644 --- a/V2.md +++ b/V2.md @@ -118,6 +118,8 @@ expected: - name: "Received request to roll dice" attributes: service_name: dice-server + custom-checks: + - script: ./verify.sh ``` For `seed.type: app`, the application is normally started by the suite fixture in @@ -161,6 +163,12 @@ Per signal under `expected.[]`: | `count` | Comparison against the number of result rows. | | `absent` | If true, the query must return zero rows. | +Per custom check under `expected.custom-checks[]`: + +| Key | Meaning | +|-----|---------| +| `script` | Either an executable path relative to the case yaml, or an inline shell script block. | + When a case declares `input`, the runner makes those HTTP requests before each assertion poll, mirroring OATS v1's “drive the app until telemetry appears” behavior. For remote fixtures, point those requests at a running app with @@ -188,8 +196,7 @@ match: - Full-fidelity `oats migrate` for fixture/input semantics. A best-effort `oats-v2 --migrate ` now converts expectation blocks into the collector-style `match` schema and prints warnings for dropped/unsupported - fields (custom-checks still need manual follow-up; multi-entry matrix cases - still expand manually). + fields (multi-entry matrix cases still expand manually). - Hermeticity static-check (runtime check applies). ## Migrating from v1 diff --git a/migrate/migrate.go b/migrate/migrate.go index 15990e93..bfdce738 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -76,9 +76,6 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, if len(def.Expected.ComposeLogs) > 0 { warnings = append(warnings, "compose-logs assertions are not migrated") } - if len(def.Expected.CustomChecks) > 0 { - warnings = append(warnings, "custom-checks are not migrated") - } if def.DockerCompose != nil { c.Seed.Type = "app" @@ -152,6 +149,15 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, warnings = append(warnings, fmt.Sprintf("profile %q matrix-condition dropped", p.Query)) } } + for _, cc := range def.Expected.CustomChecks { + if !keepForMatrix(cc.MatrixCondition, selectedMatrix) { + continue + } + c.Expected.Custom = append(c.Expected.Custom, v2case.CustomCheck{Script: cc.Script}) + if cc.MatrixCondition != "" { + warnings = append(warnings, "custom-check matrix-condition dropped after filtering selected matrix") + } + } if err := c.Validate(); err != nil { return nil, warnings, fmt.Errorf("migrated v2 case failed validation: %w", err) diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 0a984a9a..e1259ad7 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -249,6 +249,28 @@ func TestConvertDefinition_MultiMatrixEmitsExpansionHint(t *testing.T) { } } +func TestConvertDefinition_MigratesCustomChecks(t *testing.T) { + def := model.TestCaseDefinition{ + Expected: model.Expected{ + CustomChecks: []model.CustomCheck{ + {Script: "./verify.sh"}, + {Script: "./skip.sh", MatrixCondition: "docker"}, + }, + }, + } + + c, warnings, err := ConvertDefinition(def, "custom checks") + if err != nil { + fatalf(t, "ConvertDefinition custom checks: %v", err) + } + if got := len(c.Expected.Custom); got != 2 { + fatalf(t, "expected 2 custom checks, got %+v", c.Expected.Custom) + } + if len(warnings) == 0 || !strings.Contains(strings.Join(warnings, "\n"), "defaulting seed.type to inline-otlp placeholder") { + fatalf(t, "expected placeholder warning, got %v", warnings) + } +} + func fatalf(t *testing.T, format string, args ...any) { t.Helper() t.Fatalf(format, args...) diff --git a/runner/runner.go b/runner/runner.go index 39565916..67924d8d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -7,12 +7,15 @@ package runner import ( + "bytes" "context" "encoding/json" "fmt" "maps" "net/http" "os" + "os/exec" + "path/filepath" "strconv" "strings" "time" @@ -221,6 +224,11 @@ func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { ok = false } } + for i := range c.Expected.Custom { + if !r.runCustomCheck(ctx, c, &c.Expected.Custom[i]) { + ok = false + } + } durMs := time.Since(caseStart).Milliseconds() if ok { @@ -239,6 +247,95 @@ func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { return ok } +func (r *Runner) runCustomCheck(ctx context.Context, c *v2case.Case, chk *v2case.CustomCheck) bool { + dir := "." + if c.SourcePath != "" { + dir = filepath.Dir(c.SourcePath) + } + run := func() []assert.Failure { + if err := r.driveInputs(c); err != nil { + return []assert.Failure{{Rule: "input", Detail: err.Error()}} + } + deadlineCtx, cancel := context.WithTimeout(ctx, r.opts.Timeout) + defer cancel() + + cmd, cleanup, err := customCheckCommand(deadlineCtx, dir, chk.Script) + if err != nil { + return []assert.Failure{{Rule: "custom-check-setup", Detail: err.Error()}} + } + defer cleanup() + + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return []assert.Failure{{Rule: "custom-check", Detail: err.Error() + "\n" + trimOutput(out.String())}} + } + return nil + } + + result := wait.Until[assert.Failure](ctx, wait.Options{Timeout: r.opts.Timeout, Interval: r.caseInterval(c)}, run) + if result.OK { + return true + } + for _, f := range result.LastFailures { + r.reporter.Emit(report.Event{ + Type: report.EventAssertFail, + Case: c.Name, + Source: c.SourcePath, + Msg: f.Error(), + Cmd: "custom-check", + }) + } + return false +} + +func customCheckCommand(ctx context.Context, dir, script string) (*exec.Cmd, func(), error) { + cleanup := func() {} + if strings.TrimSpace(script) == "" { + return nil, cleanup, fmt.Errorf("empty script") + } + if looksLikeInlineScript(script) { + f, err := os.CreateTemp("", "oats-v2-custom-check-*.sh") + if err != nil { + return nil, cleanup, err + } + if _, err := f.WriteString(script); err != nil { + _ = f.Close() + _ = os.Remove(f.Name()) + return nil, cleanup, err + } + if err := f.Close(); err != nil { + _ = os.Remove(f.Name()) + return nil, cleanup, err + } + if err := os.Chmod(f.Name(), 0o700); err != nil { + _ = os.Remove(f.Name()) + return nil, cleanup, err + } + cleanup = func() { _ = os.Remove(f.Name()) } + cmd := exec.CommandContext(ctx, "sh", f.Name()) + cmd.Dir = dir + return cmd, cleanup, nil + } + cmd := exec.CommandContext(ctx, script) + cmd.Dir = dir + return cmd, cleanup, nil +} + +func looksLikeInlineScript(script string) bool { + trimmed := strings.TrimSpace(script) + return strings.Contains(trimmed, "\n") || strings.HasPrefix(trimmed, "#!") +} + +func trimOutput(s string) string { + s = strings.TrimSpace(s) + if len(s) > 4000 { + return s[:4000] + } + return s +} + func (r *Runner) seedCase(c *v2case.Case) error { switch c.Seed.Type { case "app": diff --git a/runner/runner_test.go b/runner/runner_test.go index d980db1c..e9c997a6 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "strconv" "strings" "testing" @@ -303,6 +304,93 @@ expected: } } +func TestRunCase_CustomCheckScriptPath(t *testing.T) { + dir := t.TempDir() + script := filepath.Join(dir, "verify.sh") + if err := os.WriteFile(script, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("WriteFile: %v", err) + } + + c := mustParse(t, ` +oats: 2 +name: custom check path +seed: + type: app +expected: + custom-checks: + - script: ./verify.sh +`) + c.SourcePath = filepath.Join(dir, "case.yaml") + + exec := &stubExec{} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + r := New(exec, rep, Endpoint{GCXContext: "test"}, Options{Timeout: 200 * time.Millisecond, SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("expected custom check path case to pass:\n%s", buf.String()) + } +} + +func TestRunCase_CustomCheckInlineScript(t *testing.T) { + c := mustParse(t, ` +oats: 2 +name: custom check inline +seed: + type: app +expected: + custom-checks: + - script: | + #!/bin/sh + exit 0 +`) + + exec := &stubExec{} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + r := New(exec, rep, Endpoint{GCXContext: "test"}, Options{Timeout: 200 * time.Millisecond, SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("expected inline custom check case to pass:\n%s", buf.String()) + } +} + +func TestRunCase_CustomCheckFailureSurfaced(t *testing.T) { + c := mustParse(t, ` +oats: 2 +name: custom check fail +seed: + type: app +expected: + custom-checks: + - script: | + #!/bin/sh + echo bad + exit 1 +`) + + exec := &stubExec{} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + r := New(exec, rep, Endpoint{GCXContext: "test"}, Options{Timeout: 200 * time.Millisecond, SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + if ok { + t.Fatalf("expected inline custom check case to fail") + } + if !strings.Contains(buf.String(), "custom-check: exit status 1") || !strings.Contains(buf.String(), "bad") { + t.Fatalf("expected custom-check failure output, got:\n%s", buf.String()) + } +} + func TestApproxRowCount(t *testing.T) { cases := []struct { in string diff --git a/v2case/case.go b/v2case/case.go index ce8413e8..55ca8d01 100644 --- a/v2case/case.go +++ b/v2case/case.go @@ -15,6 +15,7 @@ import ( "fmt" "os" "regexp" + "strings" "time" "go.yaml.in/yaml/v3" @@ -99,6 +100,11 @@ type Expected struct { Metrics []MetricAssertion `yaml:"metrics,omitempty"` Logs []LogAssertion `yaml:"logs,omitempty"` Profiles []ProfileAssertion `yaml:"profiles,omitempty"` + Custom []CustomCheck `yaml:"custom-checks,omitempty"` +} + +type CustomCheck struct { + Script string `yaml:"script"` } // AssertionCommon holds the keys every signal-type assertion supports. @@ -254,8 +260,8 @@ func (c *Case) Validate() error { default: return fmt.Errorf("seed.type: unknown value %q (expected app or inline-otlp)", c.Seed.Type) } - if len(c.Expected.Traces)+len(c.Expected.Metrics)+len(c.Expected.Logs)+len(c.Expected.Profiles) == 0 { - return fmt.Errorf("expected: at least one signal assertion required (a case with no expectations cannot fail)") + if len(c.Expected.Traces)+len(c.Expected.Metrics)+len(c.Expected.Logs)+len(c.Expected.Profiles)+len(c.Expected.Custom) == 0 { + return fmt.Errorf("expected: at least one assertion required (signal or custom-check)") } for i, in := range c.Input { if in.Path == "" { @@ -294,6 +300,11 @@ func (c *Case) Validate() error { return err } } + for i := range c.Expected.Custom { + if strings.TrimSpace(c.Expected.Custom[i].Script) == "" { + return fmt.Errorf("expected.custom-checks[%d].script: required, non-empty", i) + } + } return nil } diff --git a/v2case/case_test.go b/v2case/case_test.go index 2fa6986e..f39994a1 100644 --- a/v2case/case_test.go +++ b/v2case/case_test.go @@ -113,6 +113,25 @@ expected: } } +func TestParse_CustomChecks(t *testing.T) { + src := []byte(` +oats: 2 +name: custom checks +seed: + type: app +expected: + custom-checks: + - script: ./verify.sh +`) + c, err := Parse(src) + if err != nil { + t.Fatalf("Parse: %v", err) + } + if got := len(c.Expected.Custom); got != 1 || c.Expected.Custom[0].Script != "./verify.sh" { + t.Fatalf("custom checks: %+v", c.Expected.Custom) + } +} + func TestParse_RejectsUnknownFields(t *testing.T) { src := []byte(` oats: 2 @@ -180,6 +199,33 @@ func TestValidate_NoExpectations(t *testing.T) { } } +func TestValidate_CustomCheckOnlyCase(t *testing.T) { + c := &Case{ + OatsVersion: 2, + Name: "x", + Seed: Seed{Type: "app"}, + Expected: Expected{Custom: []CustomCheck{{Script: "./verify.sh"}}}, + } + if err := c.Validate(); err != nil { + t.Fatalf("expected custom-check-only case to validate, got %v", err) + } +} + +func TestValidate_RejectsEmptyCustomCheck(t *testing.T) { + _, err := Parse([]byte(` +oats: 2 +name: bad custom +seed: + type: app +expected: + custom-checks: + - script: "" +`)) + if err == nil || !strings.Contains(err.Error(), "expected.custom-checks[0].script") { + t.Fatalf("expected custom check error, got %v", err) + } +} + func TestValidate_RejectsInputWithoutPath(t *testing.T) { _, err := Parse([]byte(` oats: 2 From 25bb20c77142393573171e89c423919c66bb1cc2 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:36:37 +0000 Subject: [PATCH 035/124] docs(v2): expand smoke examples Signed-off-by: Gregor Zeitlinger --- V2.md | 5 +++++ discovery/discovery_test.go | 14 +++++++++----- examples/v2-smoke/cases/custom-check.yaml | 8 ++++++++ examples/v2-smoke/cases/inline-seed.yaml | 16 ++++++++++++++++ examples/v2-smoke/scripts/verify.sh | 2 ++ 5 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 examples/v2-smoke/cases/custom-check.yaml create mode 100644 examples/v2-smoke/cases/inline-seed.yaml create mode 100755 examples/v2-smoke/scripts/verify.sh diff --git a/V2.md b/V2.md index 294f409a..ad82bb53 100644 --- a/V2.md +++ b/V2.md @@ -31,6 +31,11 @@ bin/oats-v2 -v=3 # adds fixture lifecycle and full gcx stdout bin/oats-v2 --format ndjson > events.jsonl ``` +The repo includes a small reference config under `examples/v2-smoke/` showing: +- an app-backed case with `input` +- an inline-OTLP case +- a `custom-checks` case + ## Architecture ```text diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 3e07b101..23b196f8 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -262,11 +262,15 @@ func TestExampleV2SmokeConfigLoads(t *testing.T) { if err != nil { t.Fatalf("PlanRun example config: %v", err) } - if len(plans) != 1 || len(plans[0].Cases) != 1 { - t.Fatalf("expected one suite/one case, got %+v", plans) - } - if plans[0].Cases[0].Name != "rolldice smoke" { - t.Fatalf("unexpected case name: %q", plans[0].Cases[0].Name) + if len(plans) != 1 || len(plans[0].Cases) != 3 { + t.Fatalf("expected one suite/three cases, got %+v", plans) + } + got := []string{plans[0].Cases[0].Name, plans[0].Cases[1].Name, plans[0].Cases[2].Name} + want := []string{"custom check smoke", "inline seed smoke", "rolldice smoke"} + for i := range want { + if got[i] != want[i] { + t.Fatalf("unexpected case order: got %v want %v", got, want) + } } } diff --git a/examples/v2-smoke/cases/custom-check.yaml b/examples/v2-smoke/cases/custom-check.yaml new file mode 100644 index 00000000..0990fa8b --- /dev/null +++ b/examples/v2-smoke/cases/custom-check.yaml @@ -0,0 +1,8 @@ +oats: 2 +name: custom check smoke +seed: + type: app +expected: + custom-checks: + - script: ../scripts/verify.sh + diff --git a/examples/v2-smoke/cases/inline-seed.yaml b/examples/v2-smoke/cases/inline-seed.yaml new file mode 100644 index 00000000..15f10bd3 --- /dev/null +++ b/examples/v2-smoke/cases/inline-seed.yaml @@ -0,0 +1,16 @@ +oats: 2 +name: inline seed smoke +seed: + type: inline-otlp + traces: + - service: gcx-e2e-seed + spans: + - name: seed-operation +expected: + traces: + - traceql: '{ resource.service.name = "gcx-e2e-seed" }' + match: + - name: seed-operation + attributes: + service.name: gcx-e2e-seed + diff --git a/examples/v2-smoke/scripts/verify.sh b/examples/v2-smoke/scripts/verify.sh new file mode 100755 index 00000000..039e4d00 --- /dev/null +++ b/examples/v2-smoke/scripts/verify.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 From 81c62955afde9a59de45e7fafcf3098819b8b3f1 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:38:05 +0000 Subject: [PATCH 036/124] test(v2): cover richer migration samples Signed-off-by: Gregor Zeitlinger --- migrate/migrate_test.go | 68 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index e1259ad7..28084529 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -1,12 +1,15 @@ package migrate import ( + "os" + "path/filepath" "strings" "testing" "time" "github.com/grafana/oats/model" "github.com/grafana/oats/testhelpers/kubernetes" + "github.com/grafana/oats/v2case" ) func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { @@ -93,6 +96,71 @@ func TestConvertFile_RendersYAML(t *testing.T) { } } +func TestConvertFile_MatrixSampleIsParseable(t *testing.T) { + out, warnings, err := ConvertFile("/home/gregor/source/oats-v2/yaml/testdata/valid-tests/matrix-test.oats.yaml") + if err != nil { + fatalf(t, "ConvertFile matrix sample: %v", err) + } + joined := strings.Join(warnings, "\n") + for _, want := range []string{ + `matrix definitions are not migrated automatically when multiple entries exist`, + `suggested matrix expansion for "matrix test.oats"`, + `[fixture.matrix-test-oats-docker]`, + `[fixture.matrix-test-oats-k8s]`, + } { + if !strings.Contains(joined, want) { + fatalf(t, "expected matrix warning to contain %q:\n%s", want, joined) + } + } + c, err := v2case.Parse(out) + if err != nil { + fatalf(t, "migrated matrix yaml should parse as v2: %v\n%s", err, string(out)) + } + if c.Seed.Type != "inline-otlp" { + fatalf(t, "expected multi-matrix sample to fall back to inline-otlp placeholder, got %+v", c.Seed) + } + if len(c.Input) == 0 || len(c.Expected.Traces) == 0 || len(c.Expected.Metrics) == 0 { + fatalf(t, "expected included template fields to survive migration, got %+v", c) + } +} + +func TestConvertFile_CustomChecksRoundTrip(t *testing.T) { + dir := t.TempDir() + legacy := filepath.Join(dir, "custom.oats.yaml") + if err := os.WriteFile(legacy, []byte(` +oats-schema-version: 2 +docker-compose: + files: + - docker-compose.yml +expected: + custom-checks: + - script: ./verify.sh + - script: | + #!/bin/sh + exit 0 +`), 0o644); err != nil { + fatalf(t, "WriteFile: %v", err) + } + + out, warnings, err := ConvertFile(legacy) + if err != nil { + fatalf(t, "ConvertFile custom checks: %v", err) + } + if len(warnings) != 0 { + fatalf(t, "expected no warnings for straightforward custom checks, got %v", warnings) + } + c, err := v2case.Parse(out) + if err != nil { + fatalf(t, "migrated custom-check yaml should parse as v2: %v\n%s", err, string(out)) + } + if got := len(c.Expected.Custom); got != 2 { + fatalf(t, "expected 2 migrated custom checks, got %+v", c.Expected.Custom) + } + if c.Expected.Custom[0].Script != "./verify.sh" || !strings.Contains(c.Expected.Custom[1].Script, "exit 0") { + fatalf(t, "unexpected custom check migration: %+v", c.Expected.Custom) + } +} + func TestConvertDefinition_KubernetesProducesFixtureHint(t *testing.T) { def := model.TestCaseDefinition{ Kubernetes: &kubernetes.Kubernetes{ From 711099421749098f7ebfbb5fcf9ae3ff2d46c337 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:39:33 +0000 Subject: [PATCH 037/124] test(v2): run migrated legacy case end to end Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 89 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 75064f72..2942e57e 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -17,6 +17,7 @@ import ( "github.com/grafana/oats/discovery" "github.com/grafana/oats/engine" + "github.com/grafana/oats/migrate" "github.com/grafana/oats/report" "github.com/grafana/oats/runner" "github.com/grafana/oats/testhelpers/remote" @@ -396,6 +397,94 @@ expected: } } +func TestIntegration_MigratedLegacyCaseRuns(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fake-gcx is a POSIX shell script") + } + + legacyPath := filepath.Join(t.TempDir(), "legacy.oats.yaml") + writeFile(t, filepath.Dir(legacyPath), filepath.Base(legacyPath), ` +oats-schema-version: 2 +docker-compose: + files: + - docker-compose.yml +expected: + traces: + - traceql: '{ resource.service.name = "gcx-e2e-seed" }' + equals: seed-operation + attributes: + service.name: gcx-e2e-seed + logs: + - logql: '{service_name="gcx-e2e-seed"}' + equals: seed-log-line + attributes: + service_name: gcx-e2e-seed + metrics: + - promql: 'seed_counter_total{service_name="gcx-e2e-seed"}' + value: ">= 0" +`) + migrated, _, err := migrate.ConvertFile(legacyPath) + if err != nil { + t.Fatalf("ConvertFile: %v", err) + } + + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "migrated" +cases = ["cases/*.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "http://localhost:4318" +`) + writeFile(t, dir, "cases/migrated.yaml", string(migrated)) + + cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || len(plans[0].Cases) != 1 { + t.Fatalf("expected one plan with one case, got %+v", plans) + } + + ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + if err != nil { + t.Fatalf("resolveEndpoint: %v", err) + } + + _, here, _, _ := runtime.Caller(0) + fakeGCX := filepath.Join(filepath.Dir(here), "testdata", "fake-gcx.sh") + exec := &engine.GCX{Binary: fakeGCX, Context: ep.GCXContext} + + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + rep.Emit(report.Event{Type: report.EventRunStart, OatsVersion: "test", SchemaVersion: report.SchemaVersion}) + + r := runner.New(exec, rep, ep, runner.Options{ + Timeout: 500 * time.Millisecond, + Interval: 20 * time.Millisecond, + SeedSettleDelay: 5 * time.Millisecond, + }) + + ok := r.RunCase(context.Background(), plans[0].Cases[0]) + rep.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("migrated case did not pass:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), "PASS 1/1") { + t.Fatalf("summary line missing or wrong:\n%s", buf.String()) + } +} + func writeFile(t *testing.T, dir, rel, body string) { t.Helper() p := filepath.Join(dir, rel) From d075e25067762e5c1285ad350abcc55c2706cdba Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:40:51 +0000 Subject: [PATCH 038/124] test(v2): run migrated custom check end to end Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 76 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 2942e57e..70dc2f7a 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -485,6 +485,82 @@ endpoint = "http://localhost:4318" } } +func TestIntegration_MigratedLegacyCustomCheckRuns(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell-script custom checks are POSIX-oriented") + } + + legacyPath := filepath.Join(t.TempDir(), "legacy-custom.oats.yaml") + writeFile(t, filepath.Dir(legacyPath), filepath.Base(legacyPath), ` +oats-schema-version: 2 +docker-compose: + files: + - docker-compose.yml +expected: + custom-checks: + - script: | + #!/bin/sh + exit 0 +`) + migrated, _, err := migrate.ConvertFile(legacyPath) + if err != nil { + t.Fatalf("ConvertFile: %v", err) + } + + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "migrated-custom" +cases = ["cases/*.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "http://localhost:4318" +`) + writeFile(t, dir, "cases/migrated.yaml", string(migrated)) + + cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || len(plans[0].Cases) != 1 { + t.Fatalf("expected one plan with one case, got %+v", plans) + } + + ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + if err != nil { + t.Fatalf("resolveEndpoint: %v", err) + } + + exec := &engine.GCX{Binary: "does-not-run", Context: ep.GCXContext} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + rep.Emit(report.Event{Type: report.EventRunStart, OatsVersion: "test", SchemaVersion: report.SchemaVersion}) + + r := runner.New(exec, rep, ep, runner.Options{ + Timeout: 500 * time.Millisecond, + Interval: 20 * time.Millisecond, + SeedSettleDelay: 5 * time.Millisecond, + }) + + ok := r.RunCase(context.Background(), plans[0].Cases[0]) + rep.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("migrated custom-check case did not pass:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), "PASS 1/1") { + t.Fatalf("summary line missing or wrong:\n%s", buf.String()) + } +} + func writeFile(t *testing.T, dir, rel, body string) { t.Helper() p := filepath.Join(dir, rel) From df7b7c0ee870b4797819511887b0cc8b9ae7a5f3 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:42:14 +0000 Subject: [PATCH 039/124] test(v2): run flattened matrix migration end to end Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 100 +++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 70dc2f7a..357cc430 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -561,6 +561,106 @@ endpoint = "http://localhost:4318" } } +func TestIntegration_MigratedSingleMatrixCaseRuns(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fake-gcx is a POSIX shell script") + } + + legacyPath := filepath.Join(t.TempDir(), "legacy-matrix.oats.yaml") + writeFile(t, filepath.Dir(legacyPath), filepath.Base(legacyPath), ` +oats-schema-version: 2 +matrix: + - name: docker + docker-compose: + files: + - docker-compose.yml +expected: + traces: + - traceql: '{ resource.service.name = "gcx-e2e-seed" }' + equals: seed-operation + attributes: + service.name: gcx-e2e-seed + matrix-condition: docker + - traceql: '{ resource.service.name = "gcx-e2e-seed" }' + equals: should-not-run + matrix-condition: k8s + logs: + - logql: '{service_name="gcx-e2e-seed"}' + equals: seed-log-line + attributes: + service_name: gcx-e2e-seed + matrix-condition: docker +`) + migrated, warnings, err := migrate.ConvertFile(legacyPath) + if err != nil { + t.Fatalf("ConvertFile: %v", err) + } + if joined := strings.Join(warnings, "\n"); !strings.Contains(joined, `flattened single matrix entry "docker"`) { + t.Fatalf("expected flattening warning, got: %v", warnings) + } + + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "migrated-matrix" +cases = ["cases/*.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "http://localhost:4318" +`) + writeFile(t, dir, "cases/migrated.yaml", string(migrated)) + + cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || len(plans[0].Cases) != 1 { + t.Fatalf("expected one plan with one case, got %+v", plans) + } + if got := plans[0].Cases[0].Name; got != "legacy matrix.oats [docker]" { + t.Fatalf("unexpected migrated case name: %q", got) + } + if got := len(plans[0].Cases[0].Expected.Traces); got != 1 { + t.Fatalf("expected k8s-only assertion to be filtered out, got %d traces", got) + } + + ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + if err != nil { + t.Fatalf("resolveEndpoint: %v", err) + } + + _, here, _, _ := runtime.Caller(0) + fakeGCX := filepath.Join(filepath.Dir(here), "testdata", "fake-gcx.sh") + exec := &engine.GCX{Binary: fakeGCX, Context: ep.GCXContext} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + rep.Emit(report.Event{Type: report.EventRunStart, OatsVersion: "test", SchemaVersion: report.SchemaVersion}) + + r := runner.New(exec, rep, ep, runner.Options{ + Timeout: 500 * time.Millisecond, + Interval: 20 * time.Millisecond, + SeedSettleDelay: 5 * time.Millisecond, + }) + + ok := r.RunCase(context.Background(), plans[0].Cases[0]) + rep.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("migrated matrix case did not pass:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), "PASS 1/1") { + t.Fatalf("summary line missing or wrong:\n%s", buf.String()) + } +} + func writeFile(t *testing.T, dir, rel, body string) { t.Helper() p := filepath.Join(dir, rel) From a4cdfc9574145f67a6754f11570626feb45d69b7 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:43:42 +0000 Subject: [PATCH 040/124] fix(v2): resolve custom check paths explicitly Signed-off-by: Gregor Zeitlinger --- V2.md | 2 +- runner/runner.go | 16 +++++++++++++++- runner/runner_test.go | 19 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/V2.md b/V2.md index ad82bb53..8958ab30 100644 --- a/V2.md +++ b/V2.md @@ -172,7 +172,7 @@ Per custom check under `expected.custom-checks[]`: | Key | Meaning | |-----|---------| -| `script` | Either an executable path relative to the case yaml, or an inline shell script block. | +| `script` | Either an executable path resolved relative to the case yaml, or an inline shell script block. | When a case declares `input`, the runner makes those HTTP requests before each assertion poll, mirroring OATS v1's “drive the app until telemetry appears” diff --git a/runner/runner.go b/runner/runner.go index 67924d8d..2fccf07d 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -318,7 +318,7 @@ func customCheckCommand(ctx context.Context, dir, script string) (*exec.Cmd, fun cmd.Dir = dir return cmd, cleanup, nil } - cmd := exec.CommandContext(ctx, script) + cmd := exec.CommandContext(ctx, resolveCustomCheckPath(dir, script)) cmd.Dir = dir return cmd, cleanup, nil } @@ -328,6 +328,20 @@ func looksLikeInlineScript(script string) bool { return strings.Contains(trimmed, "\n") || strings.HasPrefix(trimmed, "#!") } +func resolveCustomCheckPath(dir, script string) string { + script = strings.TrimSpace(script) + if script == "" { + return script + } + if filepath.IsAbs(script) { + return script + } + if strings.ContainsRune(script, os.PathSeparator) { + return filepath.Clean(filepath.Join(dir, script)) + } + return script +} + func trimOutput(s string) string { s = strings.TrimSpace(s) if len(s) > 4000 { diff --git a/runner/runner_test.go b/runner/runner_test.go index e9c997a6..e40e3d6c 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -391,6 +391,25 @@ expected: } } +func TestResolveCustomCheckPath(t *testing.T) { + dir := "/tmp/cases" + cases := []struct { + name string + script string + want string + }{ + {name: "bare command stays bare", script: "verify", want: "verify"}, + {name: "relative path resolved against case dir", script: "./scripts/verify.sh", want: filepath.Clean("/tmp/cases/scripts/verify.sh")}, + {name: "parent path resolved against case dir", script: "../verify.sh", want: filepath.Clean("/tmp/verify.sh")}, + {name: "absolute path preserved", script: "/opt/check.sh", want: "/opt/check.sh"}, + } + for _, tc := range cases { + if got := resolveCustomCheckPath(dir, tc.script); got != tc.want { + t.Fatalf("%s: got %q want %q", tc.name, got, tc.want) + } + } +} + func TestApproxRowCount(t *testing.T) { cases := []struct { in string From 1882623d545f79de876188b22aa27fde7f662b8d Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:44:52 +0000 Subject: [PATCH 041/124] docs(v2): add top-level quick pointer Signed-off-by: Gregor Zeitlinger --- README.md | 14 ++++++++++++++ V2.md | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bf64cf9a..bcbf041c 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,20 @@ > shape. The newer gcx-driven v2 work lives behind `cmd/v2` / `oats-v2`; see > [V2.md](V2.md) for the current v2 syntax, fixture model, and migration path. +## V2 quick pointer + +If you want the newer gcx-driven flow today: + +```sh +go build -o bin/oats-v2 ./cmd/v2 +bin/oats-v2 --config examples/v2-smoke/oats.toml --list +``` + +Useful v2 entry points: +- syntax and feature status: [V2.md](V2.md) +- small runnable examples: [`examples/v2-smoke/`](examples/v2-smoke/) +- best-effort legacy migration: `bin/oats-v2 --migrate path/to/oats.yaml` + OpenTelemetry Acceptance Tests (OATs), or OATs for short, is a test framework for OpenTelemetry. - Declarative tests written in YAML diff --git a/V2.md b/V2.md index 8958ab30..5bc2c03b 100644 --- a/V2.md +++ b/V2.md @@ -194,7 +194,7 @@ match: http.route: "^/roll.*$" ``` -## What's not in v2-alpha yet +## What's not in v2 yet - Parallel cases within a suite. - k3d pool warmup / reuse. From b6f75923f998c56cc6fb0544a0d0d351ed30e1d2 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:46:26 +0000 Subject: [PATCH 042/124] docs(v2): add richer fixture examples Signed-off-by: Gregor Zeitlinger --- README.md | 1 + V2.md | 4 +++ discovery/discovery_test.go | 23 ++++++++++++++ .../v2-fixtures/cases/compose/rolldice.yaml | 18 +++++++++++ examples/v2-fixtures/cases/k3d/rolldice.yaml | 13 ++++++++ examples/v2-fixtures/oats.toml | 30 +++++++++++++++++++ examples/v2-fixtures/scripts/verify.sh | 2 ++ 7 files changed, 91 insertions(+) create mode 100644 examples/v2-fixtures/cases/compose/rolldice.yaml create mode 100644 examples/v2-fixtures/cases/k3d/rolldice.yaml create mode 100644 examples/v2-fixtures/oats.toml create mode 100755 examples/v2-fixtures/scripts/verify.sh diff --git a/README.md b/README.md index bcbf041c..9df25454 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ bin/oats-v2 --config examples/v2-smoke/oats.toml --list Useful v2 entry points: - syntax and feature status: [V2.md](V2.md) - small runnable examples: [`examples/v2-smoke/`](examples/v2-smoke/) +- richer fixture examples: [`examples/v2-fixtures/`](examples/v2-fixtures/) - best-effort legacy migration: `bin/oats-v2 --migrate path/to/oats.yaml` OpenTelemetry Acceptance Tests (OATs), or OATs for short, is a test framework for OpenTelemetry. diff --git a/V2.md b/V2.md index 5bc2c03b..69983bee 100644 --- a/V2.md +++ b/V2.md @@ -36,6 +36,10 @@ The repo includes a small reference config under `examples/v2-smoke/` showing: - an inline-OTLP case - a `custom-checks` case +For richer fixture examples, `examples/v2-fixtures/` shows: +- multi-file compose fixtures with env passthrough +- k3d fixture config with app build/import fields + ## Architecture ```text diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 23b196f8..91c8e9b1 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -274,6 +274,29 @@ func TestExampleV2SmokeConfigLoads(t *testing.T) { } } +func TestExampleV2FixturesConfigLoads(t *testing.T) { + cfg, err := Load(filepath.Join("..", "examples", "v2-fixtures", "oats.toml")) + if err != nil { + t.Fatalf("Load fixture example config: %v", err) + } + plans, err := cfg.PlanRun(Filter{}) + if err != nil { + t.Fatalf("PlanRun fixture example config: %v", err) + } + if len(plans) != 2 { + t.Fatalf("expected two suites, got %+v", plans) + } + if plans[0].Suite.Name != "compose-app" || plans[0].Fixture.Type != "compose" || len(plans[0].Cases) != 1 { + t.Fatalf("unexpected first plan: %+v", plans[0]) + } + if plans[1].Suite.Name != "k3d-app" || plans[1].Fixture.Type != "k3d" || len(plans[1].Cases) != 1 { + t.Fatalf("unexpected second plan: %+v", plans[1]) + } + if plans[1].Cases[0].Name != "k3d fixture app smoke" { + t.Fatalf("unexpected k3d case name: %q", plans[1].Cases[0].Name) + } +} + func planNames(p []Plan) []string { out := make([]string, len(p)) for i := range p { diff --git a/examples/v2-fixtures/cases/compose/rolldice.yaml b/examples/v2-fixtures/cases/compose/rolldice.yaml new file mode 100644 index 00000000..26f39969 --- /dev/null +++ b/examples/v2-fixtures/cases/compose/rolldice.yaml @@ -0,0 +1,18 @@ +oats: 2 +name: compose fixture app smoke +seed: + type: app +input: + - path: /rolldice?rolls=3 +expected: + traces: + - traceql: '{ resource.service.name = "rolldice" }' + match: + - match_type: regexp + name: '^GET /rolldice.*$' + logs: + - logql: '{service_name="rolldice"}' + match: + - match_type: regexp + name: '.*roll.*' + diff --git a/examples/v2-fixtures/cases/k3d/rolldice.yaml b/examples/v2-fixtures/cases/k3d/rolldice.yaml new file mode 100644 index 00000000..210a35c5 --- /dev/null +++ b/examples/v2-fixtures/cases/k3d/rolldice.yaml @@ -0,0 +1,13 @@ +oats: 2 +name: k3d fixture app smoke +seed: + type: app +input: + - path: /rolldice?rolls=2 +expected: + metrics: + - promql: 'http_server_active_requests{service_name="rolldice"}' + value: '>= 0' + custom-checks: + - script: ../../scripts/verify.sh + diff --git a/examples/v2-fixtures/oats.toml b/examples/v2-fixtures/oats.toml new file mode 100644 index 00000000..f6e9556c --- /dev/null +++ b/examples/v2-fixtures/oats.toml @@ -0,0 +1,30 @@ +[meta] +version = 2 + +[[suite]] +name = "compose-app" +cases = ["cases/compose/*.yaml"] +fixture = "compose-dev" +tags = ["compose", "app"] + +[[suite]] +name = "k3d-app" +cases = ["cases/k3d/*.yaml"] +fixture = "k3d-dev" +tags = ["k3d", "app"] + +[fixture.compose-dev] +type = "compose" +compose_files = ["docker-compose.yml", "docker-compose.override.yml"] +env = ["OTEL_EXPORTER_OTLP_ENDPOINT=http://lgtm:4318"] + +[fixture.k3d-dev] +type = "k3d" +k8s_dir = "k8s" +app_service = "rolldice" +app_docker_file = "Dockerfile" +app_docker_context = "." +app_docker_tag = "rolldice:test" +app_port = 18080 +import_images = ["grafana/otel-lgtm:latest"] + diff --git a/examples/v2-fixtures/scripts/verify.sh b/examples/v2-fixtures/scripts/verify.sh new file mode 100755 index 00000000..039e4d00 --- /dev/null +++ b/examples/v2-fixtures/scripts/verify.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 From 3fd20f262fd9c7cf47d2e1197330383e2c6a0bca Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:48:26 +0000 Subject: [PATCH 043/124] test(v2): run migrated custom check path end to end Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 78 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 357cc430..f8dc888f 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -561,6 +561,84 @@ endpoint = "http://localhost:4318" } } +func TestIntegration_MigratedLegacyCustomCheckRelativePathRuns(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell-script custom checks are POSIX-oriented") + } + + legacyPath := filepath.Join(t.TempDir(), "legacy-custom-path.oats.yaml") + writeFile(t, filepath.Dir(legacyPath), filepath.Base(legacyPath), ` +oats-schema-version: 2 +docker-compose: + files: + - docker-compose.yml +expected: + custom-checks: + - script: ./scripts/verify.sh +`) + migrated, _, err := migrate.ConvertFile(legacyPath) + if err != nil { + t.Fatalf("ConvertFile: %v", err) + } + + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "migrated-custom-path" +cases = ["cases/*.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "http://localhost:4318" +`) + writeFile(t, dir, "cases/migrated.yaml", string(migrated)) + writeFile(t, dir, "cases/scripts/verify.sh", "#!/bin/sh\nexit 0\n") + if err := os.Chmod(filepath.Join(dir, "cases", "scripts", "verify.sh"), 0o755); err != nil { + t.Fatalf("Chmod: %v", err) + } + + cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || len(plans[0].Cases) != 1 { + t.Fatalf("expected one plan with one case, got %+v", plans) + } + + ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + if err != nil { + t.Fatalf("resolveEndpoint: %v", err) + } + + exec := &engine.GCX{Binary: "does-not-run", Context: ep.GCXContext} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + rep.Emit(report.Event{Type: report.EventRunStart, OatsVersion: "test", SchemaVersion: report.SchemaVersion}) + + r := runner.New(exec, rep, ep, runner.Options{ + Timeout: 500 * time.Millisecond, + Interval: 20 * time.Millisecond, + SeedSettleDelay: 5 * time.Millisecond, + }) + + ok := r.RunCase(context.Background(), plans[0].Cases[0]) + rep.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("migrated custom-check relative-path case did not pass:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), "PASS 1/1") { + t.Fatalf("summary line missing or wrong:\n%s", buf.String()) + } +} + func TestIntegration_MigratedSingleMatrixCaseRuns(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("fake-gcx is a POSIX shell script") From 59809ba2a07e8a7db3ef69381c7fbcba1f0ffd7f Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:50:22 +0000 Subject: [PATCH 044/124] feat(v2): emit fixture lifecycle events Signed-off-by: Gregor Zeitlinger --- cmd/v2/main.go | 27 +++++++++++++++++++++++++++ report/report_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/cmd/v2/main.go b/cmd/v2/main.go index 07b08e97..eadfa690 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -152,11 +152,30 @@ func run() int { var totalPass, totalFail int for _, plan := range plans { + fixtureStart := time.Now() + if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { + rep.Emit(report.Event{ + Type: report.EventFixtureStart, + Suite: plan.Suite.Name, + Fixture: plan.Suite.Fixture, + FixtureType: plan.Fixture.Type, + Ts: fixtureStart, + }) + } fix, err := startFixture(ctx, cfg.SourceDir, plan) if err != nil { fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) return 2 } + if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { + rep.Emit(report.Event{ + Type: report.EventFixtureReady, + Suite: plan.Suite.Name, + Fixture: plan.Suite.Fixture, + FixtureType: plan.Fixture.Type, + DurationMs: time.Since(fixtureStart).Milliseconds(), + }) + } ep, err := resolveEndpoint(cfg.SourceDir, plan, *gcxContextOverride, *appHost, *appPort, *otlpHTTP) if err != nil { if fix != nil { @@ -217,10 +236,18 @@ func run() int { Fail: suiteFail, }) if fix != nil { + teardownStart := time.Now() if closeErr := fix.Close(); closeErr != nil { fmt.Fprintf(os.Stderr, "suite %q: fixture shutdown: %v\n", plan.Suite.Name, closeErr) return 2 } + rep.Emit(report.Event{ + Type: report.EventFixtureTeardown, + Suite: plan.Suite.Name, + Fixture: plan.Suite.Fixture, + FixtureType: plan.Fixture.Type, + DurationMs: time.Since(teardownStart).Milliseconds(), + }) } } diff --git a/report/report_test.go b/report/report_test.go index f80902f1..fdaed561 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -99,6 +99,25 @@ func TestTextReporter_VerbosePassPrintsPasses(t *testing.T) { } } +func TestTextReporter_VerboseAllPrintsFixtureLifecycle(t *testing.T) { + var buf bytes.Buffer + r := NewTextReporter(&buf, VerboseAll) + r.Emit(Event{Type: EventFixtureStart, Fixture: "local", DurationMs: 1}) + r.Emit(Event{Type: EventFixtureReady, Fixture: "local", DurationMs: 12}) + r.Emit(Event{Type: EventFixtureTeardown, Fixture: "local", DurationMs: 3}) + + out := buf.String() + for _, want := range []string{ + "[fixture local] fixture.start", + "[fixture local] fixture.ready", + "[fixture local] fixture.teardown", + } { + if !strings.Contains(out, want) { + t.Fatalf("expected fixture lifecycle line %q in:\n%s", want, out) + } + } +} + func TestNDJSONReporter_EmitsOneJSONObjectPerLine(t *testing.T) { var buf bytes.Buffer r := NewNDJSONReporter(&buf, VerboseDefault) @@ -132,6 +151,21 @@ func TestNDJSONReporter_FiltersPassByDefault(t *testing.T) { } } +func TestNDJSONReporter_EmitsFixtureLifecycleAtVerboseAll(t *testing.T) { + var buf bytes.Buffer + r := NewNDJSONReporter(&buf, VerboseAll) + r.Emit(Event{Type: EventFixtureStart, Fixture: "local"}) + r.Emit(Event{Type: EventFixtureReady, Fixture: "local"}) + r.Emit(Event{Type: EventFixtureTeardown, Fixture: "local"}) + + out := buf.String() + for _, want := range []string{`"fixture.start"`, `"fixture.ready"`, `"fixture.teardown"`} { + if !strings.Contains(out, want) { + t.Fatalf("expected %s in NDJSON output:\n%s", want, out) + } + } +} + func TestSplitSource(t *testing.T) { cases := []struct { in string From bddfda3122ed1111fa1561ea76938ff338228600 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:52:27 +0000 Subject: [PATCH 045/124] fix(v2): emit teardown events on early close Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 49 ++++++++++++++++++++++++ cmd/v2/main.go | 76 +++++++++++++++++++++++--------------- 2 files changed, 96 insertions(+), 29 deletions(-) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index f8dc888f..9ede9472 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -181,6 +181,45 @@ func TestStartFixture_K3DLifecycle(t *testing.T) { } } +func TestCloseFixture_EmitsTeardownEvent(t *testing.T) { + rep := &recordingReporter{} + fix := &fakeSuiteFixture{} + plan := discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, + Fixture: discovery.FixtureConfig{Type: "compose"}, + } + if err := closeFixture(rep, plan, fix); err != nil { + t.Fatalf("closeFixture: %v", err) + } + if fix.closeCalls != 1 { + t.Fatalf("expected Close once, got %d", fix.closeCalls) + } + if len(rep.events) != 1 || rep.events[0].Type != report.EventFixtureTeardown { + t.Fatalf("expected one teardown event, got %+v", rep.events) + } + if rep.events[0].Fixture != "local" || rep.events[0].Suite != "smoke" || rep.events[0].FixtureType != "compose" { + t.Fatalf("unexpected teardown event: %+v", rep.events[0]) + } +} + +func TestCloseFixture_RemoteDoesNotEmitTeardownEvent(t *testing.T) { + rep := &recordingReporter{} + fix := &fakeSuiteFixture{} + plan := discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "remote-lgtm"}, + Fixture: discovery.FixtureConfig{Type: "remote"}, + } + if err := closeFixture(rep, plan, fix); err != nil { + t.Fatalf("closeFixture: %v", err) + } + if fix.closeCalls != 1 { + t.Fatalf("expected Close once, got %d", fix.closeCalls) + } + if len(rep.events) != 0 { + t.Fatalf("expected no teardown events for remote fixture, got %+v", rep.events) + } +} + // TestIntegration_FullPipelineWithFakeGCX wires the v2 chain end-to-end: // discovery → seed (against an httptest OTLP stub) → engine (against the // fake-gcx.sh shell script) → assertions → report. No real gcx, no real @@ -802,3 +841,13 @@ func equalStrings(got, want []string) bool { } return true } + +type recordingReporter struct { + events []report.Event +} + +func (r *recordingReporter) Emit(e report.Event) { + r.events = append(r.events, e) +} + +func (r *recordingReporter) Close() error { return nil } diff --git a/cmd/v2/main.go b/cmd/v2/main.go index eadfa690..16938384 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -152,34 +152,17 @@ func run() int { var totalPass, totalFail int for _, plan := range plans { - fixtureStart := time.Now() - if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { - rep.Emit(report.Event{ - Type: report.EventFixtureStart, - Suite: plan.Suite.Name, - Fixture: plan.Suite.Fixture, - FixtureType: plan.Fixture.Type, - Ts: fixtureStart, - }) - } + fixtureStart := emitFixtureStart(rep, plan) fix, err := startFixture(ctx, cfg.SourceDir, plan) if err != nil { fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) return 2 } - if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { - rep.Emit(report.Event{ - Type: report.EventFixtureReady, - Suite: plan.Suite.Name, - Fixture: plan.Suite.Fixture, - FixtureType: plan.Fixture.Type, - DurationMs: time.Since(fixtureStart).Milliseconds(), - }) - } + emitFixtureReady(rep, plan, fixtureStart) ep, err := resolveEndpoint(cfg.SourceDir, plan, *gcxContextOverride, *appHost, *appPort, *otlpHTTP) if err != nil { if fix != nil { - _ = fix.Close() + _ = closeFixture(rep, plan, fix) } fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) return 2 @@ -236,18 +219,10 @@ func run() int { Fail: suiteFail, }) if fix != nil { - teardownStart := time.Now() - if closeErr := fix.Close(); closeErr != nil { + if closeErr := closeFixture(rep, plan, fix); closeErr != nil { fmt.Fprintf(os.Stderr, "suite %q: fixture shutdown: %v\n", plan.Suite.Name, closeErr) return 2 } - rep.Emit(report.Event{ - Type: report.EventFixtureTeardown, - Suite: plan.Suite.Name, - Fixture: plan.Suite.Fixture, - FixtureType: plan.Fixture.Type, - DurationMs: time.Since(teardownStart).Milliseconds(), - }) } } @@ -383,6 +358,49 @@ func startSuiteFixture(fix suiteFixture) error { return startable.Up() } +func emitFixtureStart(rep report.Reporter, plan discovery.Plan) time.Time { + start := time.Now() + if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { + rep.Emit(report.Event{ + Type: report.EventFixtureStart, + Suite: plan.Suite.Name, + Fixture: plan.Suite.Fixture, + FixtureType: plan.Fixture.Type, + Ts: start, + }) + } + return start +} + +func emitFixtureReady(rep report.Reporter, plan discovery.Plan, start time.Time) { + if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { + rep.Emit(report.Event{ + Type: report.EventFixtureReady, + Suite: plan.Suite.Name, + Fixture: plan.Suite.Fixture, + FixtureType: plan.Fixture.Type, + DurationMs: time.Since(start).Milliseconds(), + }) + } +} + +func closeFixture(rep report.Reporter, plan discovery.Plan, fix suiteFixture) error { + start := time.Now() + if err := fix.Close(); err != nil { + return err + } + if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { + rep.Emit(report.Event{ + Type: report.EventFixtureTeardown, + Suite: plan.Suite.Name, + Fixture: plan.Suite.Fixture, + FixtureType: plan.Fixture.Type, + DurationMs: time.Since(start).Milliseconds(), + }) + } + return nil +} + type endpointFixture struct { ep *remote.Endpoint } From c480d0a3598382e583c61df8d4e205569d1d6e9d Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:53:59 +0000 Subject: [PATCH 046/124] test(v2): cover fixture start and ready helpers Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 9ede9472..bd554e32 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -220,6 +220,48 @@ func TestCloseFixture_RemoteDoesNotEmitTeardownEvent(t *testing.T) { } } +func TestEmitFixtureStartAndReady(t *testing.T) { + rep := &recordingReporter{} + plan := discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, + Fixture: discovery.FixtureConfig{Type: "compose"}, + } + start := emitFixtureStart(rep, plan) + if start.IsZero() { + t.Fatalf("expected non-zero start time") + } + if len(rep.events) != 1 || rep.events[0].Type != report.EventFixtureStart { + t.Fatalf("expected one fixture.start event, got %+v", rep.events) + } + if rep.events[0].Fixture != "local" || rep.events[0].Suite != "smoke" || rep.events[0].FixtureType != "compose" { + t.Fatalf("unexpected fixture.start event: %+v", rep.events[0]) + } + + emitFixtureReady(rep, plan, start.Add(-5*time.Millisecond)) + if len(rep.events) != 2 || rep.events[1].Type != report.EventFixtureReady { + t.Fatalf("expected fixture.ready event, got %+v", rep.events) + } + if rep.events[1].DurationMs <= 0 { + t.Fatalf("expected positive ready duration, got %+v", rep.events[1]) + } +} + +func TestEmitFixtureStartAndReady_NoOpForRemote(t *testing.T) { + rep := &recordingReporter{} + plan := discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "remote-lgtm"}, + Fixture: discovery.FixtureConfig{Type: "remote"}, + } + start := emitFixtureStart(rep, plan) + if start.IsZero() { + t.Fatalf("expected non-zero start time") + } + emitFixtureReady(rep, plan, start) + if len(rep.events) != 0 { + t.Fatalf("expected no events for remote fixture lifecycle helpers, got %+v", rep.events) + } +} + // TestIntegration_FullPipelineWithFakeGCX wires the v2 chain end-to-end: // discovery → seed (against an httptest OTLP stub) → engine (against the // fake-gcx.sh shell script) → assertions → report. No real gcx, no real From 49be0e4c8db1382d2ac0438b151bc8979c0bc49f Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:55:41 +0000 Subject: [PATCH 047/124] docs(v2): add profile smoke example Signed-off-by: Gregor Zeitlinger --- V2.md | 1 + discovery/discovery_test.go | 8 ++++---- examples/v2-smoke/cases/profile.yaml | 11 +++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 examples/v2-smoke/cases/profile.yaml diff --git a/V2.md b/V2.md index 69983bee..2e560097 100644 --- a/V2.md +++ b/V2.md @@ -35,6 +35,7 @@ The repo includes a small reference config under `examples/v2-smoke/` showing: - an app-backed case with `input` - an inline-OTLP case - a `custom-checks` case +- a profile assertion case For richer fixture examples, `examples/v2-fixtures/` shows: - multi-file compose fixtures with env passthrough diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 91c8e9b1..e89e4186 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -262,11 +262,11 @@ func TestExampleV2SmokeConfigLoads(t *testing.T) { if err != nil { t.Fatalf("PlanRun example config: %v", err) } - if len(plans) != 1 || len(plans[0].Cases) != 3 { - t.Fatalf("expected one suite/three cases, got %+v", plans) + if len(plans) != 1 || len(plans[0].Cases) != 4 { + t.Fatalf("expected one suite/four cases, got %+v", plans) } - got := []string{plans[0].Cases[0].Name, plans[0].Cases[1].Name, plans[0].Cases[2].Name} - want := []string{"custom check smoke", "inline seed smoke", "rolldice smoke"} + got := []string{plans[0].Cases[0].Name, plans[0].Cases[1].Name, plans[0].Cases[2].Name, plans[0].Cases[3].Name} + want := []string{"custom check smoke", "inline seed smoke", "profile smoke", "rolldice smoke"} for i := range want { if got[i] != want[i] { t.Fatalf("unexpected case order: got %v want %v", got, want) diff --git a/examples/v2-smoke/cases/profile.yaml b/examples/v2-smoke/cases/profile.yaml new file mode 100644 index 00000000..a79b2471 --- /dev/null +++ b/examples/v2-smoke/cases/profile.yaml @@ -0,0 +1,11 @@ +oats: 2 +name: profile smoke +seed: + type: app +expected: + profiles: + - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{}' + match: + - match_type: regexp + name: 'main|worker' + From 0f846b7ea85ac2e46b57dd1a1a5c31b2f07d0bae Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:57:42 +0000 Subject: [PATCH 048/124] test(v2): cover profile queries end to end Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 72 +++++++++++++++++++++++++++++++++++++ cmd/v2/testdata/fake-gcx.sh | 5 +++ 2 files changed, 77 insertions(+) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index bd554e32..f0500312 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -478,6 +478,78 @@ expected: } } +func TestIntegration_ProfileQueryWithFakeGCX(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fake-gcx is a POSIX shell script") + } + + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "profiles" +cases = ["cases/*.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "http://localhost:4318" +`) + writeFile(t, dir, "cases/profile.yaml", `oats: 2 +name: profile query end-to-end +seed: + type: app +expected: + profiles: + - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{}' + match: + - match_type: regexp + name: 'main|worker' +`) + + cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || len(plans[0].Cases) != 1 { + t.Fatalf("expected one plan with one case, got %+v", plans) + } + + ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + if err != nil { + t.Fatalf("resolveEndpoint: %v", err) + } + + _, here, _, _ := runtime.Caller(0) + fakeGCX := filepath.Join(filepath.Dir(here), "testdata", "fake-gcx.sh") + exec := &engine.GCX{Binary: fakeGCX, Context: ep.GCXContext} + + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + rep.Emit(report.Event{Type: report.EventRunStart, OatsVersion: "test", SchemaVersion: report.SchemaVersion}) + + r := runner.New(exec, rep, ep, runner.Options{ + Timeout: 500 * time.Millisecond, + Interval: 20 * time.Millisecond, + SeedSettleDelay: 5 * time.Millisecond, + }) + + ok := r.RunCase(context.Background(), plans[0].Cases[0]) + rep.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("profile case did not pass:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), "PASS 1/1") { + t.Fatalf("summary line missing or wrong:\n%s", buf.String()) + } +} + func TestIntegration_MigratedLegacyCaseRuns(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("fake-gcx is a POSIX shell script") diff --git a/cmd/v2/testdata/fake-gcx.sh b/cmd/v2/testdata/fake-gcx.sh index d63e30ad..f2b86ec5 100755 --- a/cmd/v2/testdata/fake-gcx.sh +++ b/cmd/v2/testdata/fake-gcx.sh @@ -49,6 +49,11 @@ metrics.query) # Static JSON shaped like Prometheus instant query output. cat <<'EOF' {"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"seed_counter_total","service_name":"gcx-e2e-seed"},"value":[1700000000,"42"]}]}} +EOF + ;; +profiles.query) + cat <<'EOF' +{"flamebearer":{"names":["main","worker"]}} EOF ;; *) From 8d2eba90fc5414267ac3a34ac34209e6b61ac965 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 14:59:18 +0000 Subject: [PATCH 049/124] test(v2): run migrated legacy profile end to end Signed-off-by: Gregor Zeitlinger --- cmd/v2/integration_test.go | 79 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index f0500312..8fa1ba3d 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -550,6 +550,85 @@ expected: } } +func TestIntegration_MigratedLegacyProfileRuns(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fake-gcx is a POSIX shell script") + } + + legacyPath := filepath.Join(t.TempDir(), "legacy-profile.oats.yaml") + writeFile(t, filepath.Dir(legacyPath), filepath.Base(legacyPath), ` +oats-schema-version: 2 +docker-compose: + files: + - docker-compose.yml +expected: + profiles: + - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{}' + flamebearers: + regexp: 'main|worker' +`) + migrated, _, err := migrate.ConvertFile(legacyPath) + if err != nil { + t.Fatalf("ConvertFile: %v", err) + } + + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "migrated-profile" +cases = ["cases/*.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "http://localhost:4318" +`) + writeFile(t, dir, "cases/migrated.yaml", string(migrated)) + + cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 || len(plans[0].Cases) != 1 { + t.Fatalf("expected one plan with one case, got %+v", plans) + } + + ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + if err != nil { + t.Fatalf("resolveEndpoint: %v", err) + } + + _, here, _, _ := runtime.Caller(0) + fakeGCX := filepath.Join(filepath.Dir(here), "testdata", "fake-gcx.sh") + exec := &engine.GCX{Binary: fakeGCX, Context: ep.GCXContext} + + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + rep.Emit(report.Event{Type: report.EventRunStart, OatsVersion: "test", SchemaVersion: report.SchemaVersion}) + + r := runner.New(exec, rep, ep, runner.Options{ + Timeout: 500 * time.Millisecond, + Interval: 20 * time.Millisecond, + SeedSettleDelay: 5 * time.Millisecond, + }) + + ok := r.RunCase(context.Background(), plans[0].Cases[0]) + rep.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("migrated legacy profile case did not pass:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), "PASS 1/1") { + t.Fatalf("summary line missing or wrong:\n%s", buf.String()) + } +} + func TestIntegration_MigratedLegacyCaseRuns(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("fake-gcx is a POSIX shell script") From 97f7358db6fd90b179a4a51f22b412b0c552d825 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 15:01:04 +0000 Subject: [PATCH 050/124] docs(v2): summarize implemented scope Signed-off-by: Gregor Zeitlinger --- README.md | 8 ++++++++ V2.md | 14 +++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9df25454..68c22ade 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,14 @@ Useful v2 entry points: - richer fixture examples: [`examples/v2-fixtures/`](examples/v2-fixtures/) - best-effort legacy migration: `bin/oats-v2 --migrate path/to/oats.yaml` +Current v2 scope already includes: +- traces / logs / metrics / profiles via `gcx` +- structural `match` assertions +- inline-OTLP and app-backed cases +- custom checks +- remote / compose / k3d fixtures +- best-effort migration from legacy OATS yaml + OpenTelemetry Acceptance Tests (OATs), or OATs for short, is a test framework for OpenTelemetry. - Declarative tests written in YAML diff --git a/V2.md b/V2.md index 2e560097..0f705175 100644 --- a/V2.md +++ b/V2.md @@ -1,4 +1,4 @@ -# OATS v2 (work in progress) +# OATS v2 This document covers the v2 branch, which replaces the bespoke TraceQL / PromQL / LogQL HTTP query infrastructure with the [gcx](https://github.com/grafana/gcx) @@ -41,6 +41,18 @@ For richer fixture examples, `examples/v2-fixtures/` shows: - multi-file compose fixtures with env passthrough - k3d fixture config with app build/import fields +## Current implemented scope + +Today the v2 branch already includes: +- gcx-driven traces / logs / metrics / profiles querying +- collector-style structural `match` assertions with `match_type: strict | regexp` +- app-backed and inline-OTLP seed modes +- input-driving HTTP requests +- custom checks +- remote / compose / k3d fixture flows +- best-effort legacy migration with runnable migrated outputs for core cases +- text and NDJSON reporting, including fixture lifecycle events at verbose levels + ## Architecture ```text From ba9975a97ed30c823c1ebd94426b2515cd5a13ae Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 16 Jun 2026 15:52:31 +0000 Subject: [PATCH 051/124] Drop versioned naming from current CLI docs Signed-off-by: Gregor Zeitlinger --- V2.md => CURRENT.md | 45 +++++++++---------- README.md | 22 ++++----- cmd/v2/main.go | 23 +++++----- cmd/v2/testdata/fake-gcx.sh | 2 +- discovery/discovery_test.go | 4 +- .../cases/compose/rolldice.yaml | 0 .../cases/k3d/rolldice.yaml | 0 examples/{v2-fixtures => fixtures}/oats.toml | 0 .../scripts/verify.sh | 0 .../cases/custom-check.yaml | 0 .../cases/inline-seed.yaml | 0 .../{v2-smoke => smoke}/cases/profile.yaml | 0 .../{v2-smoke => smoke}/cases/rolldice.yaml | 0 examples/{v2-smoke => smoke}/oats.toml | 0 .../{v2-smoke => smoke}/scripts/verify.sh | 0 runner/runner.go | 2 +- 16 files changed, 48 insertions(+), 50 deletions(-) rename V2.md => CURRENT.md (86%) rename examples/{v2-fixtures => fixtures}/cases/compose/rolldice.yaml (100%) rename examples/{v2-fixtures => fixtures}/cases/k3d/rolldice.yaml (100%) rename examples/{v2-fixtures => fixtures}/oats.toml (100%) rename examples/{v2-fixtures => fixtures}/scripts/verify.sh (100%) rename examples/{v2-smoke => smoke}/cases/custom-check.yaml (100%) rename examples/{v2-smoke => smoke}/cases/inline-seed.yaml (100%) rename examples/{v2-smoke => smoke}/cases/profile.yaml (100%) rename examples/{v2-smoke => smoke}/cases/rolldice.yaml (100%) rename examples/{v2-smoke => smoke}/oats.toml (100%) rename examples/{v2-smoke => smoke}/scripts/verify.sh (100%) diff --git a/V2.md b/CURRENT.md similarity index 86% rename from V2.md rename to CURRENT.md index 0f705175..2b8e927c 100644 --- a/V2.md +++ b/CURRENT.md @@ -1,49 +1,48 @@ -# OATS v2 +# Current OATS CLI -This document covers the v2 branch, which replaces the bespoke TraceQL / +This document covers the current gcx-driven OATS CLI, which replaces the bespoke TraceQL / PromQL / LogQL HTTP query infrastructure with the [gcx](https://github.com/grafana/gcx) -CLI. The v2 binary lives at `cmd/v2/main.go` while the branch is in flight; -both `oats` and `oats-v2` build from this checkout. See +CLI. The current CLI entry point lives at `cmd/v2/main.go` while this replaces the legacy path. See [grafana/internal-docs#14](https://github.com/grafana/internal-docs/pull/14) for the full design. ## Quick start ```sh -# Build the v2 binary -go build -o bin/oats-v2 ./cmd/v2 +# Build the oats binary +go build -o bin/oats ./cmd/v2 # Print what would run, do not execute -bin/oats-v2 --config oats.toml --list +bin/oats --config oats.toml --list # Run against an already-running stack -bin/oats-v2 --config oats.toml --gcx-context my-lgtm +bin/oats --config oats.toml --gcx-context my-lgtm # Disable cache for this run -bin/oats-v2 --config oats.toml --no-cache +bin/oats --config oats.toml --no-cache # Verbose output -bin/oats-v2 -v # adds per-case PASS lines -bin/oats-v2 -v=2 # adds the gcx command behind each assertion -bin/oats-v2 -v=3 # adds fixture lifecycle and full gcx stdout +bin/oats -v # adds per-case PASS lines +bin/oats -v=2 # adds the gcx command behind each assertion +bin/oats -v=3 # adds fixture lifecycle and full gcx stdout # Machine-readable -bin/oats-v2 --format ndjson > events.jsonl +bin/oats --format ndjson > events.jsonl ``` -The repo includes a small reference config under `examples/v2-smoke/` showing: +The repo includes a small reference config under `examples/smoke/` showing: - an app-backed case with `input` - an inline-OTLP case - a `custom-checks` case - a profile assertion case -For richer fixture examples, `examples/v2-fixtures/` shows: +For richer fixture examples, `examples/fixtures/` shows: - multi-file compose fixtures with env passthrough - k3d fixture config with app build/import fields ## Current implemented scope -Today the v2 branch already includes: +Today this branch already includes: - gcx-driven traces / logs / metrics / profiles querying - collector-style structural `match` assertions with `match_type: strict | regexp` - app-backed and inline-OTLP seed modes @@ -111,7 +110,7 @@ template = "lgtm" # built-in compose template ttl_days = 7 # default ``` -## Case yaml shape (v2) +## Case yaml shape ```yaml oats: 2 @@ -211,34 +210,34 @@ match: http.route: "^/roll.*$" ``` -## What's not in v2 yet +## What's not here yet - Parallel cases within a suite. - k3d pool warmup / reuse. - Full-fidelity `oats migrate` for fixture/input semantics. A best-effort - `oats-v2 --migrate ` now converts expectation blocks into the + `oats --migrate ` now converts expectation blocks into the collector-style `match` schema and prints warnings for dropped/unsupported fields (multi-entry matrix cases still expand manually). - Hermeticity static-check (runtime check applies). ## Migrating from v1 -For the v1 → v2 migration story see the OATS v2 implementation plan in +For the legacy → current migration story see the OATS implementation plan in [grafana/internal-docs#14](https://github.com/grafana/internal-docs/pull/14). Today a best-effort converter exists: ```bash -oats-v2 --migrate path/to/oats.yaml > migrated.yaml +oats --migrate path/to/oats.yaml > migrated.yaml ``` It converts legacy `equals` / `regexp` / `attributes` / -`attribute-regexp` assertions into v2 `match:` entries and prints warnings +`attribute-regexp` assertions into `match:` entries and prints warnings for fields that still need manual follow-up. For richer legacy fixtures, the warnings now include ready-to-paste `oats.toml` fixture snippets for multi-file compose/env and kubernetes→k3d mappings. Single-entry legacy `matrix:` cases are flattened automatically; multi-entry matrix cases emit fixture-expansion hints for manual splitting. The v1 binary (`oats`) and the -v2 binary (`oats-v2`) still read different file shapes. +current binary (`oats`) still reads a different file shape than the legacy YAML runner. ## Verbosity contract diff --git a/README.md b/README.md index 68c22ade..198c68f8 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,25 @@ # OpenTelemetry Acceptance Tests (OATs) > **Note:** this README primarily documents the legacy `oats` flow and yaml -> shape. The newer gcx-driven v2 work lives behind `cmd/v2` / `oats-v2`; see -> [V2.md](V2.md) for the current v2 syntax, fixture model, and migration path. +> shape. The newer gcx-driven work lives behind `cmd/v2`; see +> [CURRENT.md](CURRENT.md) for the current syntax, fixture model, and migration path. -## V2 quick pointer +## Current CLI quick pointer If you want the newer gcx-driven flow today: ```sh -go build -o bin/oats-v2 ./cmd/v2 -bin/oats-v2 --config examples/v2-smoke/oats.toml --list +go build -o bin/oats ./cmd/v2 +bin/oats --config examples/smoke/oats.toml --list ``` -Useful v2 entry points: -- syntax and feature status: [V2.md](V2.md) -- small runnable examples: [`examples/v2-smoke/`](examples/v2-smoke/) -- richer fixture examples: [`examples/v2-fixtures/`](examples/v2-fixtures/) -- best-effort legacy migration: `bin/oats-v2 --migrate path/to/oats.yaml` +Useful entry points: +- syntax and feature status: [CURRENT.md](CURRENT.md) +- small runnable examples: [`examples/smoke/`](examples/smoke/) +- richer fixture examples: [`examples/fixtures/`](examples/fixtures/) +- best-effort legacy migration: `bin/oats --migrate path/to/oats.yaml` -Current v2 scope already includes: +Current scope already includes: - traces / logs / metrics / profiles via `gcx` - structural `match` assertions - inline-OTLP and app-backed cases diff --git a/cmd/v2/main.go b/cmd/v2/main.go index 16938384..e05596b7 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -1,13 +1,12 @@ -// Command oats-v2 is the new OATS binary entry point. +// Command oats is the OATS binary entry point. // -// While the v2 branch is in flight, this command lives alongside the v1 -// "oats" binary at the repository root. The final v2 commit replaces the -// root main.go with the contents of this file; until then, both binaries -// coexist so v1 acceptance tests keep running unmodified. +// This command currently lives under cmd/v2 while the repository still keeps +// the legacy root entrypoint around for existing tests. The gcx-driven CLI is +// intended to replace that legacy path. // // Usage: // -// oats-v2 [flags] +// oats [flags] // // Flags (subset): // @@ -76,7 +75,7 @@ func run() int { configPath := flag.String("config", "oats.toml", "path to oats.toml") gcxBin := flag.String("gcx", "gcx", "path to gcx binary (PATH-resolved if a bare name)") listOnly := flag.Bool("list", false, "print the run plan and exit (no execution)") - migratePath := flag.String("migrate", "", "convert one legacy OATS yaml file to v2 and print the result to stdout") + migratePath := flag.String("migrate", "", "convert one legacy OATS yaml file and print the result to stdout") format := flag.String("format", "text", "output format: text | ndjson") suiteFilterStr := flag.String("suite", "", "comma-separated suite names") tagFilterStr := flag.String("tags", "", "comma-separated tag any-match") @@ -140,7 +139,7 @@ func run() int { rep.Emit(report.Event{ Type: report.EventRunStart, - OatsVersion: "v2-dev", + OatsVersion: "dev", SchemaVersion: report.SchemaVersion, Ts: time.Now(), }) @@ -192,7 +191,7 @@ func run() int { fixtureBytes, _ := json.Marshal(plan.Fixture) // stable across calls r = r.WithCache(store, runner.CacheContext{ GCXVersion: gcxVersion(*gcxBin), - OatsVersion: "v2-dev", + OatsVersion: "dev", FixtureBytes: fixtureBytes, }) } @@ -281,9 +280,9 @@ func resolveEndpoint(sourceDir string, plan discovery.Plan, gcxContextOverride, } case "": // No fixture configured — caller (or --gcx-context) must supply - // everything. Useful while plumbing v2 against an external setup. + // everything. Useful while plumbing the new CLI against an external setup. default: - return ep, fmt.Errorf("fixture type %q is not supported in oats-v2", plan.Fixture.Type) + return ep, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) } if gcxContextOverride != "" { ep.GCXContext = gcxContextOverride @@ -327,7 +326,7 @@ func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (sui } return endpointFixture{ep: ep}, nil default: - return nil, fmt.Errorf("fixture type %q is not supported in oats-v2", plan.Fixture.Type) + return nil, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) } } diff --git a/cmd/v2/testdata/fake-gcx.sh b/cmd/v2/testdata/fake-gcx.sh index f2b86ec5..7ea00ca9 100755 --- a/cmd/v2/testdata/fake-gcx.sh +++ b/cmd/v2/testdata/fake-gcx.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# fake-gcx: a tiny stand-in for the gcx CLI used by oats-v2 integration tests. +# fake-gcx: a tiny stand-in for the gcx CLI used by oats integration tests. # # It accepts a "--context X" prefix (stripped and ignored), then dispatches on # the verb chain. Output is deterministic and matches what the integration diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index e89e4186..d178ac68 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -254,7 +254,7 @@ func TestSummary(t *testing.T) { } func TestExampleV2SmokeConfigLoads(t *testing.T) { - cfg, err := Load(filepath.Join("..", "examples", "v2-smoke", "oats.toml")) + cfg, err := Load(filepath.Join("..", "examples", "smoke", "oats.toml")) if err != nil { t.Fatalf("Load example config: %v", err) } @@ -275,7 +275,7 @@ func TestExampleV2SmokeConfigLoads(t *testing.T) { } func TestExampleV2FixturesConfigLoads(t *testing.T) { - cfg, err := Load(filepath.Join("..", "examples", "v2-fixtures", "oats.toml")) + cfg, err := Load(filepath.Join("..", "examples", "fixtures", "oats.toml")) if err != nil { t.Fatalf("Load fixture example config: %v", err) } diff --git a/examples/v2-fixtures/cases/compose/rolldice.yaml b/examples/fixtures/cases/compose/rolldice.yaml similarity index 100% rename from examples/v2-fixtures/cases/compose/rolldice.yaml rename to examples/fixtures/cases/compose/rolldice.yaml diff --git a/examples/v2-fixtures/cases/k3d/rolldice.yaml b/examples/fixtures/cases/k3d/rolldice.yaml similarity index 100% rename from examples/v2-fixtures/cases/k3d/rolldice.yaml rename to examples/fixtures/cases/k3d/rolldice.yaml diff --git a/examples/v2-fixtures/oats.toml b/examples/fixtures/oats.toml similarity index 100% rename from examples/v2-fixtures/oats.toml rename to examples/fixtures/oats.toml diff --git a/examples/v2-fixtures/scripts/verify.sh b/examples/fixtures/scripts/verify.sh similarity index 100% rename from examples/v2-fixtures/scripts/verify.sh rename to examples/fixtures/scripts/verify.sh diff --git a/examples/v2-smoke/cases/custom-check.yaml b/examples/smoke/cases/custom-check.yaml similarity index 100% rename from examples/v2-smoke/cases/custom-check.yaml rename to examples/smoke/cases/custom-check.yaml diff --git a/examples/v2-smoke/cases/inline-seed.yaml b/examples/smoke/cases/inline-seed.yaml similarity index 100% rename from examples/v2-smoke/cases/inline-seed.yaml rename to examples/smoke/cases/inline-seed.yaml diff --git a/examples/v2-smoke/cases/profile.yaml b/examples/smoke/cases/profile.yaml similarity index 100% rename from examples/v2-smoke/cases/profile.yaml rename to examples/smoke/cases/profile.yaml diff --git a/examples/v2-smoke/cases/rolldice.yaml b/examples/smoke/cases/rolldice.yaml similarity index 100% rename from examples/v2-smoke/cases/rolldice.yaml rename to examples/smoke/cases/rolldice.yaml diff --git a/examples/v2-smoke/oats.toml b/examples/smoke/oats.toml similarity index 100% rename from examples/v2-smoke/oats.toml rename to examples/smoke/oats.toml diff --git a/examples/v2-smoke/scripts/verify.sh b/examples/smoke/scripts/verify.sh similarity index 100% rename from examples/v2-smoke/scripts/verify.sh rename to examples/smoke/scripts/verify.sh diff --git a/runner/runner.go b/runner/runner.go index 2fccf07d..b66b09ee 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -296,7 +296,7 @@ func customCheckCommand(ctx context.Context, dir, script string) (*exec.Cmd, fun return nil, cleanup, fmt.Errorf("empty script") } if looksLikeInlineScript(script) { - f, err := os.CreateTemp("", "oats-v2-custom-check-*.sh") + f, err := os.CreateTemp("", "oats-custom-check-*.sh") if err != nil { return nil, cleanup, err } From 54fbfe0928d76247d733fd2efa8d6d7618b925a3 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 17 Jun 2026 05:33:44 +0000 Subject: [PATCH 052/124] Use match_spans for trace structural assertions Signed-off-by: Gregor Zeitlinger --- CURRENT.md | 13 +++++---- cmd/v2/integration_test.go | 4 +-- examples/fixtures/cases/compose/rolldice.yaml | 2 +- examples/smoke/cases/inline-seed.yaml | 2 +- examples/smoke/cases/rolldice.yaml | 2 +- migrate/migrate.go | 7 ++++- migrate/migrate_test.go | 6 ++-- runner/runner.go | 29 +++++++++++++++++-- signalcmd/signalcmd.go | 2 +- signalcmd/signalcmd_test.go | 6 ++-- v2case/case.go | 23 ++++++++++++--- v2case/case_test.go | 4 +-- 12 files changed, 72 insertions(+), 28 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index 2b8e927c..3a8719ed 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -44,7 +44,7 @@ For richer fixture examples, `examples/fixtures/` shows: Today this branch already includes: - gcx-driven traces / logs / metrics / profiles querying -- collector-style structural `match` assertions with `match_type: strict | regexp` +- collector-style structural assertions (`match_spans` for traces, `match` elsewhere) with `match_type: strict | regexp` - app-backed and inline-OTLP seed modes - input-driving HTTP requests - custom checks @@ -125,7 +125,7 @@ input: expected: traces: - traceql: '{ span.http.route = "/rolldice" }' - match: + match_spans: - name: "GET /rolldice" metrics: - promql: 'dice_lib_rolls_counter_total{service_name="dice-server"}' @@ -163,7 +163,7 @@ seed: expected: traces: - traceql: '{ resource.service.name = "gcx-e2e-seed" }' - match: + match_spans: - name: seed-operation attributes: service.name: gcx-e2e-seed @@ -179,7 +179,8 @@ Per signal under `expected.[]`: | `contains` | Substrings that must appear in gcx stdout. | | `not_contains` | Substrings that must not appear. | | `regex` | Patterns that must match. | -| `match` | Structural assertions using collector-style `match_type: strict | regexp`. | +| `match_spans` | Trace-only structural assertions over returned spans using collector-style `match_type: strict | regexp`. | +| `match` | Structural assertions for logs / metrics / profiles using collector-style `match_type: strict | regexp`. | | `value` | Metrics only — numeric comparison (`>= 0`, `== 42`). | | `count` | Comparison against the number of result rows. | | `absent` | If true, the query must return zero rows. | @@ -195,7 +196,7 @@ assertion poll, mirroring OATS v1's “drive the app until telemetry appears” behavior. For remote fixtures, point those requests at a running app with `--app-host` and `--app-port` (defaults: `localhost:8080`). -`match` entries default to `match_type: strict`. OATS also supports the +`match_spans` (for traces) and `match` (for logs / metrics / profiles) default to `match_type: strict`. OATS also supports the small convenience extension `present: true` for attribute existence checks: ```yaml @@ -231,7 +232,7 @@ oats --migrate path/to/oats.yaml > migrated.yaml ``` It converts legacy `equals` / `regexp` / `attributes` / -`attribute-regexp` assertions into `match:` entries and prints warnings +`attribute-regexp` assertions into structural matcher entries (`match_spans:` for traces, `match:` elsewhere) and prints warnings for fields that still need manual follow-up. For richer legacy fixtures, the warnings now include ready-to-paste `oats.toml` fixture snippets for multi-file compose/env and kubernetes→k3d mappings. Single-entry legacy diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 8fa1ba3d..86f5b64f 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -305,7 +305,7 @@ seed: expected: traces: - traceql: '{ resource.service.name = "gcx-e2e-seed" }' - match: + match_spans: - name: seed-operation attributes: service.name: gcx-e2e-seed @@ -421,7 +421,7 @@ input: expected: traces: - traceql: '{ resource.service.name = "gcx-e2e-seed" }' - match: + match_spans: - name: seed-operation attributes: service.name: gcx-e2e-seed diff --git a/examples/fixtures/cases/compose/rolldice.yaml b/examples/fixtures/cases/compose/rolldice.yaml index 26f39969..9608351d 100644 --- a/examples/fixtures/cases/compose/rolldice.yaml +++ b/examples/fixtures/cases/compose/rolldice.yaml @@ -7,7 +7,7 @@ input: expected: traces: - traceql: '{ resource.service.name = "rolldice" }' - match: + match_spans: - match_type: regexp name: '^GET /rolldice.*$' logs: diff --git a/examples/smoke/cases/inline-seed.yaml b/examples/smoke/cases/inline-seed.yaml index 15f10bd3..85b96a66 100644 --- a/examples/smoke/cases/inline-seed.yaml +++ b/examples/smoke/cases/inline-seed.yaml @@ -9,7 +9,7 @@ seed: expected: traces: - traceql: '{ resource.service.name = "gcx-e2e-seed" }' - match: + match_spans: - name: seed-operation attributes: service.name: gcx-e2e-seed diff --git a/examples/smoke/cases/rolldice.yaml b/examples/smoke/cases/rolldice.yaml index 40f518b1..af2102e3 100644 --- a/examples/smoke/cases/rolldice.yaml +++ b/examples/smoke/cases/rolldice.yaml @@ -8,7 +8,7 @@ seed: expected: traces: - traceql: '{ resource.service.name = "rolldice" }' - match: + match_spans: - match_type: regexp name: '^GET /rolldice.*$' metrics: diff --git a/migrate/migrate.go b/migrate/migrate.go index bfdce738..9388440c 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -111,7 +111,7 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, } assertion, ws := convertSignal(tr.TraceQL, tr.Signal) warnings = append(warnings, ws...) - c.Expected.Traces = append(c.Expected.Traces, v2case.TraceAssertion{TraceQL: tr.TraceQL, AssertionCommon: assertion}) + c.Expected.Traces = append(c.Expected.Traces, v2case.TraceAssertion{TraceQL: tr.TraceQL, MatchSpans: assertion.Match, AssertionCommon: withoutMatch(assertion)}) } for _, lg := range def.Expected.Logs { if !keepForMatrix(lg.Signal.MatrixCondition, selectedMatrix) { @@ -222,6 +222,11 @@ func convertSignal(label string, s model.ExpectedSignal) (v2case.AssertionCommon return out, warnings } +func withoutMatch(a v2case.AssertionCommon) v2case.AssertionCommon { + a.Match = nil + return a +} + func deriveName(path string) string { base := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) base = strings.ReplaceAll(base, "_", " ") diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 28084529..defc8ba6 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -59,10 +59,10 @@ func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { if len(c.Input) != 1 || c.Input[0].Path != "/stock" { fatalf(t, "expected input to carry over, got %+v", c.Input) } - if len(c.Expected.Traces) != 1 || len(c.Expected.Traces[0].Match) != 2 { + if len(c.Expected.Traces) != 1 || len(c.Expected.Traces[0].MatchSpans) != 2 { fatalf(t, "expected trace strict+regexp split, got %+v", c.Expected.Traces) } - if got := c.Expected.Traces[0].Match[1].Attributes["trace_id"].Present; got == nil || !*got { + if got := c.Expected.Traces[0].MatchSpans[1].Attributes["trace_id"].Present; got == nil || !*got { fatalf(t, "expected trace_id .* to map to present:true") } if c.Expected.Logs[0].Count != ">= 1" { @@ -89,7 +89,7 @@ func TestConvertFile_RendersYAML(t *testing.T) { fatalf(t, "expected at least one warning for flattened include or fixture migration") } text := string(out) - for _, want := range []string{"oats: 2", "seed:", "input:", "path: /stock", "match:", "match_type: regexp", "db.system: h2", "promql: foo"} { + for _, want := range []string{"oats: 2", "seed:", "input:", "path: /stock", "match_spans:", "match_type: regexp", "db.system: h2", "promql: foo"} { if !strings.Contains(text, want) { fatalf(t, "expected migrated yaml to contain %q:\n%s", want, text) } diff --git a/runner/runner.go b/runner/runner.go index b66b09ee..1243a301 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -460,11 +460,11 @@ func (r *Runner) caseInterval(c *v2case.Case) time.Duration { func (r *Runner) runTrace(ctx context.Context, c *v2case.Case, a *v2case.TraceAssertion) bool { args := signalcmd.Traces(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { - if len(a.Match) == 0 { + if len(a.MatchSpans) == 0 { return evalCommonText(stdout, a.AssertionCommon) } rows, count, err := extractTraceRows(stdout) - return evalCommonStructured(stdout, a.AssertionCommon, rows, count, err) + return evalTraceStructured(stdout, *a, rows, count, err) }) } @@ -525,6 +525,31 @@ func evalCommonText(stdout string, c v2case.AssertionCommon) []assert.Failure { return fails } +func evalTraceStructured(stdout string, a v2case.TraceAssertion, rows []assert.Row, count int, parseErr error) []assert.Failure { + var fails []assert.Failure + fails = append(fails, assert.Contains(stdout, a.Contains)...) + fails = append(fails, assert.NotContains(stdout, a.NotContains)...) + fails = append(fails, assert.Regex(stdout, a.Regex)...) + if parseErr != nil { + fails = append(fails, assert.Failure{Rule: "match_spans", Detail: parseErr.Error()}) + return fails + } + if len(a.MatchSpans) > 0 { + spanFails := assert.MatchRows(rows, a.MatchSpans) + for i := range spanFails { + spanFails[i].Rule = "match_spans" + } + fails = append(fails, spanFails...) + } + if a.Count != "" { + fails = append(fails, assert.Count(count, a.Count)...) + } + if a.Absent { + fails = append(fails, assert.Absent(count)...) + } + return fails +} + func evalCommonStructured(stdout string, c v2case.AssertionCommon, rows []assert.Row, count int, parseErr error) []assert.Failure { var fails []assert.Failure fails = append(fails, assert.Contains(stdout, c.Contains)...) diff --git a/signalcmd/signalcmd.go b/signalcmd/signalcmd.go index 07e0de50..46e36d6c 100644 --- a/signalcmd/signalcmd.go +++ b/signalcmd/signalcmd.go @@ -30,7 +30,7 @@ func Traces(a v2case.TraceAssertion, since time.Duration) []string { "traces", "search", "--since", since.String(), } - if len(a.Match) > 0 { + if len(a.MatchSpans) > 0 { args = append(args, "-o", "json") } args = append(args, a.TraceQL) diff --git a/signalcmd/signalcmd_test.go b/signalcmd/signalcmd_test.go index 7db0a018..17cc765e 100644 --- a/signalcmd/signalcmd_test.go +++ b/signalcmd/signalcmd_test.go @@ -18,10 +18,8 @@ func TestTraces(t *testing.T) { func TestTraces_WithMatchAsksForJSON(t *testing.T) { got := Traces(v2case.TraceAssertion{ - TraceQL: `{ span.http.route = "/x" }`, - AssertionCommon: v2case.AssertionCommon{ - Match: []v2case.MatchEntry{{Name: strPtr("GET /x")}}, - }, + TraceQL: `{ span.http.route = "/x" }`, + MatchSpans: []v2case.MatchEntry{{Name: strPtr("GET /x")}}, }, 0) if !contains(got, "-o", "json") { t.Errorf("expected -o json in: %v", got) diff --git a/v2case/case.go b/v2case/case.go index 55ca8d01..5128ea2a 100644 --- a/v2case/case.go +++ b/v2case/case.go @@ -180,7 +180,8 @@ func (a AttributeExpectation) MarshalYAML() (any, error) { } type TraceAssertion struct { - TraceQL string `yaml:"traceql"` + TraceQL string `yaml:"traceql"` + MatchSpans []MatchEntry `yaml:"match_spans,omitempty"` AssertionCommon `yaml:",inline"` } @@ -272,7 +273,7 @@ func (c *Case) Validate() error { if c.Expected.Traces[i].TraceQL == "" { return fmt.Errorf("expected.traces[%d].traceql: required, non-empty", i) } - if err := validateAssertionCommon("expected.traces", i, c.Expected.Traces[i].AssertionCommon); err != nil { + if err := validateTraceAssertion(i, c.Expected.Traces[i]); err != nil { return err } } @@ -318,14 +319,28 @@ func (c *Case) IsHermetic() bool { return *c.Hermetic } +func validateTraceAssertion(idx int, a TraceAssertion) error { + if err := validateMatchEntries(fmt.Sprintf("expected.traces[%d].match_spans", idx), a.MatchSpans); err != nil { + return err + } + return validateAssertionCommon("expected.traces", idx, a.AssertionCommon) +} + func validateAssertionCommon(path string, idx int, a AssertionCommon) error { for j, p := range a.Regex { if _, err := regexp.Compile(p); err != nil { return fmt.Errorf("%s[%d].regex[%d]: invalid regexp %q: %v", path, idx, j, p, err) } } - for j, m := range a.Match { - matchPath := fmt.Sprintf("%s[%d].match[%d]", path, idx, j) + if err := validateMatchEntries(fmt.Sprintf("%s[%d].match", path, idx), a.Match); err != nil { + return err + } + return nil +} + +func validateMatchEntries(path string, entries []MatchEntry) error { + for j, m := range entries { + matchPath := fmt.Sprintf("%s[%d]", path, j) switch m.EffectiveMatchType() { case MatchTypeStrict, MatchTypeRegexp: default: diff --git a/v2case/case_test.go b/v2case/case_test.go index f39994a1..2e3545c0 100644 --- a/v2case/case_test.go +++ b/v2case/case_test.go @@ -255,7 +255,7 @@ seed: expected: traces: - traceql: '{}' - match: + match_spans: - match_type: glob name: x `)) @@ -294,7 +294,7 @@ seed: expected: traces: - traceql: '{}' - match: + match_spans: - match_type: regexp name: '[' `)) From 307f8630fce475980d38accf31b1caaecffa889d Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 17 Jun 2026 14:03:06 +0000 Subject: [PATCH 053/124] fix(v2): handle empty gcx JSON output and script paths Signed-off-by: Gregor Zeitlinger --- runner/runner.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/runner/runner.go b/runner/runner.go index 1243a301..fc1bbd4a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -337,7 +337,11 @@ func resolveCustomCheckPath(dir, script string) string { return script } if strings.ContainsRune(script, os.PathSeparator) { - return filepath.Clean(filepath.Join(dir, script)) + joined := filepath.Clean(filepath.Join(dir, script)) + if abs, err := filepath.Abs(joined); err == nil { + return abs + } + return joined } return script } @@ -595,6 +599,9 @@ func approxRowCount(stdout string) int { // query -o json` output. The schema follows gcx's JSON shape; we only look // at the fields we need so additions don't break us. func extractMetricRows(stdout string) ([]assert.Row, int, float64, error) { + if strings.TrimSpace(stdout) == "" { + return nil, 0, 0, fmt.Errorf("metric value parse: empty result") + } var generic struct { Data struct { Result []struct { @@ -690,6 +697,9 @@ func (r *Runner) doInput(in v2case.Input) error { } func extractLogRows(stdout string) ([]assert.Row, int, error) { + if strings.TrimSpace(stdout) == "" { + return nil, 0, nil + } var generic struct { Data struct { Result []struct { @@ -716,6 +726,9 @@ func extractLogRows(stdout string) ([]assert.Row, int, error) { } func extractTraceRows(stdout string) ([]assert.Row, int, error) { + if strings.TrimSpace(stdout) == "" { + return nil, 0, nil + } var root any if err := json.Unmarshal([]byte(stdout), &root); err != nil { return nil, 0, fmt.Errorf("trace JSON parse: %w", err) @@ -732,6 +745,9 @@ func extractTraceRows(stdout string) ([]assert.Row, int, error) { } func extractProfileRows(stdout string) ([]assert.Row, int, error) { + if strings.TrimSpace(stdout) == "" { + return nil, 0, nil + } var root any if err := json.Unmarshal([]byte(stdout), &root); err != nil { return nil, 0, fmt.Errorf("profile JSON parse: %w", err) @@ -841,9 +857,10 @@ func extractOTLPTraceRows(root any) ([]assert.Row, bool) { if !ok { // Some wrappers may nest under "data" first. if data, ok := top["data"].(map[string]any); ok { - resourceSpans, ok = data["resourceSpans"].([]any) - if !ok { - resourceSpans, ok = data["batches"].([]any) + if nested, ok := data["resourceSpans"].([]any); ok { + resourceSpans = nested + } else if nested, ok := data["batches"].([]any); ok { + resourceSpans = nested } } if !ok { From c2c3ab72a44215f445ee9765dbecde62a2348234 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 19 Jun 2026 07:52:08 +0200 Subject: [PATCH 054/124] Simplify consumer fixture wiring Signed-off-by: Gregor Zeitlinger --- CURRENT.md | 51 +++++---- README.md | 3 +- cmd/v2/integration_test.go | 71 +++++++----- cmd/v2/main.go | 214 +++++++++++++++++++++++++++++++---- discovery/discovery.go | 149 +++++++++++++++++++++--- runner/runner.go | 16 ++- scripts/build-local-tools.sh | 15 +++ v2case/case.go | 58 +++++++++- 8 files changed, 486 insertions(+), 91 deletions(-) create mode 100755 scripts/build-local-tools.sh diff --git a/CURRENT.md b/CURRENT.md index 3a8719ed..3d104c64 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -9,8 +9,8 @@ for the full design. ## Quick start ```sh -# Build the oats binary -go build -o bin/oats ./cmd/v2 +# Build oats plus the gcx binary it shells out to +./scripts/build-local-tools.sh # Print what would run, do not execute bin/oats --config oats.toml --list @@ -66,7 +66,7 @@ discovery → seed → engine → assert → report | Package | Responsibility | |------------|---------------| -| `discovery` | Parse `oats.toml`, expand case globs, apply filters. | +| `discovery` | Parse `oats.toml`, expand case globs, apply filters, and derive case-local fixtures when a suite omits one. | | `v2case` | Parse and validate one case yaml file. | | `seed` | Push inline-OTLP payloads at an OTLP/HTTP endpoint. | | `engine` | Execute a gcx command, capture stdout/stderr/exit. | @@ -78,6 +78,14 @@ discovery → seed → engine → assert → report | `runner` | Orchestrates a suite: seed → poll-and-assert → report, with optional cache. | | `cmd/v2` | The new binary entry point. | +## Consumer-shape notes + +For simple consumer repos, keep `oats.toml` thin: a suite usually only needs `cases = ["..."]`. +Case-local `fixture:` blocks now cover the common one-case-per-suite path. Shared root-level `[fixture.*]` blocks are still useful when many suites intentionally reuse the same fixture or when one case is run against multiple fixtures. + +Local LGTM compose boot plus Grafana auth bootstrap are now owned by OATS itself: +consumer repos should not need their own shared `docker-compose.lgtm.yml` or a custom `gcx-wrapper.sh` just to talk to a local LGTM stack. + ## `oats.toml` shape ```toml @@ -85,26 +93,18 @@ discovery → seed → engine → assert → report version = 2 [[suite]] -name = "lgtm-examples" -cases = ["examples/*/oats.yaml"] -fixture = "lgtm-shared" -tags = ["traces", "metrics", "logs"] - -[fixture.lgtm-shared] -type = "compose" # compose | k3d | remote -template = "lgtm" # built-in compose template -# compose_file = "./my-compose.yml" # alternative to template -# compose_files = ["./base.yml", "./override.yml"] # multi-file compose -# env = ["FOO=bar"] # extra env for compose config/up -# k8s_dir = "./k8s" # k3d only -# app_service = "dice" # k3d only -# app_docker_file = "Dockerfile" -# app_docker_context = "." -# app_docker_tag = "dice:test" -# app_port = 8080 -# import_images = ["grafana/docker-otel-lgtm:latest"] -# pool_size = 4 # reserved for future k3d pooling -# endpoint = "http://..." # remote only +cases = ["examples/nodejs/oats.yaml"] + +# Optional when many suites share one fixture: +# [[suite]] +# name = "lgtm-examples" +# cases = ["examples/*/oats.yaml"] +# fixture = "lgtm-shared" +# tags = ["traces", "metrics", "logs"] +# +# [fixture.lgtm-shared] +# type = "compose" +# template = "lgtm" [cache] ttl_days = 7 # default @@ -116,6 +116,11 @@ ttl_days = 7 # default oats: 2 name: rolldice traces have route attribute +fixture: + type: compose + template: lgtm + compose_file: docker-compose.oats.yml + seed: type: app compose: docker-compose.app.yml # optional legacy shorthand; suite fixture usually owns boot diff --git a/README.md b/README.md index 198c68f8..e7b9e4a2 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ If you want the newer gcx-driven flow today: ```sh -go build -o bin/oats ./cmd/v2 +./scripts/build-local-tools.sh bin/oats --config examples/smoke/oats.toml --list ``` @@ -20,6 +20,7 @@ Useful entry points: - best-effort legacy migration: `bin/oats --migrate path/to/oats.yaml` Current scope already includes: +- built-in local-LGTM fixture/bootstrap owned by OATS (consumer repos do not need a shared `docker-compose.lgtm.yml` or `gcx` wrapper) - traces / logs / metrics / profiles via `gcx` - structural `match` assertions - inline-OTLP and app-backed cases diff --git a/cmd/v2/integration_test.go b/cmd/v2/integration_test.go index 86f5b64f..d206b497 100644 --- a/cmd/v2/integration_test.go +++ b/cmd/v2/integration_test.go @@ -24,53 +24,71 @@ import ( ) func TestResolveComposeFiles(t *testing.T) { - got, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFile: "stack/compose.yml"}) + got, cleanup, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFile: "stack/compose.yml"}) if err != nil { t.Fatalf("resolveComposeFiles compose_file: %v", err) } + if cleanup != nil { + t.Fatalf("unexpected cleanup for compose_file fixture") + } if want := []string{"/tmp/work/stack/compose.yml"}; len(got) != 1 || got[0] != want[0] { t.Fatalf("got %q want %q", got, want) } - got, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFiles: []string{"a.yml", "b.yml"}}) + got, cleanup, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFiles: []string{"a.yml", "b.yml"}}) if err != nil { t.Fatalf("resolveComposeFiles compose_files: %v", err) } + if cleanup != nil { + t.Fatalf("unexpected cleanup for compose_files fixture") + } if len(got) != 2 || got[0] != "/tmp/work/a.yml" || got[1] != "/tmp/work/b.yml" { t.Fatalf("unexpected compose_files resolution: %v", got) } - got, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "lgtm"}) + got, cleanup, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "lgtm"}) if err != nil { t.Fatalf("resolveComposeFiles template=lgtm: %v", err) } - if want := []string{"/tmp/work/docker-compose.yml"}; len(got) != 1 || got[0] != want[0] { - t.Fatalf("got %q want %q", got, want) + if cleanup == nil { + t.Fatalf("expected cleanup for template=lgtm fixture") + } + defer func() { _ = cleanup() }() + if len(got) != 1 || !strings.HasSuffix(got[0], ".oats.lgtm.compose.yml") { + t.Fatalf("unexpected template=lgtm resolution: %v", got) } } func TestResolveEndpoint_ComposeDefaults(t *testing.T) { - ep, err := resolveEndpoint("/tmp/work", discovery.Plan{ + oldToken := waitForGrafanaToken + waitForGrafanaToken = func(plan discovery.Plan) (string, error) { return "tok", nil } + defer func() { waitForGrafanaToken = oldToken }() + + ep, err := resolveEndpoint(discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, Fixture: discovery.FixtureConfig{Type: "compose", Template: "lgtm"}, }, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } - if ep.GCXContext != "local" || ep.AppHost != "localhost" || ep.AppPort != 8080 || ep.OTLPHTTP != "http://localhost:4318" { + if ep.GCXContext != "" || ep.AppHost != "localhost" || ep.AppPort != 8080 || ep.OTLPHTTP != "http://localhost:4318" { t.Fatalf("unexpected endpoint: %+v", ep) } } func TestResolveEndpoint_K3DUsesFixtureAppPort(t *testing.T) { - ep, err := resolveEndpoint("/tmp/work", discovery.Plan{ + oldToken := waitForGrafanaToken + waitForGrafanaToken = func(plan discovery.Plan) (string, error) { return "tok", nil } + defer func() { waitForGrafanaToken = oldToken }() + + ep, err := resolveEndpoint(discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "cluster"}, Fixture: discovery.FixtureConfig{Type: "k3d", AppPort: 18080}, }, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } - if ep.GCXContext != "cluster" || ep.AppPort != 18080 { + if ep.GCXContext != "" || ep.AppPort != 18080 { t.Fatalf("unexpected endpoint: %+v", ep) } } @@ -87,13 +105,14 @@ func TestStartFixture_ComposeLifecycle(t *testing.T) { return fake, nil } - fix, err := startFixture(context.Background(), "/tmp/work", discovery.Plan{ + fix, err := startFixture(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, Fixture: discovery.FixtureConfig{ Type: "compose", ComposeFiles: []string{"a.yml", "b.yml"}, Env: []string{"FOO=bar"}, }, + FixtureSourceDir: "/tmp/work", }) if err != nil { t.Fatalf("startFixture compose: %v", err) @@ -123,9 +142,10 @@ func TestStartFixture_ComposeStartFailure(t *testing.T) { return &fakeSuiteFixture{upErr: fmt.Errorf("boom")}, nil } - _, err := startFixture(context.Background(), "/tmp/work", discovery.Plan{ - Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, - Fixture: discovery.FixtureConfig{Type: "compose", ComposeFile: "compose.yml"}, + _, err := startFixture(context.Background(), discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, + Fixture: discovery.FixtureConfig{Type: "compose", ComposeFile: "compose.yml"}, + FixtureSourceDir: "/tmp/work", }) if err == nil || !strings.Contains(err.Error(), "boom") { t.Fatalf("expected compose startup error, got %v", err) @@ -136,11 +156,9 @@ func TestStartFixture_K3DLifecycle(t *testing.T) { oldFactory := newKubernetesEndpoint defer func() { newKubernetesEndpoint = oldFactory }() - var capturedSource string var capturedPlan discovery.Plan var starts, stops int - newKubernetesEndpoint = func(sourceDir string, plan discovery.Plan) *remote.Endpoint { - capturedSource = sourceDir + newKubernetesEndpoint = func(plan discovery.Plan) *remote.Endpoint { capturedPlan = plan return remote.NewEndpoint("localhost", remote.PortsConfig{}, func(ctx context.Context) error { starts++ @@ -151,7 +169,7 @@ func TestStartFixture_K3DLifecycle(t *testing.T) { }, nil) } - fix, err := startFixture(context.Background(), "/tmp/work", discovery.Plan{ + fix, err := startFixture(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, Fixture: discovery.FixtureConfig{ Type: "k3d", @@ -163,6 +181,7 @@ func TestStartFixture_K3DLifecycle(t *testing.T) { AppPort: 18080, ImportImages: []string{"busybox:latest"}, }, + FixtureSourceDir: "/tmp/work", }) if err != nil { t.Fatalf("startFixture k3d: %v", err) @@ -170,8 +189,8 @@ func TestStartFixture_K3DLifecycle(t *testing.T) { if starts != 1 { t.Fatalf("expected one endpoint start, got %d", starts) } - if capturedSource != "/tmp/work" || capturedPlan.Suite.Name != "cluster-smoke" || capturedPlan.Fixture.AppPort != 18080 { - t.Fatalf("unexpected endpoint factory args: source=%q plan=%+v", capturedSource, capturedPlan) + if capturedPlan.FixtureSourceDir != "/tmp/work" || capturedPlan.Suite.Name != "cluster-smoke" || capturedPlan.Fixture.AppPort != 18080 { + t.Fatalf("unexpected endpoint factory args: plan=%+v", capturedPlan) } if err := fix.Close(); err != nil { t.Fatalf("fixture close: %v", err) @@ -445,7 +464,7 @@ expected: t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(dir, plans[0], "", appHost, appPort, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], "", appHost, appPort, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -521,7 +540,7 @@ expected: t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -600,7 +619,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -688,7 +707,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -767,7 +786,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -845,7 +864,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -943,7 +962,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected k8s-only assertion to be filtered out, got %d traces", got) } - ep, err := resolveEndpoint(dir, plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } diff --git a/cmd/v2/main.go b/cmd/v2/main.go index e05596b7..bd3d611f 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -47,9 +47,13 @@ var ( newComposeSuite = func(files []string, env []string) (suiteFixture, error) { return compose.SuiteFiles(files, env) } - newKubernetesEndpoint = func(sourceDir string, plan discovery.Plan) *remote.Endpoint { + newKubernetesEndpoint = func(plan discovery.Plan) *remote.Endpoint { + sourceDir := plan.FixtureSourceDir + if sourceDir == "" { + sourceDir = "." + } model := &kubernetes.Kubernetes{ - Dir: filepath.Join(sourceDir, plan.Fixture.K8sDir), + Dir: plan.Fixture.K8sDir, AppService: plan.Fixture.AppService, AppDockerFile: plan.Fixture.AppDockerFile, AppDockerContext: plan.Fixture.AppDockerContext, @@ -64,6 +68,7 @@ var ( PyroscopeHttpPort: 4040, }, plan.Suite.Name, sourceDir) } + waitForGrafanaToken = waitForGrafanaTokenImpl ) func main() { @@ -152,13 +157,13 @@ func run() int { for _, plan := range plans { fixtureStart := emitFixtureStart(rep, plan) - fix, err := startFixture(ctx, cfg.SourceDir, plan) + fix, err := startFixture(ctx, plan) if err != nil { fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) return 2 } emitFixtureReady(rep, plan, fixtureStart) - ep, err := resolveEndpoint(cfg.SourceDir, plan, *gcxContextOverride, *appHost, *appPort, *otlpHTTP) + ep, err := resolveEndpoint(plan, *gcxContextOverride, *appHost, *appPort, *otlpHTTP) if err != nil { if fix != nil { _ = closeFixture(rep, plan, fix) @@ -175,7 +180,7 @@ func run() int { CaseCount: len(plan.Cases), }) - gcxExec := &engine.GCX{Binary: *gcxBin, Context: ep.GCXContext} + gcxExec := &engine.GCX{Binary: *gcxBin, Context: ep.GCXContext, Env: ep.GCXEnv} r := runner.New(gcxExec, rep, ep, runner.Options{ Timeout: *timeout, Interval: *interval, @@ -262,7 +267,7 @@ func verbosityFromInt(n int) report.Verbosity { // resolveEndpoint maps a fixture config + an explicit override into the // concrete endpoint the runner needs. -func resolveEndpoint(sourceDir string, plan discovery.Plan, gcxContextOverride, appHost string, appPort int, otlpHTTP string) (runner.Endpoint, error) { +func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, appPort int, otlpHTTP string) (runner.Endpoint, error) { ep := runner.Endpoint{AppHost: appHost, AppPort: appPort, OTLPHTTP: otlpHTTP} switch plan.Fixture.Type { case "remote": @@ -271,23 +276,33 @@ func resolveEndpoint(sourceDir string, plan discovery.Plan, gcxContextOverride, // best-effort default; --gcx-context overrides. ep.GCXContext = plan.Suite.Fixture ep.OTLPHTTP = plan.Fixture.Endpoint + ep.CustomCheckEnv = append(ep.CustomCheckEnv, + "OATS_FIXTURE_TYPE=remote", + "OATS_GRAFANA_URL="+grafanaURL(), + "OATS_APP_URL="+fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), + ) case "compose": - ep.GCXContext = plan.Suite.Fixture + ep.GCXEnv = append(ep.GCXEnv, localGrafanaEnv(plan)...) + ep.CustomCheckEnv = append(ep.CustomCheckEnv, composeCheckEnv(plan, ep)...) case "k3d": - ep.GCXContext = plan.Suite.Fixture + ep.GCXEnv = append(ep.GCXEnv, localGrafanaEnv(plan)...) if plan.Fixture.AppPort > 0 { ep.AppPort = plan.Fixture.AppPort } + ep.CustomCheckEnv = append(ep.CustomCheckEnv, k3dCheckEnv(ep)...) case "": // No fixture configured — caller (or --gcx-context) must supply // everything. Useful while plumbing the new CLI against an external setup. + ep.CustomCheckEnv = append(ep.CustomCheckEnv, + "OATS_APP_URL="+fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), + ) default: return ep, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) } if gcxContextOverride != "" { ep.GCXContext = gcxContextOverride } - if ep.GCXContext == "" { + if ep.GCXContext == "" && len(ep.GCXEnv) == 0 { return ep, fmt.Errorf("gcx context unresolved; set fixture..endpoint or pass --gcx-context") } return ep, nil @@ -302,25 +317,31 @@ type startableSuiteFixture interface { Up() error } -func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (suiteFixture, error) { +func startFixture(_ context.Context, plan discovery.Plan) (suiteFixture, error) { switch plan.Fixture.Type { case "", "remote": return nil, nil case "compose": - composeFiles, err := resolveComposeFiles(sourceDir, plan.Fixture) + composeFiles, cleanup, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) if err != nil { return nil, err } suite, err := newComposeSuite(composeFiles, plan.Fixture.Env) if err != nil { + if cleanup != nil { + _ = cleanup() + } return nil, err } if err := startSuiteFixture(suite); err != nil { + if cleanup != nil { + _ = cleanup() + } return nil, err } - return suite, nil + return composeFixture{suite: suite, cleanup: cleanup}, nil case "k3d": - ep := newKubernetesEndpoint(sourceDir, plan) + ep := newKubernetesEndpoint(plan) if err := ep.Start(context.Background()); err != nil { return nil, err } @@ -330,8 +351,19 @@ func startFixture(_ context.Context, sourceDir string, plan discovery.Plan) (sui } } -func resolveComposeFiles(sourceDir string, fixture discovery.FixtureConfig) ([]string, error) { +func resolveComposeFiles(sourceDir string, fixture discovery.FixtureConfig) ([]string, func() error, error) { var files []string + var cleanup func() error + if fixture.Template == "lgtm" { + f, err := writeBuiltinLGTMCompose(sourceDir) + if err != nil { + return nil, nil, err + } + files = append(files, f) + cleanup = func() error { return os.Remove(f) } + } else if fixture.Template != "" { + return nil, nil, fmt.Errorf("unsupported compose fixture template %q", fixture.Template) + } switch { case fixture.ComposeFile != "": files = append(files, filepath.Join(sourceDir, fixture.ComposeFile)) @@ -339,14 +371,10 @@ func resolveComposeFiles(sourceDir string, fixture discovery.FixtureConfig) ([]s for _, file := range fixture.ComposeFiles { files = append(files, filepath.Join(sourceDir, file)) } - case fixture.Template == "lgtm": - files = append(files, filepath.Join(sourceDir, "docker-compose.yml")) case fixture.Template == "": - return nil, fmt.Errorf("compose fixture requires compose_file, compose_files, or supported template") - default: - return nil, fmt.Errorf("unsupported compose fixture template %q", fixture.Template) + return nil, nil, fmt.Errorf("compose fixture requires compose_file, compose_files, or supported template") } - return files, nil + return files, cleanup, nil } func startSuiteFixture(fix suiteFixture) error { @@ -408,6 +436,152 @@ func (e endpointFixture) Close() error { return e.ep.Stop(context.Background()) } +type composeFixture struct { + suite suiteFixture + cleanup func() error +} + +func (c composeFixture) Close() error { + err := c.suite.Close() + if c.cleanup != nil { + if cleanupErr := c.cleanup(); cleanupErr != nil && err == nil { + err = cleanupErr + } + } + return err +} + +func writeBuiltinLGTMCompose(sourceDir string) (string, error) { + path := filepath.Join(sourceDir, ".oats.lgtm.compose.yml") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return "", err + } + f, err := os.Create(path) + if err != nil { + return "", err + } + const body = `services: + lgtm: + image: ${LGTM_IMAGE:-docker.io/grafana/otel-lgtm:latest} + ports: + - "3000:3000" + - "4317:4317" + - "4318:4318" + - "3200:3200" + - "4040:4040" + - "9090:9090" +` + if _, err := f.WriteString(body); err != nil { + _ = f.Close() + _ = os.Remove(path) + return "", err + } + if err := f.Close(); err != nil { + _ = os.Remove(path) + return "", err + } + return path, nil +} + +func grafanaURL() string { return "http://localhost:3000" } +func pyroscopeURL() string { return "http://localhost:4040" } + +func localGrafanaEnv(plan discovery.Plan) []string { + token, err := waitForGrafanaToken(plan) + if err != nil { + return nil + } + return []string{ + "GRAFANA_SERVER=" + grafanaURL(), + "GRAFANA_TOKEN=" + token, + } +} + +func composeCheckEnv(plan discovery.Plan, ep runner.Endpoint) []string { + files, _, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) + if err != nil { + return []string{"OATS_FIXTURE_TYPE=compose"} + } + return []string{ + "OATS_FIXTURE_TYPE=compose", + "COMPOSE_FILE=" + strings.Join(files, string(os.PathListSeparator)), + "OATS_COMPOSE_FILE_ARGS=" + composeFileArgs(files), + "OATS_APP_URL=" + fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), + "OATS_GRAFANA_URL=" + grafanaURL(), + "OATS_PYROSCOPE_URL=" + pyroscopeURL(), + } +} + +func composeFileArgs(files []string) string { + var parts []string + for _, f := range files { + parts = append(parts, "-f", shellQuote(f)) + } + return strings.Join(parts, " ") +} + +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +func k3dCheckEnv(ep runner.Endpoint) []string { + return []string{ + "OATS_FIXTURE_TYPE=k3d", + "OATS_APP_URL=" + fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), + "OATS_GRAFANA_URL=" + grafanaURL(), + "OATS_PYROSCOPE_URL=" + pyroscopeURL(), + } +} + +func waitForGrafanaTokenImpl(plan discovery.Plan) (string, error) { + deadline := time.Now().Add(3 * time.Minute) + for time.Now().Before(deadline) { + var token string + var err error + switch plan.Fixture.Type { + case "compose": + token, err = readComposeGrafanaToken(plan) + case "k3d": + token, err = readK3DGrafanaToken() + default: + return "", nil + } + if err == nil && strings.TrimSpace(token) != "" { + return strings.TrimSpace(token), nil + } + time.Sleep(time.Second) + } + return "", fmt.Errorf("timed out waiting for Grafana service-account token") +} + +func readComposeGrafanaToken(plan discovery.Plan) (string, error) { + files, _, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) + if err != nil { + return "", err + } + args := []string{"compose"} + for _, f := range files { + args = append(args, "-f", f) + } + args = append(args, "exec", "-T", "lgtm", "sh", "-c", "cat /tmp/grafana-sa-token 2>/dev/null || true") + cmd := exec.Command("docker", args...) + cmd.Env = append(cmd.Environ(), plan.Fixture.Env...) + out, err := cmd.Output() + if err != nil { + return "", err + } + return string(out), nil +} + +func readK3DGrafanaToken() (string, error) { + cmd := exec.Command("kubectl", "exec", "deploy/lgtm", "--", "sh", "-c", "cat /tmp/grafana-sa-token 2>/dev/null || true") + out, err := cmd.Output() + if err != nil { + return "", err + } + return string(out), nil +} + func splitCSV(s string) []string { if strings.TrimSpace(s) == "" { return nil diff --git a/discovery/discovery.go b/discovery/discovery.go index c7d773ee..154ee9f3 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -12,7 +12,9 @@ import ( "fmt" "os" "path/filepath" + "reflect" "sort" + "strings" "github.com/BurntSushi/toml" "github.com/grafana/oats/v2case" @@ -98,15 +100,12 @@ func (c *RootConfig) Validate() error { return fmt.Errorf("at least one [[suite]] required") } for i, s := range c.Suites { - if s.Name == "" { - return fmt.Errorf("suite[%d].name: required", i) - } if len(s.Cases) == 0 { - return fmt.Errorf("suite[%d] (%q): cases is required and non-empty", i, s.Name) + return fmt.Errorf("suite[%d]: cases is required and non-empty", i) } if s.Fixture != "" { if _, ok := c.Fixture[s.Fixture]; !ok { - return fmt.Errorf("suite[%d] (%q): fixture %q not defined", i, s.Name, s.Fixture) + return fmt.Errorf("suite[%d] (%q): fixture %q not defined", i, suiteLabel(s), s.Fixture) } } } @@ -147,9 +146,10 @@ type Filter struct { // Plan is one suite plus the cases it expanded to. type Plan struct { - Suite SuiteConfig - Fixture FixtureConfig - Cases []*v2case.Case + Suite SuiteConfig + Fixture FixtureConfig + FixtureSourceDir string + Cases []*v2case.Case } // PlanRun expands globs and applies the filter against the loaded config. @@ -183,22 +183,27 @@ func (c *RootConfig) PlanRun(f Filter) ([]Plan, error) { var plans []Plan for _, suite := range c.Suites { + cases, err := c.loadSuiteCases(suite) + if err != nil { + return nil, fmt.Errorf("suite %q: %w", suiteLabel(suite), err) + } + suite = materializeSuite(suite, cases) if !wantSuite(suite.Name) { continue } if !wantTag(suite.Tags) { continue } - - cases, err := c.loadSuiteCases(suite) + fixture, fixtureSourceDir, err := c.resolveSuiteFixture(suite, cases) if err != nil { return nil, fmt.Errorf("suite %q: %w", suite.Name, err) } plans = append(plans, Plan{ - Suite: suite, - Fixture: c.Fixture[suite.Fixture], - Cases: cases, + Suite: suite, + Fixture: fixture, + FixtureSourceDir: fixtureSourceDir, + Cases: cases, }) } return plans, nil @@ -234,12 +239,128 @@ func (c *RootConfig) loadSuiteCases(suite SuiteConfig) ([]*v2case.Case, error) { return cases, nil } +func suiteLabel(s SuiteConfig) string { + if s.Name != "" { + return s.Name + } + if len(s.Cases) == 1 { + p := filepath.Clean(s.Cases[0]) + if base := dirLabel(filepath.Dir(p)); base != "" { + return base + } + return strings.TrimSuffix(filepath.Base(p), filepath.Ext(p)) + } + return fmt.Sprintf("suite[%d cases]", len(s.Cases)) +} + +func materializeSuite(s SuiteConfig, cases []*v2case.Case) SuiteConfig { + if s.Name == "" { + s.Name = deriveSuiteName(s, cases) + } + if len(s.Tags) == 0 { + seen := make(map[string]struct{}) + for _, c := range cases { + for _, tag := range c.Tags { + if _, ok := seen[tag]; ok { + continue + } + seen[tag] = struct{}{} + s.Tags = append(s.Tags, tag) + } + } + sort.Strings(s.Tags) + } + return s +} + +func deriveSuiteName(s SuiteConfig, cases []*v2case.Case) string { + if len(cases) == 1 { + if base := dirLabel(filepath.Dir(cases[0].SourcePath)); base != "" { + return base + } + } + if len(s.Cases) == 1 { + p := filepath.Clean(s.Cases[0]) + if base := dirLabel(filepath.Dir(p)); base != "" { + return base + } + return strings.TrimSuffix(filepath.Base(p), filepath.Ext(p)) + } + return fmt.Sprintf("suite-%d-cases", len(cases)) +} + +func dirLabel(dir string) string { + base := filepath.Base(dir) + if base == "oats" { + parent := filepath.Base(filepath.Dir(dir)) + if parent != "." && parent != string(filepath.Separator) && parent != "" { + return parent + } + } + if base == "." || base == string(filepath.Separator) { + return "" + } + return base +} + +func (c *RootConfig) resolveSuiteFixture(suite SuiteConfig, cases []*v2case.Case) (FixtureConfig, string, error) { + if suite.Fixture != "" { + return c.Fixture[suite.Fixture], c.SourceDir, nil + } + var ( + fixture FixtureConfig + sourceDir string + seen bool + ) + for _, tc := range cases { + if tc.Fixture == nil { + continue + } + next := fixtureConfigFromCase(*tc.Fixture) + nextSourceDir := filepath.Dir(tc.SourcePath) + if !seen { + fixture, sourceDir, seen = next, nextSourceDir, true + continue + } + if !reflect.DeepEqual(fixture, next) || sourceDir != nextSourceDir { + return FixtureConfig{}, "", fmt.Errorf("suite omits fixture but cases do not agree on one shared fixture") + } + } + if !seen { + return FixtureConfig{}, "", nil + } + return fixture, sourceDir, nil +} + +func fixtureConfigFromCase(f v2case.FixtureConfig) FixtureConfig { + return FixtureConfig{ + Type: f.Type, + Template: f.Template, + ComposeFile: f.ComposeFile, + ComposeFiles: append([]string(nil), f.ComposeFiles...), + Env: append([]string(nil), f.Env...), + K8sDir: f.K8sDir, + AppService: f.AppService, + AppDockerFile: f.AppDockerFile, + AppDockerContext: f.AppDockerContext, + AppDockerTag: f.AppDockerTag, + AppPort: f.AppPort, + ImportImages: append([]string(nil), f.ImportImages...), + PoolSize: f.PoolSize, + Endpoint: f.Endpoint, + } +} + // Summary renders a single line per plan suitable for `oats list`. It does // not load any cases — useful for a dry-run before deciding to expand globs. func (c *RootConfig) Summary() string { var out string for _, s := range c.Suites { - out += fmt.Sprintf("suite=%s fixture=%s tags=%v cases=%v\n", s.Name, s.Fixture, s.Tags, s.Cases) + label := s.Name + if label == "" { + label = suiteLabel(s) + } + out += fmt.Sprintf("suite=%s fixture=%s tags=%v cases=%v\n", label, s.Fixture, s.Tags, s.Cases) } return out } diff --git a/runner/runner.go b/runner/runner.go index fc1bbd4a..dfe58c96 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -47,6 +47,16 @@ type Endpoint struct { // comes from here. AppHost string AppPort int + + // GCXEnv supplements the gcx subprocess environment for every assertion. + // OATS uses this for local LGTM fixtures so consumer repos do not need a + // custom gcx wrapper script just to inject Grafana auth. + GCXEnv []string + + // CustomCheckEnv is appended to each custom-check subprocess environment. + // It carries fixture-specific helpers like COMPOSE_FILE and stable local + // endpoint URLs. + CustomCheckEnv []string } // Options configures the polling cadence and per-case deadline. Sensible @@ -259,7 +269,7 @@ func (r *Runner) runCustomCheck(ctx context.Context, c *v2case.Case, chk *v2case deadlineCtx, cancel := context.WithTimeout(ctx, r.opts.Timeout) defer cancel() - cmd, cleanup, err := customCheckCommand(deadlineCtx, dir, chk.Script) + cmd, cleanup, err := customCheckCommand(deadlineCtx, dir, chk.Script, r.endpoint.CustomCheckEnv) if err != nil { return []assert.Failure{{Rule: "custom-check-setup", Detail: err.Error()}} } @@ -290,7 +300,7 @@ func (r *Runner) runCustomCheck(ctx context.Context, c *v2case.Case, chk *v2case return false } -func customCheckCommand(ctx context.Context, dir, script string) (*exec.Cmd, func(), error) { +func customCheckCommand(ctx context.Context, dir, script string, extraEnv []string) (*exec.Cmd, func(), error) { cleanup := func() {} if strings.TrimSpace(script) == "" { return nil, cleanup, fmt.Errorf("empty script") @@ -316,10 +326,12 @@ func customCheckCommand(ctx context.Context, dir, script string) (*exec.Cmd, fun cleanup = func() { _ = os.Remove(f.Name()) } cmd := exec.CommandContext(ctx, "sh", f.Name()) cmd.Dir = dir + cmd.Env = append(cmd.Environ(), extraEnv...) return cmd, cleanup, nil } cmd := exec.CommandContext(ctx, resolveCustomCheckPath(dir, script)) cmd.Dir = dir + cmd.Env = append(cmd.Environ(), extraEnv...) return cmd, cleanup, nil } diff --git a/scripts/build-local-tools.sh b/scripts/build-local-tools.sh new file mode 100755 index 00000000..2431caf7 --- /dev/null +++ b/scripts/build-local-tools.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "$0")/.." && pwd)" +bin_dir="${1:-$root/bin}" +mkdir -p "$bin_dir" + +# Keep the gcx bootstrap logic owned by OATS so consumer repos only need to +# fetch/build OATS itself. +: "${GCX_VERSION:=v0.4.0}" + +GOWORK=off go -C "$root" build -buildvcs=false -o "$bin_dir/oats" ./cmd/v2 +GOBIN="$bin_dir" GOWORK=off go install "github.com/grafana/gcx/cmd/gcx@${GCX_VERSION}" + +printf 'built %s/oats and %s/gcx\n' "$bin_dir" "$bin_dir" diff --git a/v2case/case.go b/v2case/case.go index 5128ea2a..0284c7a1 100644 --- a/v2case/case.go +++ b/v2case/case.go @@ -28,11 +28,12 @@ const SchemaVersion = 2 // Case is one entry point yaml file. Cases are independently runnable; // suites group them via oats.toml. type Case struct { - OatsVersion int `yaml:"oats"` - Name string `yaml:"name"` - Tags []string `yaml:"tags,omitempty"` - Hermetic *bool `yaml:"hermetic,omitempty"` // pointer: distinguish unset vs explicit false - Interval time.Duration `yaml:"interval,omitempty"` + OatsVersion int `yaml:"oats"` + Name string `yaml:"name"` + Tags []string `yaml:"tags,omitempty"` + Hermetic *bool `yaml:"hermetic,omitempty"` // pointer: distinguish unset vs explicit false + Interval time.Duration `yaml:"interval,omitempty"` + Fixture *FixtureConfig `yaml:"fixture,omitempty"` Seed Seed `yaml:"seed"` Input []Input `yaml:"input,omitempty"` @@ -56,6 +57,23 @@ type Seed struct { Vars map[string]any `yaml:"vars,omitempty"` } +type FixtureConfig struct { + Type string `yaml:"type,omitempty"` + Template string `yaml:"template,omitempty"` + ComposeFile string `yaml:"compose_file,omitempty"` + ComposeFiles []string `yaml:"compose_files,omitempty"` + Env []string `yaml:"env,omitempty"` + K8sDir string `yaml:"k8s_dir,omitempty"` + AppService string `yaml:"app_service,omitempty"` + AppDockerFile string `yaml:"app_docker_file,omitempty"` + AppDockerContext string `yaml:"app_docker_context,omitempty"` + AppDockerTag string `yaml:"app_docker_tag,omitempty"` + AppPort int `yaml:"app_port,omitempty"` + ImportImages []string `yaml:"import_images,omitempty"` + PoolSize int `yaml:"pool_size,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` +} + type SeedTrace struct { Service string `yaml:"service"` Spans []SeedSpan `yaml:"spans"` @@ -247,6 +265,11 @@ func (c *Case) Validate() error { if c.Interval < 0 { return fmt.Errorf("interval: must be >= 0") } + if c.Fixture != nil { + if err := c.Fixture.Validate("fixture"); err != nil { + return err + } + } switch c.Seed.Type { case "app": // App-backed cases are normally booted by the suite fixture declared in @@ -309,6 +332,31 @@ func (c *Case) Validate() error { return nil } +func (f FixtureConfig) Validate(path string) error { + switch f.Type { + case "compose": + if f.Template == "" && f.ComposeFile == "" && len(f.ComposeFiles) == 0 { + return fmt.Errorf("%s: type=compose requires template, compose_file, or compose_files", path) + } + if f.ComposeFile != "" && len(f.ComposeFiles) > 0 { + return fmt.Errorf("%s: use compose_file or compose_files, not both", path) + } + case "k3d": + if f.K8sDir == "" || f.AppService == "" || f.AppDockerFile == "" || f.AppDockerTag == "" || f.AppPort == 0 { + return fmt.Errorf("%s: type=k3d requires k8s_dir, app_service, app_docker_file, app_docker_tag, and app_port", path) + } + case "remote": + if f.Endpoint == "" { + return fmt.Errorf("%s: type=remote requires endpoint", path) + } + case "": + return fmt.Errorf("%s.type: required (compose | k3d | remote)", path) + default: + return fmt.Errorf("%s.type: unknown value %q (expected compose, k3d, or remote)", path, f.Type) + } + return nil +} + // IsHermetic reports whether this case promises not to interfere with siblings // sharing the same fixture. Default is true; cases opt out by setting // `hermetic: false`. From 28d7463cbce4d65526b2283fb5765dfbe06ba9fc Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 19 Jun 2026 07:52:45 +0200 Subject: [PATCH 055/124] Polish docs after slimmer consumer wiring Signed-off-by: Gregor Zeitlinger --- CURRENT.md | 22 ++++++++++++++++------ README.md | 6 +++++- examples/fixtures/oats.toml | 1 - 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index 3d104c64..df01df38 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -31,18 +31,21 @@ bin/oats --format ndjson > events.jsonl ``` The repo includes a small reference config under `examples/smoke/` showing: + - an app-backed case with `input` - an inline-OTLP case - a `custom-checks` case - a profile assertion case For richer fixture examples, `examples/fixtures/` shows: + - multi-file compose fixtures with env passthrough - k3d fixture config with app build/import fields ## Current implemented scope Today this branch already includes: + - gcx-driven traces / logs / metrics / profiles querying - collector-style structural assertions (`match_spans` for traces, `match` elsewhere) with `match_type: strict | regexp` - app-backed and inline-OTLP seed modes @@ -66,7 +69,8 @@ discovery → seed → engine → assert → report | Package | Responsibility | |------------|---------------| -| `discovery` | Parse `oats.toml`, expand case globs, apply filters, and derive case-local fixtures when a suite omits one. | +| `discovery` | Parse `oats.toml`, expand case globs, apply filters, and derive +| | case-local fixtures when a suite omits one. | | `v2case` | Parse and validate one case yaml file. | | `seed` | Push inline-OTLP payloads at an OTLP/HTTP endpoint. | | `engine` | Execute a gcx command, capture stdout/stderr/exit. | @@ -74,17 +78,23 @@ discovery → seed → engine → assert → report | `assert` | The expectation vocabulary: `contains`, `not_contains`, `regex`, `value`, `count`, `absent`. | | `wait` | `Until` / `While` polling primitives (replaces gomega.Eventually). | | `report` | Compact-text and NDJSON renderers driven by an Event stream. | -| `cache` | Skip-when-unchanged store keyed by `(case yaml + fixture + gcx version + oats version)`. | +| `cache` | Skip-when-unchanged store keyed by +| | `(case yaml + fixture + gcx version + oats version)`. | | `runner` | Orchestrates a suite: seed → poll-and-assert → report, with optional cache. | | `cmd/v2` | The new binary entry point. | ## Consumer-shape notes -For simple consumer repos, keep `oats.toml` thin: a suite usually only needs `cases = ["..."]`. -Case-local `fixture:` blocks now cover the common one-case-per-suite path. Shared root-level `[fixture.*]` blocks are still useful when many suites intentionally reuse the same fixture or when one case is run against multiple fixtures. +For simple consumer repos, keep `oats.toml` thin: a suite usually only needs +`cases = ["..."]`. Case-local `fixture:` blocks now cover the common +one-case-per-suite path. Shared root-level `[fixture.*]` blocks are still useful +when many suites intentionally reuse the same fixture or when one case is run +against multiple fixtures. -Local LGTM compose boot plus Grafana auth bootstrap are now owned by OATS itself: -consumer repos should not need their own shared `docker-compose.lgtm.yml` or a custom `gcx-wrapper.sh` just to talk to a local LGTM stack. +Local LGTM compose boot plus Grafana auth bootstrap are now owned by OATS +itself: consumer repos should not need their own shared +`docker-compose.lgtm.yml` or a custom `gcx-wrapper.sh` just to talk to a local +LGTM stack. ## `oats.toml` shape diff --git a/README.md b/README.md index e7b9e4a2..b05ebe37 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,17 @@ bin/oats --config examples/smoke/oats.toml --list ``` Useful entry points: + - syntax and feature status: [CURRENT.md](CURRENT.md) - small runnable examples: [`examples/smoke/`](examples/smoke/) - richer fixture examples: [`examples/fixtures/`](examples/fixtures/) - best-effort legacy migration: `bin/oats --migrate path/to/oats.yaml` Current scope already includes: -- built-in local-LGTM fixture/bootstrap owned by OATS (consumer repos do not need a shared `docker-compose.lgtm.yml` or `gcx` wrapper) + +- built-in local-LGTM fixture/bootstrap owned by OATS + (consumer repos do not need a shared `docker-compose.lgtm.yml` or `gcx` + wrapper) - traces / logs / metrics / profiles via `gcx` - structural `match` assertions - inline-OTLP and app-backed cases diff --git a/examples/fixtures/oats.toml b/examples/fixtures/oats.toml index f6e9556c..cdc142ba 100644 --- a/examples/fixtures/oats.toml +++ b/examples/fixtures/oats.toml @@ -27,4 +27,3 @@ app_docker_context = "." app_docker_tag = "rolldice:test" app_port = 18080 import_images = ["grafana/otel-lgtm:latest"] - From f8a4e31ea4bc6ea5b6edf761d0dffab75dc9310a Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 19 Jun 2026 07:53:11 +0200 Subject: [PATCH 056/124] Fix CURRENT.md lint Signed-off-by: Gregor Zeitlinger --- CURRENT.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index df01df38..2c45b965 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -69,7 +69,7 @@ discovery → seed → engine → assert → report | Package | Responsibility | |------------|---------------| -| `discovery` | Parse `oats.toml`, expand case globs, apply filters, and derive +| `discovery` | Parse `oats.toml`, expand case globs, apply filters, and derive | | | case-local fixtures when a suite omits one. | | `v2case` | Parse and validate one case yaml file. | | `seed` | Push inline-OTLP payloads at an OTLP/HTTP endpoint. | @@ -78,7 +78,7 @@ discovery → seed → engine → assert → report | `assert` | The expectation vocabulary: `contains`, `not_contains`, `regex`, `value`, `count`, `absent`. | | `wait` | `Until` / `While` polling primitives (replaces gomega.Eventually). | | `report` | Compact-text and NDJSON renderers driven by an Event stream. | -| `cache` | Skip-when-unchanged store keyed by +| `cache` | Skip-when-unchanged store keyed by | | | `(case yaml + fixture + gcx version + oats version)`. | | `runner` | Orchestrates a suite: seed → poll-and-assert → report, with optional cache. | | `cmd/v2` | The new binary entry point. | @@ -211,8 +211,9 @@ assertion poll, mirroring OATS v1's “drive the app until telemetry appears” behavior. For remote fixtures, point those requests at a running app with `--app-host` and `--app-port` (defaults: `localhost:8080`). -`match_spans` (for traces) and `match` (for logs / metrics / profiles) default to `match_type: strict`. OATS also supports the -small convenience extension `present: true` for attribute existence checks: +`match_spans` (for traces) and `match` (for logs / metrics / profiles) +default to `match_type: strict`. OATS also supports the small convenience +extension `present: true` for attribute existence checks: ```yaml match: @@ -247,8 +248,9 @@ oats --migrate path/to/oats.yaml > migrated.yaml ``` It converts legacy `equals` / `regexp` / `attributes` / -`attribute-regexp` assertions into structural matcher entries (`match_spans:` for traces, `match:` elsewhere) and prints warnings -for fields that still need manual follow-up. For richer legacy fixtures, the +`attribute-regexp` assertions into structural matcher entries +(`match_spans:` for traces, `match:` elsewhere) and prints warnings for +fields that still need manual follow-up. For richer legacy fixtures, the warnings now include ready-to-paste `oats.toml` fixture snippets for multi-file compose/env and kubernetes→k3d mappings. Single-entry legacy `matrix:` cases are flattened automatically; multi-entry matrix cases emit From 9deed28497a4fa0b05b93d3670a9f4e82d903f50 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 19 Jun 2026 11:26:38 +0200 Subject: [PATCH 057/124] Support top-level cases lists in oats.toml Signed-off-by: Gregor Zeitlinger --- cmd/v2/main.go | 1 + discovery/discovery.go | 31 ++++++++++++++++++++++++--- discovery/discovery_test.go | 42 +++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/cmd/v2/main.go b/cmd/v2/main.go index bd3d611f..435db9f7 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -494,6 +494,7 @@ func localGrafanaEnv(plan discovery.Plan) []string { return []string{ "GRAFANA_SERVER=" + grafanaURL(), "GRAFANA_TOKEN=" + token, + "GRAFANA_ORG_ID=1", } } diff --git a/discovery/discovery.go b/discovery/discovery.go index 154ee9f3..c2a1e5b2 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -25,6 +25,7 @@ import ( // the toml decoder rather than a silent miss. type RootConfig struct { Meta Meta `toml:"meta"` + Cases []string `toml:"cases"` Suites []SuiteConfig `toml:"suite"` Fixture map[string]FixtureConfig `toml:"fixture"` Cache CacheConfig `toml:"cache,omitempty"` @@ -96,8 +97,18 @@ func (c *RootConfig) Validate() error { if c.Meta.Version != SupportedVersion { return fmt.Errorf("meta.version: expected %d, got %d", SupportedVersion, c.Meta.Version) } - if len(c.Suites) == 0 { - return fmt.Errorf("at least one [[suite]] required") + if len(c.Cases) > 0 && len(c.Suites) > 0 { + return fmt.Errorf("use top-level cases or [[suite]], not both") + } + if len(c.Cases) == 0 && len(c.Suites) == 0 { + return fmt.Errorf("at least one top-level case or [[suite]] required") + } + if len(c.Cases) > 0 { + for i, path := range c.Cases { + if strings.TrimSpace(path) == "" { + return fmt.Errorf("cases[%d]: path is required and non-empty", i) + } + } } for i, s := range c.Suites { if len(s.Cases) == 0 { @@ -152,6 +163,17 @@ type Plan struct { Cases []*v2case.Case } +func (c *RootConfig) effectiveSuites() []SuiteConfig { + if len(c.Suites) > 0 { + return c.Suites + } + suites := make([]SuiteConfig, 0, len(c.Cases)) + for _, path := range c.Cases { + suites = append(suites, SuiteConfig{Cases: []string{path}}) + } + return suites +} + // PlanRun expands globs and applies the filter against the loaded config. // Returns plans in oats.toml order; cases within a plan are sorted by // SourcePath for stable test ordering. @@ -182,7 +204,7 @@ func (c *RootConfig) PlanRun(f Filter) ([]Plan, error) { } var plans []Plan - for _, suite := range c.Suites { + for _, suite := range c.effectiveSuites() { cases, err := c.loadSuiteCases(suite) if err != nil { return nil, fmt.Errorf("suite %q: %w", suiteLabel(suite), err) @@ -275,6 +297,9 @@ func materializeSuite(s SuiteConfig, cases []*v2case.Case) SuiteConfig { func deriveSuiteName(s SuiteConfig, cases []*v2case.Case) string { if len(cases) == 1 { + if strings.TrimSpace(cases[0].Name) != "" { + return cases[0].Name + } if base := dirLabel(filepath.Dir(cases[0].SourcePath)); base != "" { return base } diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index d178ac68..8872cda9 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -304,3 +304,45 @@ func planNames(p []Plan) []string { } return out } + +func TestLoadTopLevelCases(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +cases = ["cases/a.yaml", "cases/b.yaml"] + +[meta] +version = 2 +`) + writeFile(t, dir, "cases/a.yaml", ` +oats: 2 +name: a +seed: { type: app } +expected: + traces: + - traceql: "{}" + absent: true +`) + writeFile(t, dir, "cases/b.yaml", ` +oats: 2 +name: b +seed: { type: app } +expected: + traces: + - traceql: "{}" + absent: true +`) + cfg, err := Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatal(err) + } + plans, err := cfg.PlanRun(Filter{}) + if err != nil { + t.Fatal(err) + } + if len(plans) != 2 { + t.Fatalf("expected 2 plans, got %d", len(plans)) + } + if plans[0].Suite.Name != "a" || plans[1].Suite.Name != "b" { + t.Fatalf("unexpected suite names: %q, %q", plans[0].Suite.Name, plans[1].Suite.Name) + } +} From 4a8e160ef50756803a67264e6de7f47a14a423de Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 19 Jun 2026 12:54:43 +0200 Subject: [PATCH 058/124] Own local gcx config for example assertions Signed-off-by: Gregor Zeitlinger --- cmd/v2/main.go | 51 ++++++++++++++++++++++++++++++++++++++++-------- engine/engine.go | 10 +++++++++- runner/runner.go | 7 ++++++- 3 files changed, 58 insertions(+), 10 deletions(-) diff --git a/cmd/v2/main.go b/cmd/v2/main.go index 435db9f7..112eed05 100644 --- a/cmd/v2/main.go +++ b/cmd/v2/main.go @@ -180,7 +180,7 @@ func run() int { CaseCount: len(plan.Cases), }) - gcxExec := &engine.GCX{Binary: *gcxBin, Context: ep.GCXContext, Env: ep.GCXEnv} + gcxExec := &engine.GCX{Binary: *gcxBin, Context: ep.GCXContext, Config: ep.GCXConfig, Env: ep.GCXEnv} r := runner.New(gcxExec, rep, ep, runner.Options{ Timeout: *timeout, Interval: *interval, @@ -283,9 +283,15 @@ func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, ap ) case "compose": ep.GCXEnv = append(ep.GCXEnv, localGrafanaEnv(plan)...) + if cfg, err := writeLocalGCXConfig(plan.FixtureSourceDir); err == nil { + ep.GCXConfig = cfg + } ep.CustomCheckEnv = append(ep.CustomCheckEnv, composeCheckEnv(plan, ep)...) case "k3d": ep.GCXEnv = append(ep.GCXEnv, localGrafanaEnv(plan)...) + if cfg, err := writeLocalGCXConfig(plan.FixtureSourceDir); err == nil { + ep.GCXConfig = cfg + } if plan.Fixture.AppPort > 0 { ep.AppPort = plan.Fixture.AppPort } @@ -302,7 +308,7 @@ func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, ap if gcxContextOverride != "" { ep.GCXContext = gcxContextOverride } - if ep.GCXContext == "" && len(ep.GCXEnv) == 0 { + if ep.GCXContext == "" && ep.GCXConfig == "" && len(ep.GCXEnv) == 0 { return ep, fmt.Errorf("gcx context unresolved; set fixture..endpoint or pass --gcx-context") } return ep, nil @@ -487,15 +493,44 @@ func grafanaURL() string { return "http://localhost:3000" } func pyroscopeURL() string { return "http://localhost:4040" } func localGrafanaEnv(plan discovery.Plan) []string { - token, err := waitForGrafanaToken(plan) + return nil +} + +func writeLocalGCXConfig(_ string) (string, error) { + cfg := `current-context: local +contexts: + local: + grafana: + server: http://localhost:3000 + user: admin + password: admin + org-id: 1 + auth-method: basic + datasources: + prometheus: prometheus + loki: loki + tempo: tempo + pyroscope: pyroscope +` + f, err := os.CreateTemp("", "oats-gcx-*.yaml") if err != nil { - return nil + return "", err } - return []string{ - "GRAFANA_SERVER=" + grafanaURL(), - "GRAFANA_TOKEN=" + token, - "GRAFANA_ORG_ID=1", + path := f.Name() + if _, err := f.WriteString(cfg); err != nil { + _ = f.Close() + _ = os.Remove(path) + return "", err + } + if err := f.Close(); err != nil { + _ = os.Remove(path) + return "", err } + if err := os.Chmod(path, 0o600); err != nil { + _ = os.Remove(path) + return "", err + } + return path, nil } func composeCheckEnv(plan discovery.Plan, ep runner.Endpoint) []string { diff --git a/engine/engine.go b/engine/engine.go index 1a21f7e8..92c6bb69 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -49,6 +49,11 @@ type GCX struct { // can drive multiple Grafana endpoints. Context string + // Config is an optional path passed via --config before every invocation. + // OATS uses this for local LGTM fixtures so gcx can query local datasources + // without consumer-managed wrapper scripts or ambient config state. + Config string + // Env supplements os.Environ() for the child process. Use to pass HOME / // XDG_CONFIG_HOME for config isolation. Env []string @@ -68,7 +73,10 @@ func (g *GCX) Execute(ctx context.Context, args ...string) (*Result, error) { full := args if g.Context != "" { - full = append([]string{"--context", g.Context}, args...) + full = append([]string{"--context", g.Context}, full...) + } + if g.Config != "" { + full = append([]string{"--config", g.Config}, full...) } if g.Timeout > 0 { diff --git a/runner/runner.go b/runner/runner.go index dfe58c96..f648889a 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -35,9 +35,14 @@ import ( // purpose: the v2 runner only needs to know where gcx points and where to // seed OTLP. Everything else lives in the gcx config context. type Endpoint struct { - // GCXContext is the value passed to gcx --context. Required. + // GCXContext is the value passed to gcx --context when using a named gcx + // context. Local LGTM fixtures may leave this empty and instead provide + // GCXConfig. GCXContext string + // GCXConfig is an optional gcx config file path for local fixtures. + GCXConfig string + // OTLPHTTP is the base URL for OTLP/HTTP seed POSTs ("http://localhost:4318"). // Required when any case uses seed.type = inline-otlp. OTLPHTTP string From 424bad9295ec5a4ae79c3314e05d0e4d23210024 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 19 Jun 2026 13:17:32 +0200 Subject: [PATCH 059/124] Ignore gcx table headers in empty result counts Signed-off-by: Gregor Zeitlinger --- runner/runner.go | 26 +++++++++++++++++++++++++- runner/runner_test.go | 3 +++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/runner/runner.go b/runner/runner.go index f648889a..1a2dab71 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -604,7 +604,7 @@ func approxRowCount(stdout string) int { continue } // Skip lines that look like table-headers / dividers in gcx text mode. - if strings.HasPrefix(t, "─") || strings.HasPrefix(t, "═") || strings.HasPrefix(t, "+") { + if strings.HasPrefix(t, "─") || strings.HasPrefix(t, "═") || strings.HasPrefix(t, "+") || looksLikeGCXHeader(t) { continue } n++ @@ -612,6 +612,30 @@ func approxRowCount(stdout string) int { return n } +func looksLikeGCXHeader(line string) bool { + fields := strings.Fields(line) + if len(fields) < 2 { + return false + } + for _, f := range fields { + hasLetter := false + for _, r := range f { + switch { + case r >= 'A' && r <= 'Z': + hasLetter = true + case r >= '0' && r <= '9': + case r == '_' || r == '.' || r == '/': + default: + return false + } + } + if !hasLetter { + return false + } + } + return true +} + // extractMetricValue parses the first numeric data point out of `gcx metrics // query -o json` output. The schema follows gcx's JSON shape; we only look // at the fields we need so additions don't break us. diff --git a/runner/runner_test.go b/runner/runner_test.go index e40e3d6c..3641017c 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -419,6 +419,9 @@ func TestApproxRowCount(t *testing.T) { {"a\nb\nc", 3}, {"a\n\nb", 2}, {"─\na\n═\n+\nb", 2}, + {"TRACE_ID SERVICE NAME DURATION START", 0}, + {"TIME STREAM MESSAGE", 0}, + {"TRACE_ID SERVICE NAME DURATION START\nabc svc GET /x 1ms now", 1}, } for _, tc := range cases { got := approxRowCount(tc.in) From 6886479dc3806e389a4b8e90947ed7167fe94eb0 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 19 Jun 2026 13:29:13 +0200 Subject: [PATCH 060/124] Normalize k3d cluster names for suite tests Signed-off-by: Gregor Zeitlinger --- testhelpers/kubernetes/kubernetes.go | 31 +++++++++++++++++++++-- testhelpers/kubernetes/kubernetes_test.go | 9 +++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/testhelpers/kubernetes/kubernetes.go b/testhelpers/kubernetes/kubernetes.go index 630fd4d8..37cd1913 100644 --- a/testhelpers/kubernetes/kubernetes.go +++ b/testhelpers/kubernetes/kubernetes.go @@ -7,7 +7,9 @@ import ( "log/slog" "os" "os/exec" + "strings" "sync" + "unicode" "github.com/grafana/oats/testhelpers/remote" ) @@ -128,9 +130,34 @@ func start(model *Kubernetes, ports remote.PortsConfig, testName string, run fun } func clusterName(testName string) string { - cluster := testName + var b strings.Builder + lastDash := false + for _, r := range strings.ToLower(testName) { + switch { + case (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9'): + b.WriteRune(r) + lastDash = false + case r == '-' || r == '_' || unicode.IsSpace(r): + if !lastDash && b.Len() > 0 { + b.WriteByte('-') + lastDash = true + } + default: + if !lastDash && b.Len() > 0 { + b.WriteByte('-') + lastDash = true + } + } + } + cluster := strings.Trim(b.String(), "-") + if cluster == "" { + cluster = "oats" + } if len(cluster) > 32 { - cluster = cluster[len(cluster)-32:] + cluster = strings.Trim(cluster[len(cluster)-32:], "-") + if cluster == "" { + cluster = "oats" + } } return cluster } diff --git a/testhelpers/kubernetes/kubernetes_test.go b/testhelpers/kubernetes/kubernetes_test.go index 5681f25c..87e5fc11 100644 --- a/testhelpers/kubernetes/kubernetes_test.go +++ b/testhelpers/kubernetes/kubernetes_test.go @@ -20,6 +20,15 @@ func TestClusterName_TruncatesFromEnd(t *testing.T) { } } +func TestClusterName_NormalizesRFC1123(t *testing.T) { + if got := clusterName("logging k8s probe"); got != "logging-k8s-probe" { + t.Fatalf("clusterName() = %q, want %q", got, "logging-k8s-probe") + } + if got := clusterName("!!!"); got != "oats" { + t.Fatalf("clusterName() fallback = %q, want %q", got, "oats") + } +} + func TestStart_DefaultDockerContextAndCommandSequence(t *testing.T) { model := &Kubernetes{ Dir: "k8s", From b5c0336745ce110fc4d5312c6b2979fd262febc2 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 19 Jun 2026 13:51:50 +0200 Subject: [PATCH 061/124] Fix gcx profile query argument mapping Signed-off-by: Gregor Zeitlinger --- signalcmd/signalcmd.go | 26 +++++++++++++++++++++++++- signalcmd/signalcmd_test.go | 13 +++++++++++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/signalcmd/signalcmd.go b/signalcmd/signalcmd.go index 46e36d6c..457946fe 100644 --- a/signalcmd/signalcmd.go +++ b/signalcmd/signalcmd.go @@ -11,6 +11,7 @@ package signalcmd import ( + "strings" "time" "github.com/grafana/oats/v2case" @@ -77,6 +78,7 @@ func Profiles(a v2case.ProfileAssertion, since time.Duration) []string { if since <= 0 { since = DefaultSince } + profileType, expr := splitProfileQuery(a.Query) args := []string{ "profiles", "query", "--since", since.String(), @@ -84,10 +86,32 @@ func Profiles(a v2case.ProfileAssertion, since time.Duration) []string { if len(a.Match) > 0 { args = append(args, "-o", "json") } - args = append(args, a.Query) + if profileType != "" { + args = append(args, "--profile-type", profileType) + } + args = append(args, expr) return args } +func splitProfileQuery(query string) (profileType string, expr string) { + q := strings.TrimSpace(query) + if q == "" { + return "", "{}" + } + if i := strings.Index(q, "{"); i >= 0 { + profileType = strings.TrimSpace(q[:i]) + expr = strings.TrimSpace(q[i:]) + if expr == "" { + expr = "{}" + } + return profileType, expr + } + if strings.Contains(q, ":") { + return q, "{}" + } + return "", q +} + // Render is a convenience used by the report layer to show the gcx // invocation in a FAIL block. It mirrors what a shell would print. func Render(args []string) string { diff --git a/signalcmd/signalcmd_test.go b/signalcmd/signalcmd_test.go index 17cc765e..9484482c 100644 --- a/signalcmd/signalcmd_test.go +++ b/signalcmd/signalcmd_test.go @@ -67,8 +67,17 @@ func TestMetrics_WithValueAsksForJSON(t *testing.T) { func TestProfiles(t *testing.T) { got := Profiles(v2case.ProfileAssertion{Query: "process_cpu:cpu:nanoseconds:cpu:nanoseconds{}"}, 0) - if got[0] != "profiles" || got[1] != "query" { - t.Errorf("wrong verb chain: %v", got) + want := []string{"profiles", "query", "--since", "10m0s", "--profile-type", "process_cpu:cpu:nanoseconds:cpu:nanoseconds", "{}"} + if !equal(got, want) { + t.Errorf("got %v\nwant %v", got, want) + } +} + +func TestProfiles_QueryWithoutSelectorDefaultsToEmptySelector(t *testing.T) { + got := Profiles(v2case.ProfileAssertion{Query: "process_cpu:cpu:nanoseconds:cpu:nanoseconds"}, 0) + want := []string{"profiles", "query", "--since", "10m0s", "--profile-type", "process_cpu:cpu:nanoseconds:cpu:nanoseconds", "{}"} + if !equal(got, want) { + t.Errorf("got %v\nwant %v", got, want) } } From dfaf7492e58eaa01f5fc377bd4ef464821762eb0 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 23 Jun 2026 15:15:54 +0000 Subject: [PATCH 062/124] feat: finish gcx rollout assertion cleanup Signed-off-by: Gregor Zeitlinger --- CURRENT.md | 30 +++--- assert/assert.go | 20 ++-- assert/assert_test.go | 19 ++-- migrate/migrate.go | 11 +-- migrate/migrate_test.go | 6 +- runner/runner.go | 210 +++++++++++++++++++++++++++++++++++++++- signalcmd/signalcmd.go | 13 +++ v2case/case.go | 159 +++++++++++++++++++++--------- v2case/case_test.go | 84 ++++++++++++++-- 9 files changed, 449 insertions(+), 103 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index 2c45b965..b6c08b59 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -147,13 +147,15 @@ expected: value: '>= 0' match: - attributes: - service_name: dice-server + - key: service_name + value: dice-server logs: - logql: '{service_name="dice-server"} |~ `Received request`' match: - name: "Received request to roll dice" attributes: - service_name: dice-server + - key: service_name + value: dice-server custom-checks: - script: ./verify.sh ``` @@ -181,7 +183,8 @@ expected: match_spans: - name: seed-operation attributes: - service.name: gcx-e2e-seed + - key: service.name + value: gcx-e2e-seed ``` ### Assertion keys @@ -191,9 +194,9 @@ Per signal under `expected.[]`: | Key | Meaning | |-----|---------| | `traceql` / `promql` / `logql` / `query` | The query string. | -| `contains` | Substrings that must appear in gcx stdout. | -| `not_contains` | Substrings that must not appear. | -| `regex` | Patterns that must match. | +| `contains` | Substrings that must appear in gcx stdout. Accepts a string or list of strings. | +| `not_contains` | Substrings that must not appear. Accepts a string or list of strings. | +| `regex` | Patterns that must match. Accepts a string or list of strings. | | `match_spans` | Trace-only structural assertions over returned spans using collector-style `match_type: strict | regexp`. | | `match` | Structural assertions for logs / metrics / profiles using collector-style `match_type: strict | regexp`. | | `value` | Metrics only — numeric comparison (`>= 0`, `== 42`). | @@ -212,19 +215,22 @@ behavior. For remote fixtures, point those requests at a running app with `--app-host` and `--app-port` (defaults: `localhost:8080`). `match_spans` (for traces) and `match` (for logs / metrics / profiles) -default to `match_type: strict`. OATS also supports the small convenience -extension `present: true` for attribute existence checks: +default to `match_type: strict`. Attributes follow collector-style +`[{ key, value? }]` entries; omitting `value` means “the key must be present”. +Text assertions keep list semantics internally, but author-facing YAML may use +either a scalar string or a list of strings. ```yaml match: - name: seed-operation attributes: - service.name: gcx-e2e-seed - trace_id: - present: true + - key: service.name + value: gcx-e2e-seed + - key: trace_id - match_type: regexp attributes: - http.route: "^/roll.*$" + - key: http.route + value: "^/roll.*$" ``` ## What's not here yet diff --git a/assert/assert.go b/assert/assert.go index ccb5117f..d32eabf4 100644 --- a/assert/assert.go +++ b/assert/assert.go @@ -162,15 +162,15 @@ func rowMatches(row Row, entry v2case.MatchEntry) bool { return false } } - for key, expected := range entry.Attributes { - actual, ok := row.Attributes[key] - if expected.Present != nil { + for _, expected := range entry.Attributes { + actual, ok := row.Attributes[expected.Key] + if expected.Value == nil { if !ok { return false } continue } - if !ok || expected.Value == nil { + if !ok { return false } if !matchesValue(actual, *expected.Value, matchType) { @@ -201,12 +201,12 @@ func describeMatch(entry v2case.MatchEntry) string { if entry.Name != nil { parts = append(parts, fmt.Sprintf("name=%q", *entry.Name)) } - for key, expected := range entry.Attributes { - switch { - case expected.Present != nil: - parts = append(parts, fmt.Sprintf("attribute %s present", key)) - case expected.Value != nil: - parts = append(parts, fmt.Sprintf("attribute %s=%q", key, *expected.Value)) + for _, expected := range entry.Attributes { + switch expected.Value { + case nil: + parts = append(parts, fmt.Sprintf("attribute %s present", expected.Key)) + default: + parts = append(parts, fmt.Sprintf("attribute %s=%q", expected.Key, *expected.Value)) } } if len(parts) == 0 { diff --git a/assert/assert_test.go b/assert/assert_test.go index 630f2fc0..84ad1262 100644 --- a/assert/assert_test.go +++ b/assert/assert_test.go @@ -99,7 +99,6 @@ func TestAbsent(t *testing.T) { } func TestMatchRows(t *testing.T) { - present := true rows := []Row{ { Name: "seed-operation", @@ -113,9 +112,9 @@ func TestMatchRows(t *testing.T) { got := MatchRows(rows, []v2case.MatchEntry{ { Name: strPtr("seed-operation"), - Attributes: map[string]v2case.AttributeExpectation{ - "service.name": {Value: strPtr("gcx-e2e-seed")}, - "trace_id": {Present: &present}, + Attributes: v2case.AttributeMatchers{ + {Key: "service.name", Value: strPtr("gcx-e2e-seed")}, + {Key: "trace_id"}, }, }, }) @@ -125,11 +124,9 @@ func TestMatchRows(t *testing.T) { got = MatchRows(rows, []v2case.MatchEntry{ { - MatchType: v2case.MatchTypeRegexp, - Name: strPtr("^seed-.*$"), - Attributes: map[string]v2case.AttributeExpectation{ - "trace_id": {Value: strPtr("^abc")}, - }, + MatchType: v2case.MatchTypeRegexp, + Name: strPtr("^seed-.*$"), + Attributes: v2case.AttributeMatchers{{Key: "trace_id", Value: strPtr("^abc")}}, }, }) if len(got) != 0 { @@ -138,9 +135,7 @@ func TestMatchRows(t *testing.T) { got = MatchRows(rows, []v2case.MatchEntry{ { - Attributes: map[string]v2case.AttributeExpectation{ - "missing": {Present: &present}, - }, + Attributes: v2case.AttributeMatchers{{Key: "missing"}}, }, }) if len(got) != 1 || got[0].Rule != "match" { diff --git a/migrate/migrate.go b/migrate/migrate.go index 9388440c..50be02d0 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -190,26 +190,25 @@ func convertSignal(label string, s model.ExpectedSignal) (v2case.AssertionCommon warnings = append(warnings, fmt.Sprintf("%s matrix-condition dropped", label)) } if s.NameEquals != "" || len(s.Attributes) > 0 { - entry := v2case.MatchEntry{Attributes: map[string]v2case.AttributeExpectation{}} + entry := v2case.MatchEntry{} if s.NameEquals != "" { entry.Name = strPtr(s.NameEquals) } for k, v := range s.Attributes { - entry.Attributes[k] = v2case.AttributeExpectation{Value: strPtr(v)} + entry.Attributes = append(entry.Attributes, v2case.AttributeMatcher{Key: k, Value: strPtr(v)}) } out.Match = append(out.Match, entry) } if s.NameRegexp != "" || len(s.AttributeRegexp) > 0 { - entry := v2case.MatchEntry{MatchType: v2case.MatchTypeRegexp, Attributes: map[string]v2case.AttributeExpectation{}} + entry := v2case.MatchEntry{MatchType: v2case.MatchTypeRegexp} if s.NameRegexp != "" { entry.Name = strPtr(s.NameRegexp) } for k, v := range s.AttributeRegexp { if v == ".*" { - present := true - entry.Attributes[k] = v2case.AttributeExpectation{Present: &present} + entry.Attributes = append(entry.Attributes, v2case.AttributeMatcher{Key: k}) } else { - entry.Attributes[k] = v2case.AttributeExpectation{Value: strPtr(v)} + entry.Attributes = append(entry.Attributes, v2case.AttributeMatcher{Key: k, Value: strPtr(v)}) } } if entry.Name != nil || len(entry.Attributes) > 0 { diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index defc8ba6..c43a4669 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -62,8 +62,8 @@ func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { if len(c.Expected.Traces) != 1 || len(c.Expected.Traces[0].MatchSpans) != 2 { fatalf(t, "expected trace strict+regexp split, got %+v", c.Expected.Traces) } - if got := c.Expected.Traces[0].MatchSpans[1].Attributes["trace_id"].Present; got == nil || !*got { - fatalf(t, "expected trace_id .* to map to present:true") + if got := c.Expected.Traces[0].MatchSpans[1].Attributes[0]; got.Key != "trace_id" || got.Value != nil { + fatalf(t, "expected trace_id .* to map to presence, got %+v", got) } if c.Expected.Logs[0].Count != ">= 1" { fatalf(t, "expected lossy count min mapping, got %q", c.Expected.Logs[0].Count) @@ -89,7 +89,7 @@ func TestConvertFile_RendersYAML(t *testing.T) { fatalf(t, "expected at least one warning for flattened include or fixture migration") } text := string(out) - for _, want := range []string{"oats: 2", "seed:", "input:", "path: /stock", "match_spans:", "match_type: regexp", "db.system: h2", "promql: foo"} { + for _, want := range []string{"oats: 2", "seed:", "input:", "path: /stock", "match_spans:", "match_type: regexp", "key: db.system", "value: h2", "promql: foo"} { if !strings.Contains(text, want) { fatalf(t, "expected migrated yaml to contain %q:\n%s", want, text) } diff --git a/runner/runner.go b/runner/runner.go index 1a2dab71..9df298f5 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -239,6 +239,11 @@ func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { ok = false } } + for _, msg := range c.Expected.ComposeLogs { + if !r.runComposeLogCheck(ctx, c, msg) { + ok = false + } + } for i := range c.Expected.Custom { if !r.runCustomCheck(ctx, c, &c.Expected.Custom[i]) { ok = false @@ -262,6 +267,37 @@ func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { return ok } +func (r *Runner) runComposeLogCheck(ctx context.Context, c *v2case.Case, msg string) bool { + run := func() []assert.Failure { + if err := r.driveInputs(c); err != nil { + return []assert.Failure{{Rule: "input", Detail: err.Error()}} + } + ok, err := searchComposeLogs(ctx, r.endpoint.CustomCheckEnv, msg) + if err != nil { + return []assert.Failure{{Rule: "compose-logs", Detail: err.Error()}} + } + if !ok { + return []assert.Failure{{Rule: "compose-logs", Detail: fmt.Sprintf("missing %q", msg)}} + } + return nil + } + + result := wait.Until[assert.Failure](ctx, wait.Options{Timeout: r.opts.Timeout, Interval: r.caseInterval(c)}, run) + if result.OK { + return true + } + for _, f := range result.LastFailures { + r.reporter.Emit(report.Event{ + Type: report.EventAssertFail, + Case: c.Name, + Source: c.SourcePath, + Msg: f.Error(), + Cmd: "compose-logs", + }) + } + return false +} + func (r *Runner) runCustomCheck(ctx context.Context, c *v2case.Case, chk *v2case.CustomCheck) bool { dir := "." if c.SourcePath != "" { @@ -340,6 +376,31 @@ func customCheckCommand(ctx context.Context, dir, script string, extraEnv []stri return cmd, cleanup, nil } +func searchComposeLogs(ctx context.Context, extraEnv []string, needle string) (bool, error) { + if envValue(extraEnv, "OATS_FIXTURE_TYPE") != "compose" { + return false, fmt.Errorf("compose-logs requires a compose fixture") + } + cmd := exec.CommandContext(ctx, "docker", "compose", "logs", "--no-color") + cmd.Env = append(cmd.Environ(), extraEnv...) + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return false, fmt.Errorf("docker compose logs: %w\n%s", err, trimOutput(out.String())) + } + return strings.Contains(out.String(), needle), nil +} + +func envValue(env []string, key string) string { + prefix := key + "=" + for _, kv := range env { + if strings.HasPrefix(kv, prefix) { + return strings.TrimPrefix(kv, prefix) + } + } + return "" +} + func looksLikeInlineScript(script string) bool { trimmed := strings.TrimSpace(script) return strings.Contains(trimmed, "\n") || strings.HasPrefix(trimmed, "#!") @@ -479,16 +540,94 @@ func (r *Runner) caseInterval(c *v2case.Case) time.Duration { } func (r *Runner) runTrace(ctx context.Context, c *v2case.Case, a *v2case.TraceAssertion) bool { + if len(a.MatchSpans) > 0 { + return r.runTraceStructured(ctx, c, a) + } args := signalcmd.Traces(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { - if len(a.MatchSpans) == 0 { - return evalCommonText(stdout, a.AssertionCommon) - } - rows, count, err := extractTraceRows(stdout) - return evalTraceStructured(stdout, *a, rows, count, err) + return evalCommonText(stdout, a.AssertionCommon) }) } +func (r *Runner) runTraceStructured(ctx context.Context, c *v2case.Case, a *v2case.TraceAssertion) bool { + run := func() []assert.Failure { + if err := r.driveInputs(c); err != nil { + return []assert.Failure{{Rule: "input", Detail: err.Error()}} + } + searchArgs := signalcmd.Traces(*a, r.opts.Timeout) + searchCmd := signalcmd.Render(searchArgs) + searchRes, err := r.exec.Execute(ctx, searchArgs...) + if err != nil { + return []assert.Failure{{Rule: "exec", Detail: err.Error()}} + } + r.reporter.Emit(report.Event{Type: report.EventGCXExec, Case: c.Name, Cmd: searchCmd}) + if searchRes.ExitCode != 0 { + detail := strings.TrimSpace(searchRes.Stderr) + if detail == "" { + detail = fmt.Sprintf("gcx exit code %d", searchRes.ExitCode) + } + return []assert.Failure{{Rule: "exec", Detail: detail}} + } + rows, count, err := r.fetchTraceRows(ctx, c, searchRes.Stdout) + return evalTraceStructured(searchRes.Stdout, *a, rows, count, err) + } + + result := wait.Until[assert.Failure](ctx, wait.Options{Timeout: r.opts.Timeout, Interval: r.caseInterval(c)}, run) + if result.OK { + return true + } + cmdStr := signalcmd.Render(signalcmd.Traces(*a, r.opts.Timeout)) + for _, f := range result.LastFailures { + r.reporter.Emit(report.Event{ + Type: report.EventAssertFail, + Case: c.Name, + Source: c.SourcePath, + Msg: f.Error(), + Cmd: cmdStr, + }) + } + return false +} + +func (r *Runner) fetchTraceRows(ctx context.Context, c *v2case.Case, searchStdout string) ([]assert.Row, int, error) { + traceIDs, count, err := extractTraceIDs(searchStdout) + if err != nil { + return nil, 0, err + } + if len(traceIDs) == 0 { + rows, parsedCount, err := extractTraceRows(searchStdout) + if err != nil { + return nil, count, err + } + if count == 0 { + count = parsedCount + } + return rows, count, nil + } + var rows []assert.Row + for _, traceID := range traceIDs { + args := signalcmd.TraceGet(traceID, r.opts.Timeout) + res, err := r.exec.Execute(ctx, args...) + if err != nil { + return nil, count, fmt.Errorf("trace %s fetch: %w", traceID, err) + } + r.reporter.Emit(report.Event{Type: report.EventGCXExec, Case: c.Name, Cmd: signalcmd.Render(args)}) + if res.ExitCode != 0 { + detail := strings.TrimSpace(res.Stderr) + if detail == "" { + detail = fmt.Sprintf("gcx exit code %d", res.ExitCode) + } + return nil, count, fmt.Errorf("trace %s fetch: %s", traceID, detail) + } + traceRows, _, err := extractTraceRows(res.Stdout) + if err != nil { + return nil, count, err + } + rows = append(rows, traceRows...) + } + return rows, count, nil +} + func (r *Runner) runLog(ctx context.Context, c *v2case.Case, a *v2case.LogAssertion) bool { args := signalcmd.Logs(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { @@ -785,6 +924,27 @@ func extractTraceRows(stdout string) ([]assert.Row, int, error) { return rows, count, nil } +func extractTraceIDs(stdout string) ([]string, int, error) { + if strings.TrimSpace(stdout) == "" { + return nil, 0, nil + } + var payload struct { + Traces []struct { + TraceID string `json:"traceID"` + } `json:"traces"` + } + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + return nil, 0, fmt.Errorf("trace JSON parse: %w", err) + } + ids := make([]string, 0, len(payload.Traces)) + for _, tr := range payload.Traces { + if tr.TraceID != "" { + ids = append(ids, tr.TraceID) + } + } + return ids, len(payload.Traces), nil +} + func extractProfileRows(stdout string) ([]assert.Row, int, error) { if strings.TrimSpace(stdout) == "" { return nil, 0, nil @@ -800,6 +960,13 @@ func extractProfileRows(stdout string) ([]assert.Row, int, error) { } return rows, len(rows), nil } + if names := flamegraphNames(root); len(names) > 0 { + rows := make([]assert.Row, 0, len(names)) + for _, name := range names { + rows = append(rows, assert.Row{Name: name, Attributes: map[string]string{}}) + } + return rows, len(rows), nil + } rows := collectNamedRows(root) return rows, len(rows), nil } @@ -891,6 +1058,9 @@ func extractOTLPTraceRows(root any) ([]assert.Row, bool) { if !ok { return nil, false } + if trace, ok := top["trace"].(map[string]any); ok { + top = trace + } resourceSpans, ok := top["resourceSpans"].([]any) if !ok { resourceSpans, ok = top["batches"].([]any) @@ -943,6 +1113,7 @@ func extractOTLPTraceRows(root any) ([]assert.Row, bool) { } if scopeName != "" { attrs["otel.scope.name"] = scopeName + attrs["otel.library.name"] = scopeName } if kind, ok := sp["kind"]; ok { attrs["kind"] = fmt.Sprint(kind) @@ -1025,3 +1196,32 @@ func flamebearerNames(root any) []string { } return out } + +func flamegraphNames(root any) []string { + top, ok := root.(map[string]any) + if !ok { + return nil + } + flamegraph, ok := top["flamegraph"].(map[string]any) + if !ok { + if data, ok := top["data"].(map[string]any); ok { + flamegraph, _ = data["flamegraph"].(map[string]any) + } + } + if flamegraph == nil { + return nil + } + raw, ok := flamegraph["names"].([]any) + if !ok { + return nil + } + out := make([]string, 0, len(raw)) + for _, item := range raw { + s := fmt.Sprint(item) + if s == "" { + continue + } + out = append(out, s) + } + return out +} diff --git a/signalcmd/signalcmd.go b/signalcmd/signalcmd.go index 457946fe..f68a4a04 100644 --- a/signalcmd/signalcmd.go +++ b/signalcmd/signalcmd.go @@ -38,6 +38,19 @@ func Traces(a v2case.TraceAssertion, since time.Duration) []string { return args } +// TraceGet builds the gcx args to retrieve one trace by ID as OTLP-shaped JSON. +func TraceGet(traceID string, since time.Duration) []string { + if since <= 0 { + since = DefaultSince + } + return []string{ + "traces", "get", + "--since", since.String(), + "-o", "json", + traceID, + } +} + // Logs builds the gcx args for a LogAssertion. func Logs(a v2case.LogAssertion, since time.Duration) []string { if since <= 0 { diff --git a/v2case/case.go b/v2case/case.go index 0284c7a1..2799c8f8 100644 --- a/v2case/case.go +++ b/v2case/case.go @@ -114,11 +114,12 @@ type Input struct { // does not care about; an empty Expected makes the case a no-op (rejected at // validation). type Expected struct { - Traces []TraceAssertion `yaml:"traces,omitempty"` - Metrics []MetricAssertion `yaml:"metrics,omitempty"` - Logs []LogAssertion `yaml:"logs,omitempty"` - Profiles []ProfileAssertion `yaml:"profiles,omitempty"` - Custom []CustomCheck `yaml:"custom-checks,omitempty"` + Traces []TraceAssertion `yaml:"traces,omitempty"` + Metrics []MetricAssertion `yaml:"metrics,omitempty"` + Logs []LogAssertion `yaml:"logs,omitempty"` + Profiles []ProfileAssertion `yaml:"profiles,omitempty"` + ComposeLogs []string `yaml:"compose-logs,omitempty"` + Custom []CustomCheck `yaml:"custom-checks,omitempty"` } type CustomCheck struct { @@ -129,9 +130,9 @@ type CustomCheck struct { // Embedded into each concrete assertion struct so a case author can mix and // match contains / regex / count / absent on any signal. type AssertionCommon struct { - Contains []string `yaml:"contains,omitempty"` - NotContains []string `yaml:"not_contains,omitempty"` - Regex []string `yaml:"regex,omitempty"` + Contains StringList `yaml:"contains,omitempty"` + NotContains StringList `yaml:"not_contains,omitempty"` + Regex StringList `yaml:"regex,omitempty"` Match []MatchEntry `yaml:"match,omitempty"` Count string `yaml:"count,omitempty"` // ">= 1", "== 0", ... Absent bool `yaml:"absent,omitempty"` @@ -145,9 +146,9 @@ const ( ) type MatchEntry struct { - MatchType MatchType `yaml:"match_type,omitempty"` - Name *string `yaml:"name,omitempty"` - Attributes map[string]AttributeExpectation `yaml:"attributes,omitempty"` + MatchType MatchType `yaml:"match_type,omitempty"` + Name *string `yaml:"name,omitempty"` + Attributes AttributeMatchers `yaml:"attributes,omitempty"` } func (m MatchEntry) EffectiveMatchType() MatchType { @@ -157,46 +158,115 @@ func (m MatchEntry) EffectiveMatchType() MatchType { return m.MatchType } -type AttributeExpectation struct { - Value *string - Present *bool -} +type StringList []string -func (a *AttributeExpectation) UnmarshalYAML(node *yaml.Node) error { +func (s *StringList) UnmarshalYAML(node *yaml.Node) error { switch node.Kind { - case yaml.ScalarNode: - v := node.Value - a.Value = &v - a.Present = nil + case 0: + *s = nil return nil - case yaml.MappingNode: - var aux struct { - Present *bool `yaml:"present"` + case yaml.ScalarNode: + var v string + if err := node.Decode(&v); err != nil { + return err } - if err := node.Decode(&aux); err != nil { + *s = []string{v} + return nil + case yaml.SequenceNode: + var v []string + if err := node.Decode(&v); err != nil { return err } - a.Value = nil - a.Present = aux.Present + *s = v return nil default: - return fmt.Errorf("expected scalar string or mapping {present: true}") + return fmt.Errorf("expected string or list of strings") } } -func (a AttributeExpectation) MarshalYAML() (any, error) { - switch { - case a.Value != nil && a.Present == nil: - return *a.Value, nil - case a.Value == nil && a.Present != nil: - return map[string]bool{"present": *a.Present}, nil - case a.Value == nil && a.Present == nil: +func (s StringList) MarshalYAML() (any, error) { + switch len(s) { + case 0: return nil, nil + case 1: + return s[0], nil default: - return nil, fmt.Errorf("attribute expectation cannot marshal value and present simultaneously") + return []string(s), nil } } +type AttributeMatcher struct { + Key string + Value *string +} + +type AttributeMatchers []AttributeMatcher + +func (a *AttributeMatchers) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.SequenceNode: + type attrDoc struct { + Key string `yaml:"key"` + Value *string `yaml:"value,omitempty"` + } + var docs []attrDoc + if err := node.Decode(&docs); err != nil { + return err + } + out := make([]AttributeMatcher, 0, len(docs)) + for _, doc := range docs { + out = append(out, AttributeMatcher(doc)) + } + *a = out + return nil + case yaml.MappingNode: + // Backward-compatible input: attributes: {key: value} or + // attributes: {key: {present: true}}. + out := make([]AttributeMatcher, 0, len(node.Content)/2) + for i := 0; i < len(node.Content); i += 2 { + keyNode, valueNode := node.Content[i], node.Content[i+1] + attr := AttributeMatcher{Key: keyNode.Value} + switch valueNode.Kind { + case yaml.ScalarNode: + v := valueNode.Value + attr.Value = &v + case yaml.MappingNode: + var aux struct { + Present *bool `yaml:"present"` + } + if err := valueNode.Decode(&aux); err != nil { + return err + } + if aux.Present == nil || !*aux.Present { + return fmt.Errorf("expected scalar value or {present: true}") + } + default: + return fmt.Errorf("expected scalar value or {present: true}") + } + out = append(out, attr) + } + *a = out + return nil + default: + return fmt.Errorf("expected list of {key, value?} or legacy mapping") + } +} + +func (a AttributeMatchers) MarshalYAML() (any, error) { + if len(a) == 0 { + return nil, nil + } + type attrDoc struct { + Key string `yaml:"key"` + Value *string `yaml:"value,omitempty"` + } + out := make([]attrDoc, 0, len(a)) + for _, attr := range a { + out = append(out, attrDoc(attr)) + } + return out, nil +} + type TraceAssertion struct { TraceQL string `yaml:"traceql"` MatchSpans []MatchEntry `yaml:"match_spans,omitempty"` @@ -402,20 +472,19 @@ func validateMatchEntries(path string, entries []MatchEntry) error { return fmt.Errorf("%s.name: invalid regexp %q: %v", matchPath, *m.Name, err) } } - for key, attr := range m.Attributes { - attrPath := fmt.Sprintf("%s.attributes[%q]", matchPath, key) - if attr.Value != nil && attr.Present != nil { - return fmt.Errorf("%s: expected either scalar value or {present: true}, not both", attrPath) - } - if attr.Value == nil && attr.Present == nil { - return fmt.Errorf("%s: expected scalar value or {present: true}", attrPath) + seenKeys := map[string]struct{}{} + for i, attr := range m.Attributes { + attrPath := fmt.Sprintf("%s.attributes[%d]", matchPath, i) + if strings.TrimSpace(attr.Key) == "" { + return fmt.Errorf("%s.key: required", attrPath) } - if attr.Present != nil && !*attr.Present { - return fmt.Errorf("%s.present: only true is allowed", attrPath) + if _, ok := seenKeys[attr.Key]; ok { + return fmt.Errorf("%s.key: duplicate key %q", attrPath, attr.Key) } + seenKeys[attr.Key] = struct{}{} if m.EffectiveMatchType() == MatchTypeRegexp && attr.Value != nil { if _, err := regexp.Compile(*attr.Value); err != nil { - return fmt.Errorf("%s: invalid regexp %q: %v", attrPath, *attr.Value, err) + return fmt.Errorf("%s.value: invalid regexp %q: %v", attrPath, *attr.Value, err) } } } diff --git a/v2case/case_test.go b/v2case/case_test.go index 2e3545c0..d4e35557 100644 --- a/v2case/case_test.go +++ b/v2case/case_test.go @@ -88,12 +88,13 @@ expected: match: - name: "seed-log-line" attributes: - service_name: svc - trace_id: - present: true + - key: service_name + value: svc + - key: trace_id - match_type: regexp attributes: - level: "info|warn" + - key: level + value: "info|warn" `) c, err := Parse(src) if err != nil { @@ -105,14 +106,75 @@ expected: if c.Expected.Logs[0].Match[0].EffectiveMatchType() != MatchTypeStrict { t.Fatalf("default match_type should be strict") } - if c.Expected.Logs[0].Match[0].Attributes["trace_id"].Present == nil { - t.Fatalf("trace_id.present should be set") + if got := c.Expected.Logs[0].Match[0].Attributes[1]; got.Key != "trace_id" || got.Value != nil { + t.Fatalf("trace_id presence should be encoded as key-only entry, got %+v", got) } if c.Expected.Logs[0].Match[1].EffectiveMatchType() != MatchTypeRegexp { t.Fatalf("second entry should be regexp") } } +func TestParse_StringListScalarOrList(t *testing.T) { + src := []byte(` +oats: 2 +name: text assertions +seed: + type: app +expected: + logs: + - logql: '{service_name="svc"}' + contains: hello + not_contains: + - panic + regex: 'warn|error' +`) + c, err := Parse(src) + if err != nil { + t.Fatalf("Parse: %v", err) + } + log := c.Expected.Logs[0] + if got := []string(log.Contains); len(got) != 1 || got[0] != "hello" { + t.Fatalf("contains scalar not normalized correctly: %#v", got) + } + if got := []string(log.NotContains); len(got) != 1 || got[0] != "panic" { + t.Fatalf("not_contains list parse failed: %#v", got) + } + if got := []string(log.Regex); len(got) != 1 || got[0] != "warn|error" { + t.Fatalf("regex scalar not normalized correctly: %#v", got) + } +} + +func TestParse_LegacyAttributeMapStillAccepted(t *testing.T) { + src := []byte(` +oats: 2 +name: legacy attrs +seed: + type: app +expected: + traces: + - traceql: '{}' + match_spans: + - attributes: + service.name: svc + trace_id: + present: true +`) + c, err := Parse(src) + if err != nil { + t.Fatalf("Parse: %v", err) + } + attrs := c.Expected.Traces[0].MatchSpans[0].Attributes + if len(attrs) != 2 { + t.Fatalf("expected 2 attrs, got %+v", attrs) + } + if attrs[0].Key != "service.name" || attrs[0].Value == nil || *attrs[0].Value != "svc" { + t.Fatalf("unexpected first attr: %+v", attrs[0]) + } + if attrs[1].Key != "trace_id" || attrs[1].Value != nil { + t.Fatalf("unexpected presence attr: %+v", attrs[1]) + } +} + func TestParse_CustomChecks(t *testing.T) { src := []byte(` oats: 2 @@ -276,11 +338,13 @@ expected: - logql: '{job="x"}' match: - attributes: - trace_id: - present: false + - key: trace_id + value: foo + - key: trace_id + value: bar `)) - if err == nil || !strings.Contains(err.Error(), "only true is allowed") { - t.Fatalf("expected present error, got %v", err) + if err == nil || !strings.Contains(err.Error(), "duplicate key") { + t.Fatalf("expected duplicate key error, got %v", err) } } From f78781aff620b6a4039cfcc02750a64dffd3eb4b Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 2 Jul 2026 10:22:37 +0000 Subject: [PATCH 063/124] test: normalize yaml fixtures after rebase Signed-off-by: Gregor Zeitlinger --- examples/fixtures/cases/compose/rolldice.yaml | 1 - examples/fixtures/cases/k3d/rolldice.yaml | 1 - examples/smoke/cases/custom-check.yaml | 1 - examples/smoke/cases/inline-seed.yaml | 1 - examples/smoke/cases/profile.yaml | 1 - 5 files changed, 5 deletions(-) diff --git a/examples/fixtures/cases/compose/rolldice.yaml b/examples/fixtures/cases/compose/rolldice.yaml index 9608351d..3f800de6 100644 --- a/examples/fixtures/cases/compose/rolldice.yaml +++ b/examples/fixtures/cases/compose/rolldice.yaml @@ -15,4 +15,3 @@ expected: match: - match_type: regexp name: '.*roll.*' - diff --git a/examples/fixtures/cases/k3d/rolldice.yaml b/examples/fixtures/cases/k3d/rolldice.yaml index 210a35c5..a2cf6409 100644 --- a/examples/fixtures/cases/k3d/rolldice.yaml +++ b/examples/fixtures/cases/k3d/rolldice.yaml @@ -10,4 +10,3 @@ expected: value: '>= 0' custom-checks: - script: ../../scripts/verify.sh - diff --git a/examples/smoke/cases/custom-check.yaml b/examples/smoke/cases/custom-check.yaml index 0990fa8b..eac350c1 100644 --- a/examples/smoke/cases/custom-check.yaml +++ b/examples/smoke/cases/custom-check.yaml @@ -5,4 +5,3 @@ seed: expected: custom-checks: - script: ../scripts/verify.sh - diff --git a/examples/smoke/cases/inline-seed.yaml b/examples/smoke/cases/inline-seed.yaml index 85b96a66..26ca5653 100644 --- a/examples/smoke/cases/inline-seed.yaml +++ b/examples/smoke/cases/inline-seed.yaml @@ -13,4 +13,3 @@ expected: - name: seed-operation attributes: service.name: gcx-e2e-seed - diff --git a/examples/smoke/cases/profile.yaml b/examples/smoke/cases/profile.yaml index a79b2471..257418b9 100644 --- a/examples/smoke/cases/profile.yaml +++ b/examples/smoke/cases/profile.yaml @@ -8,4 +8,3 @@ expected: match: - match_type: regexp name: 'main|worker' - From 6eca67c2a5ac2d9dbcf5508112028fa69cdf2c9b Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 2 Jul 2026 10:28:36 +0000 Subject: [PATCH 064/124] fix: make current docs and tests CI-safe Signed-off-by: Gregor Zeitlinger --- CURRENT.md | 4 ++-- migrate/migrate_test.go | 6 ++++-- runner/runner_test.go | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CURRENT.md b/CURRENT.md index b6c08b59..6a84b376 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -3,7 +3,7 @@ This document covers the current gcx-driven OATS CLI, which replaces the bespoke TraceQL / PromQL / LogQL HTTP query infrastructure with the [gcx](https://github.com/grafana/gcx) CLI. The current CLI entry point lives at `cmd/v2/main.go` while this replaces the legacy path. See -[grafana/internal-docs#14](https://github.com/grafana/internal-docs/pull/14) +the internal design doc (grafana/internal-docs#14) for the full design. ## Quick start @@ -246,7 +246,7 @@ match: ## Migrating from v1 For the legacy → current migration story see the OATS implementation plan in -[grafana/internal-docs#14](https://github.com/grafana/internal-docs/pull/14). +the internal design doc (grafana/internal-docs#14). Today a best-effort converter exists: ```bash diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index c43a4669..c7c6c57b 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -81,7 +81,8 @@ func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { } func TestConvertFile_RendersYAML(t *testing.T) { - out, warnings, err := ConvertFile("/home/gregor/source/oats-v2/yaml/testdata/valid-tests/oats.yaml") + sample := filepath.Join("..", "yaml", "testdata", "valid-tests", "oats.yaml") + out, warnings, err := ConvertFile(sample) if err != nil { fatalf(t, "ConvertFile: %v", err) } @@ -97,7 +98,8 @@ func TestConvertFile_RendersYAML(t *testing.T) { } func TestConvertFile_MatrixSampleIsParseable(t *testing.T) { - out, warnings, err := ConvertFile("/home/gregor/source/oats-v2/yaml/testdata/valid-tests/matrix-test.oats.yaml") + sample := filepath.Join("..", "yaml", "testdata", "valid-tests", "matrix-test.oats.yaml") + out, warnings, err := ConvertFile(sample) if err != nil { fatalf(t, "ConvertFile matrix sample: %v", err) } diff --git a/runner/runner_test.go b/runner/runner_test.go index 3641017c..a709c16e 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -432,7 +432,8 @@ func TestApproxRowCount(t *testing.T) { } func TestExtractTraceRows_OTLPShape(t *testing.T) { - data, err := os.ReadFile("/home/gregor/source/oats-v2/testhelpers/tempo/responses/testdata/trace_by_id.json") + path := filepath.Join("..", "testhelpers", "tempo", "responses", "testdata", "trace_by_id.json") + data, err := os.ReadFile(path) if err != nil { t.Fatalf("ReadFile: %v", err) } From 1e7ba714660193cc2693c82d6083010c82672145 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 2 Jul 2026 17:36:54 +0200 Subject: [PATCH 065/124] refactor tests and remove legacy v2 naming Signed-off-by: Gregor Zeitlinger --- .github/renovate-tracked-deps.json | 5 - .github/workflows/e2e_test.yml | 24 +- .github/workflows/integration_test.yml | 21 - AGENTS.md | 36 +- CURRENT.md | 8 +- README.md | 462 +------------- UPGRADING.md | 18 + assert/assert.go | 14 +- assert/assert_test.go | 16 +- {v2case => casefile}/case.go | 8 +- {v2case => casefile}/case_test.go | 2 +- discovery/discovery.go | 20 +- {cmd/v2 => internal/cli}/integration_test.go | 46 +- {cmd/v2 => internal/cli}/main.go | 15 +- {cmd/v2 => internal/cli}/testdata/fake-gcx.sh | 51 +- main.go | 91 +-- migrate/migrate.go | 58 +- migrate/migrate_test.go | 6 +- mise.toml | 9 +- runner/cache_integration_test.go | 8 +- runner/runner.go | 60 +- runner/runner_test.go | 131 +++- scripts/build-local-tools.sh | 2 +- signalcmd/signalcmd.go | 12 +- signalcmd/signalcmd_test.go | 30 +- testhelpers/kubernetes/kubernetes.go | 11 +- testhelpers/kubernetes/kubernetes_test.go | 39 +- tests/e2e/README.md | 54 ++ .../cases/assert/absent-fail/files/oats.yaml | 14 + tests/e2e/cases/assert/absent-fail/test.yaml | 17 + tests/e2e/cases/assert/absent/files/oats.yaml | 14 + tests/e2e/cases/assert/absent/test.yaml | 6 + .../assert/contains-fail/files/oats.yaml | 14 + .../e2e/cases/assert/contains-fail/test.yaml | 15 + .../e2e/cases/assert/contains/files/oats.yaml | 14 + tests/e2e/cases/assert/contains/test.yaml | 6 + tests/e2e/cases/assert/count/files/oats.yaml | 16 + tests/e2e/cases/assert/count/test.yaml | 6 + .../cases/assert/match-fail/files/oats.yaml | 18 + tests/e2e/cases/assert/match-fail/test.yaml | 15 + .../cases/assert/match-regexp/files/oats.yaml | 19 + tests/e2e/cases/assert/match-regexp/test.yaml | 6 + .../assert/match-spans-fail/files/oats.yaml | 16 + .../cases/assert/match-spans-fail/test.yaml | 15 + .../cases/assert/not-contains/files/oats.yaml | 14 + tests/e2e/cases/assert/not-contains/test.yaml | 6 + .../cases/assert/regex-fail/files/oats.yaml | 14 + tests/e2e/cases/assert/regex-fail/test.yaml | 15 + tests/e2e/cases/assert/regex/files/oats.yaml | 14 + tests/e2e/cases/assert/regex/test.yaml | 6 + .../custom/custom-check-fail/files/oats.yaml | 13 + .../custom/custom-check-fail/files/verify.sh | 4 + .../cases/custom/custom-check-fail/test.yaml | 15 + .../cases/custom/custom-check/files/oats.yaml | 13 + .../cases/custom/custom-check/files/verify.sh | 7 + tests/e2e/cases/custom/custom-check/test.yaml | 6 + .../files/docker-compose.oats.yml | 4 + .../fixture/compose-logs-fail/files/oats.yaml | 17 + .../cases/fixture/compose-logs-fail/test.yaml | 18 + .../files/docker-compose.oats.yml | 4 + .../fixture/compose-logs/files/oats.yaml | 17 + .../e2e/cases/fixture/compose-logs/test.yaml | 9 + .../cases/fixture/k3d-fail/files/Dockerfile | 3 + .../fixture/k3d-fail/files/k8s/invalid.yaml | 3 + .../cases/fixture/k3d-fail/files/oats.yaml | 19 + tests/e2e/cases/fixture/k3d-fail/test.yaml | 11 + .../cases/fixture/k3d-smoke/files/Dockerfile | 3 + .../fixture/k3d-smoke/files/k8s/dice-pod.yaml | 24 + .../fixture/k3d-smoke/files/k8s/lgtm-pod.yaml | 39 ++ .../cases/fixture/k3d-smoke/files/oats.yaml | 19 + tests/e2e/cases/fixture/k3d-smoke/test.yaml | 9 + .../files/docker-compose.remote.yml | 6 + .../fixture/remote-basic/files/oats.yaml | 18 + .../e2e/cases/fixture/remote-basic/test.yaml | 37 ++ .../seed/inline-otlp-invalid/files/oats.yaml | 9 + .../cases/seed/inline-otlp-invalid/test.yaml | 7 + .../cases/seed/inline-otlp/files/oats.yaml | 29 + tests/e2e/cases/seed/inline-otlp/test.yaml | 6 + tests/e2e/e2e_test.go | 567 ++++++++++++++++++ tests/e2e/oats.yaml | 26 - tests/integration_test.go | 319 ---------- 81 files changed, 1754 insertions(+), 1064 deletions(-) delete mode 100644 .github/workflows/integration_test.yml rename {v2case => casefile}/case.go (98%) rename {v2case => casefile}/case_test.go (99%) rename {cmd/v2 => internal/cli}/integration_test.go (94%) rename {cmd/v2 => internal/cli}/main.go (98%) rename {cmd/v2 => internal/cli}/testdata/fake-gcx.sh (64%) create mode 100644 tests/e2e/README.md create mode 100644 tests/e2e/cases/assert/absent-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/absent-fail/test.yaml create mode 100644 tests/e2e/cases/assert/absent/files/oats.yaml create mode 100644 tests/e2e/cases/assert/absent/test.yaml create mode 100644 tests/e2e/cases/assert/contains-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/contains-fail/test.yaml create mode 100644 tests/e2e/cases/assert/contains/files/oats.yaml create mode 100644 tests/e2e/cases/assert/contains/test.yaml create mode 100644 tests/e2e/cases/assert/count/files/oats.yaml create mode 100644 tests/e2e/cases/assert/count/test.yaml create mode 100644 tests/e2e/cases/assert/match-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/match-fail/test.yaml create mode 100644 tests/e2e/cases/assert/match-regexp/files/oats.yaml create mode 100644 tests/e2e/cases/assert/match-regexp/test.yaml create mode 100644 tests/e2e/cases/assert/match-spans-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/match-spans-fail/test.yaml create mode 100644 tests/e2e/cases/assert/not-contains/files/oats.yaml create mode 100644 tests/e2e/cases/assert/not-contains/test.yaml create mode 100644 tests/e2e/cases/assert/regex-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/regex-fail/test.yaml create mode 100644 tests/e2e/cases/assert/regex/files/oats.yaml create mode 100644 tests/e2e/cases/assert/regex/test.yaml create mode 100644 tests/e2e/cases/custom/custom-check-fail/files/oats.yaml create mode 100644 tests/e2e/cases/custom/custom-check-fail/files/verify.sh create mode 100644 tests/e2e/cases/custom/custom-check-fail/test.yaml create mode 100644 tests/e2e/cases/custom/custom-check/files/oats.yaml create mode 100644 tests/e2e/cases/custom/custom-check/files/verify.sh create mode 100644 tests/e2e/cases/custom/custom-check/test.yaml create mode 100644 tests/e2e/cases/fixture/compose-logs-fail/files/docker-compose.oats.yml create mode 100644 tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml create mode 100644 tests/e2e/cases/fixture/compose-logs-fail/test.yaml create mode 100644 tests/e2e/cases/fixture/compose-logs/files/docker-compose.oats.yml create mode 100644 tests/e2e/cases/fixture/compose-logs/files/oats.yaml create mode 100644 tests/e2e/cases/fixture/compose-logs/test.yaml create mode 100644 tests/e2e/cases/fixture/k3d-fail/files/Dockerfile create mode 100644 tests/e2e/cases/fixture/k3d-fail/files/k8s/invalid.yaml create mode 100644 tests/e2e/cases/fixture/k3d-fail/files/oats.yaml create mode 100644 tests/e2e/cases/fixture/k3d-fail/test.yaml create mode 100644 tests/e2e/cases/fixture/k3d-smoke/files/Dockerfile create mode 100644 tests/e2e/cases/fixture/k3d-smoke/files/k8s/dice-pod.yaml create mode 100644 tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml create mode 100644 tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml create mode 100644 tests/e2e/cases/fixture/k3d-smoke/test.yaml create mode 100644 tests/e2e/cases/fixture/remote-basic/files/docker-compose.remote.yml create mode 100644 tests/e2e/cases/fixture/remote-basic/files/oats.yaml create mode 100644 tests/e2e/cases/fixture/remote-basic/test.yaml create mode 100644 tests/e2e/cases/seed/inline-otlp-invalid/files/oats.yaml create mode 100644 tests/e2e/cases/seed/inline-otlp-invalid/test.yaml create mode 100644 tests/e2e/cases/seed/inline-otlp/files/oats.yaml create mode 100644 tests/e2e/cases/seed/inline-otlp/test.yaml create mode 100644 tests/e2e/e2e_test.go delete mode 100644 tests/e2e/oats.yaml delete mode 100644 tests/integration_test.go diff --git a/.github/renovate-tracked-deps.json b/.github/renovate-tracked-deps.json index de42130e..8e140593 100644 --- a/.github/renovate-tracked-deps.json +++ b/.github/renovate-tracked-deps.json @@ -16,11 +16,6 @@ "mise" ] }, - ".github/workflows/integration_test.yml": { - "regex": [ - "mise" - ] - }, ".github/workflows/lint.yml": { "regex": [ "mise" diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index c6d51c94..7ffb76e6 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -1,12 +1,12 @@ --- -name: End-to-end tests +name: E2E tests on: [pull_request] permissions: {} jobs: - test: + test-core-and-compose: runs-on: ubuntu-24.04 steps: - name: Check out @@ -17,5 +17,23 @@ jobs: with: version: v2026.6.14 sha256: 96ae1ef7b00a6ebbbec23ba1016d6e722f5e904966272f621d15326429e90d53 - - name: Run end-to-end test + - name: Run core e2e tests + env: + OATS_E2E_FILTER: "assert/,custom/,seed/,fixture/compose,fixture/remote" + run: mise run e2e-test + + test-k3d: + runs-on: ubuntu-24.04 + steps: + - name: Check out + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + with: + version: v2026.6.14 + sha256: 96ae1ef7b00a6ebbbec23ba1016d6e722f5e904966272f621d15326429e90d53 + - name: Run k3d e2e tests + env: + OATS_E2E_FILTER: "fixture/k3d" run: mise run e2e-test diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml deleted file mode 100644 index fdd1dc97..00000000 --- a/.github/workflows/integration_test.yml +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: Integration tests - -on: [pull_request] - -permissions: {} - -jobs: - test: - runs-on: ubuntu-24.04 - steps: - - name: Check out - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 - with: - version: v2026.6.14 - sha256: 96ae1ef7b00a6ebbbec23ba1016d6e722f5e904966272f621d15326429e90d53 - - name: Run integration test - run: mise run integration-test diff --git a/AGENTS.md b/AGENTS.md index b3e0372c..495d6b28 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,10 +22,7 @@ mise run build # Run unit tests mise run test -# Run integration tests (requires Docker) -mise run integration-test - -# Run end-to-end tests +# Run e2e tests mise run e2e-test ``` @@ -51,7 +48,8 @@ EditorConfig rules live in `.editorconfig`. ### Package Organization -- **`main.go`** — CLI entry point. Parses flags, discovers YAML test files, runs them sequentially +- **`main.go`** — Root `oats` CLI entry point +- **`internal/cli/`** — The gcx-driven CLI implementation used by the root binary - **`model/`** — Core data models (`TestCaseDefinition`, expected signals) - **`yaml/`** — Test case parsing, execution, signal-specific assertions (`runner.go`, `traces.go`, `metrics.go`, `logs.go`, `profiles.go`) @@ -61,31 +59,22 @@ EditorConfig rules live in `.editorconfig`. ### Test Case Schema -Required fields: - -- `oats-schema-version: 2` (must be present in all test files) -- `oats-template: true` (for template files used in `include`) - -Core sections: `include`, `docker-compose`, `kubernetes`, `matrix`, `input`, `interval`, `expected` - -File discovery scans for `.yaml`/`.yml` files containing -`oats-schema-version`. Files with `oats-template: true` are skipped as entry -points. `.oatsignore` causes a directory to be ignored. +Current user-facing syntax is documented in `CURRENT.md`. Legacy yaml parsing +still exists in-package for migration support, but the repo's CLI surface is +the current `oats.toml` + case-yaml flow. ## CLI Usage ```bash -# Run specific test file -oats /path/to/test.yaml - -# Scan directory for all tests -oats /path/to/tests/ +# Print a plan +oats --config oats.toml --list # With flags -oats -timeout 1m -lgtm-version latest /path/to/test.yaml +oats --config oats.toml --timeout 1m ``` -Key flags: `-timeout` (default 30s), `-lgtm-version` (default "latest"), `-manual-debug` (keep containers running) +Key flags: `--config`, `--suite`, `--tags`, `--timeout`, `--interval`, +`--absent-timeout`, `--gcx`, `--gcx-context` ## Code Conventions @@ -98,11 +87,10 @@ Key flags: `-timeout` (default 30s), `-lgtm-version` (default "latest"), `-manua ## Testing - Unit tests: `mise run test` -- Integration tests require `INTEGRATION_TESTS=true` env var - Uses gomega for assertions, stretchr/testify for test utilities ## CI - Lint on PRs (`mise run lint`), build on PRs (`mise run build`), tests on PRs (`mise run test`) -- Integration tests and e2e tests in separate workflows +- E2E tests in a separate workflow - Linting via flint diff --git a/CURRENT.md b/CURRENT.md index 6a84b376..0add1d7d 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -2,7 +2,7 @@ This document covers the current gcx-driven OATS CLI, which replaces the bespoke TraceQL / PromQL / LogQL HTTP query infrastructure with the [gcx](https://github.com/grafana/gcx) -CLI. The current CLI entry point lives at `cmd/v2/main.go` while this replaces the legacy path. See +CLI. The current `oats` binary runs the gcx-driven implementation. See the internal design doc (grafana/internal-docs#14) for the full design. @@ -71,17 +71,17 @@ discovery → seed → engine → assert → report |------------|---------------| | `discovery` | Parse `oats.toml`, expand case globs, apply filters, and derive | | | case-local fixtures when a suite omits one. | -| `v2case` | Parse and validate one case yaml file. | +| `casefile` | Parse and validate one current-format case yaml file. | | `seed` | Push inline-OTLP payloads at an OTLP/HTTP endpoint. | | `engine` | Execute a gcx command, capture stdout/stderr/exit. | -| `signalcmd` | Translate a `v2case` assertion into gcx args. | +| `signalcmd` | Translate a `casefile` assertion into gcx args. | | `assert` | The expectation vocabulary: `contains`, `not_contains`, `regex`, `value`, `count`, `absent`. | | `wait` | `Until` / `While` polling primitives (replaces gomega.Eventually). | | `report` | Compact-text and NDJSON renderers driven by an Event stream. | | `cache` | Skip-when-unchanged store keyed by | | | `(case yaml + fixture + gcx version + oats version)`. | | `runner` | Orchestrates a suite: seed → poll-and-assert → report, with optional cache. | -| `cmd/v2` | The new binary entry point. | +| `internal/cli` | The gcx-driven CLI implementation package used by the root `oats` binary. | ## Consumer-shape notes diff --git a/README.md b/README.md index b05ebe37..55ac19cf 100644 --- a/README.md +++ b/README.md @@ -1,457 +1,41 @@ # OpenTelemetry Acceptance Tests (OATs) -> **Note:** this README primarily documents the legacy `oats` flow and yaml -> shape. The newer gcx-driven work lives behind `cmd/v2`; see -> [CURRENT.md](CURRENT.md) for the current syntax, fixture model, and migration path. +OATs is a declarative acceptance-test framework for OpenTelemetry. -## Current CLI quick pointer +The current `oats` binary is the gcx-driven CLI. The legacy root runner has +been removed; existing users must migrate to the current `oats.toml` + +case-yaml flow when upgrading. -If you want the newer gcx-driven flow today: +## Install + +```sh +go install github.com/grafana/oats@latest +``` + +## Quick start ```sh ./scripts/build-local-tools.sh bin/oats --config examples/smoke/oats.toml --list +bin/oats --config examples/smoke/oats.toml ``` -Useful entry points: +## Key docs -- syntax and feature status: [CURRENT.md](CURRENT.md) +- current syntax and feature status: [CURRENT.md](CURRENT.md) +- migration guidance: [UPGRADING.md](UPGRADING.md) - small runnable examples: [`examples/smoke/`](examples/smoke/) - richer fixture examples: [`examples/fixtures/`](examples/fixtures/) -- best-effort legacy migration: `bin/oats --migrate path/to/oats.yaml` -Current scope already includes: +## Current scope -- built-in local-LGTM fixture/bootstrap owned by OATS - (consumer repos do not need a shared `docker-compose.lgtm.yml` or `gcx` - wrapper) - traces / logs / metrics / profiles via `gcx` -- structural `match` assertions -- inline-OTLP and app-backed cases -- custom checks +- structural collector-style `match_spans` / `match` +- app-backed and inline-OTLP seed modes - remote / compose / k3d fixtures -- best-effort migration from legacy OATS yaml - -OpenTelemetry Acceptance Tests (OATs), or OATs for short, is a test framework for OpenTelemetry. - -- Declarative tests written in YAML -- Supported signals: traces, logs, metrics -- Full round-trip testing: from the application to the observability stack - - Data is stored in the LGTM stack ([Loki], [Grafana], [Tempo], [Prometheus], [OpenTelemetry Collector]) - - Data is queried using LogQL, PromQL, and TraceQL - - All data is sent to the observability stack via OTLP - so OATs can also be used with other observability stacks -- End-to-end testing - - Docker Compose with the [docker-otel-lgtm] image - - Kubernetes with the [docker-otel-lgtm] and [k3d] - -## Installation - -1. Install the `oats` binary: - -```sh -go install github.com/grafana/oats@latest -``` - -1. You can confirm it was installed with: - -```sh -❯ ls $GOPATH/bin -oats -``` - -## Getting Started - -> [!TIP] -> You can use the test cases in [prom_client_java](https://github.com/prometheus/client_java/tree/main/examples/example-exporter-opentelemetry/oats-tests) as a reference. -> The [GitHub action](https://github.com/prometheus/client_java/blob/main/.github/workflows/acceptance-tests.yml) -> uses `mise run acceptance-test` to run the tests. - -1. Create a folder `oats-tests` for the following files -1. Create `Dockerfile` to build the application you want to test - - ```Dockerfile - FROM eclipse-temurin:21-jre - COPY target/example-exporter-opentelemetry.jar ./app.jar - ENTRYPOINT [ "java", "-jar", "./app.jar" ] - ``` - -1. Create `docker-compose.yaml` to start the application and any dependencies - - ```yaml - services: - java: - build: - dockerfile: Dockerfile - environment: - OTEL_SERVICE_NAME: "rolldice" - OTEL_EXPORTER_OTLP_ENDPOINT: http://lgtm:4318 - OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf - ``` - -1. Create `oats.yaml` with the test cases - - ```yaml - # OATs is an acceptance testing framework for OpenTelemetry - https://github.com/grafana/oats - oats-schema-version: 2 - docker-compose: - files: - - ./docker-compose.yaml - expected: - metrics: - - promql: "uptime_seconds_total{}" - value: ">= 0" - ``` - -1. Run the tests: - -```sh -oats /path/to/oats-tests/oats.yaml -``` - -## Running OATs Directly - -OATs can be run directly using the command-line interface: - -```sh -# Run specific test files -oats /path/to/oats-tests/oats.yaml - -# Run multiple specific test files -oats /path/to/repo/test1.yaml /path/to/repo/test2.yaml - -# Run all tests in a directory (scans for .yaml/.yml files with oats-schema-version) -oats /path/to/oats-tests - -# With flags -oats --timeout=1m --lgtm-version=latest --manual-debug=false /path/to/oats-tests/oats.yaml -``` - -## Running multiple tests - -It can run multiple tests: - -```sh -# Scan directory for all test files -oats /path/to/repo - -# Or specify individual test files (better performance) -oats /path/to/repo/test1.yaml /path/to/repo/test2.yaml -``` - -When scanning a directory, OATs will search for all `.yaml` and `.yml` files -that contain the `oats-schema-version` tag. Files marked with -`oats-template: true` will be skipped as entry points but can still be included -by other test files. - -## Flags - -The following flags are available: - -- `-timeout`: Set the timeout for test cases (default: 30s) -- `-absent-timeout`: Set the timeout for tests that assert absence (default: 10s) -- `-lgtm-version`: Specify the version of [docker-otel-lgtm] to use (default: `"latest"`) -- `-manual-debug`: Enable debug mode to keep containers running (default: `false`) -- `-lgtm-log-all`: Enable logging for all containers (default: `false`) -- `-lgtm-log-grafana`: Enable logging for Grafana (default: `false`) -- `-lgtm-log-loki`: Enable logging for Loki (default: `false`) -- `-lgtm-log-tempo`: Enable logging for Tempo (default: `false`) -- `-lgtm-log-prometheus`: Enable logging for Prometheus (default: `false`) -- `-lgtm-log-pyroscope`: Enable logging for Pyroscope (default: `false`) -- `-lgtm-log-collector`: Enable logging for OpenTelemetry Collector (default: `false`) -- `-host`: Override the host used to issue requests to applications and LGTM (default: `localhost`) -- `-log-limit`: Maximum log output length per log entry - -## Run OATs in GitHub Actions - -The [docker-otel-lgtm](https://github.com/grafana/docker-otel-lgtm) repo uses -[mise](https://mise.jdx.dev/) to install and run OATs from GitHub Actions. You -can also [install OATs directly](#installation). - -## Test Case Syntax - -> [!TIP] -> All test files must include `oats-schema-version: 2` at the top level. -> Template files (used in `include` sections) must also include -> `oats-template: true` to prevent them from being run as entry points. -> You can use any file name with `.yaml` or `.yml` extension. - -The syntax is a bit similar to [Tracetest](https://github.com/kubeshop/tracetest). - -Here is an example: - -```yaml -oats-schema-version: 2 -include: - - ../oats-template.yaml -docker-compose: - files: - - ../docker-compose.yaml -input: - - path: /stock - status: 200 # expected status code, 200 is the default -interval: 500ms # interval between requests to the input URL -expected: - traces: - - traceql: '{ name =~ "SELECT .*product"}' - regexp: "SELECT .*" - attributes: - db.system: h2 - logs: - - logql: '{exporter = "OTLP"}' - equals: "hello LGTM" - metrics: - - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' - value: "== 10" -``` - -### Template Files - -Template files are used to share common configuration across multiple test -files. They must include both `oats-schema-version` and `oats-template: true`: - -```yaml -# oats-template.yaml -oats-schema-version: 2 -oats-template: true - -docker-compose: - files: - - ./docker-compose.yaml -``` - -Here is another example with a more specific input: - -```yaml -oats-schema-version: 2 -include: - - ../oats-template.yaml -docker-compose: - files: - - ../docker-compose.yaml -input: - - path: /users - method: POST - scheme: https - host: 127.0.0.1 - status: 201 - headers: - Authorization: Bearer my-access-token - Content-Type: application/json - body: |- - { - "name": "Grot" - } -interval: 500ms -expected: - traces: - - traceql: '{ name =~ "SELECT .*product"}' - regexp: "SELECT .*" - attributes: - db.system: h2 -``` - -### Query traces - -Each entry in the `traces` array is a test case for traces. - -```yaml -expected: - traces: - - traceql: '{ name =~ "SELECT .*product"}' - regexp: "SELECT .*" - attributes: - db.system: h2 - count: - min: 1 # allow multiple spans with the same attributes - - traceql: '{ span.kind = "client" }' - equals: "HTTP GET" - - traceql: '{ name =~ "dropped-span" }' - count: - max: 0 # assert this span does NOT exist (e.g., filtered/dropped spans) -``` - -#### Trace assertion options - -- **`traceql`**: TraceQL query to find the trace (required) -- **`equals`**: Exact string match for the span name (any span in the trace) -- **`regexp`**: Regular expression pattern to match against the span name (any span in the trace) -- **`attributes`**: Key-value pairs that must match exactly on the span (the span name matched by `equals` or `regexp`) -- **`attribute-regexp`**: Key-value pairs where values are regex patterns to - match against span attributes (the span name matched by `equals` or - `regexp`) -- **`no-extra-attributes`**: Set to `true` to fail if the span has attributes - beyond those specified in `attributes` and `attribute-regexp` -- **`count`**: Control expected number of matching spans, ignoring if they match other criteria - - **`min`**: Minimum number of spans expected (default: `1` if not specified) - - **`max`**: Maximum number of spans expected (`0` means no upper limit, or exactly `0` when `min` is also `0`) - - Examples: - - Not specified: at least 1 span expected - - `{ min: 2, max: 5 }`: between 2 and 5 spans (inclusive) - - `{ min: 3 }`: 3 or more spans - - `{ max: 0 }`: exactly 0 spans (assert absence) -- **`matrix-condition`**: Regex to match against matrix test case names (only run this assertion for matching matrix cases) - -### Query logs - -Each entry in the `logs` array is a test case for logs. - -```yaml -expected: - logs: - - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' - equals: "Anonymous player is rolling the dice" - attributes: - service_name: rolldice - attribute-regexp: - container_id: ".*" - no-extra-attributes: true # fail if there are extra attributes - - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' - regexp: "Anonymous player is .*" -``` - -#### Log assertion options - -- **`logql`**: LogQL query to find the log line (required) -- **`equals`**: Exact string match for the log line -- **`regexp`**: Regular expression pattern to match against the log line -- **`attributes`**: Key-value pairs that must match exactly on the log labels -- **`attribute-regexp`**: Key-value pairs where values are regex patterns to match against log labels -- **`no-extra-attributes`**: Set to `true` to fail if the log has labels beyond those specified in `attributes` and `attribute-regexp` -- **`count`**: Expected count range for returned log lines, ignoring if they match other criteria - - **`min`**: Minimum expected count (defaults to `0` if not specified) - - **`max`**: Maximum expected count. Set to `0` for no upper limit. To assert absence, set both `min: 0` and `max: 0` -- **`matrix-condition`**: Regex to match against matrix test case names - -Example: - -```yaml -expected: - logs: - - logql: '{service_name="rolldice"}' - equals: "Rolling dice" - count: - min: 1 - max: 5 # expect between 1-5 matching logs (inclusive) -``` - -### Query metrics - -```yaml -expected: - metrics: - - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' - value: "== 10" -``` - -#### Metric assertion options - -- **`promql`**: PromQL query to retrieve the metric (required) -- **`value`**: Expected value with comparison operator. Supported operators: - `==`, `!=`, `>`, `<`, `>=`, `<=` (e.g., `">= 0"`, `"== 10"`) -- **`matrix-condition`**: Regex to match against matrix test case names - -### Query profiles - -```yaml -expected: - profiles: - - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{service_name="my-service"}' - flamebearers: - equals: "main" -``` - -#### Profile assertion options - -- **`query`**: Pyroscope query to retrieve the profile (required) -- **`flamebearers`**: Assertions on the flamebearer response - - **`equals`**: String that must appear in the flamebearer names - - **`regexp`**: Regular expression pattern to match against the flamebearer names -- **`matrix-condition`**: Regex to match against matrix test case names - -### Custom checks - -Custom checks allow you to run arbitrary scripts for advanced validation scenarios. - -```yaml -expected: - custom-checks: - - script: | - #!/bin/bash - # Your custom validation script here - exit 0 -``` - -#### Custom check options - -- **`script`**: Script to execute (required) -- **`matrix-condition`**: Regex to match against matrix test case names - -### Matrix of test cases - -Matrix tests are useful to test different configurations of the same application, -e.g. with different settings of the otel collector or different flags in the application. - -```yaml -matrix: - - name: default - docker-compose: - files: - - ./docker-compose.oats.yml - - name: self-contained - docker-compose: - files: - - ./docker-compose.self-contained.oats.yml - - name: net8 - docker-compose: - files: - - ./docker-compose.net8.oats.yml -``` - -You can then make test cases depend on the matrix name: - -```yaml -expected: - metrics: - - promql: 'db_client_connections_max{pool_name="HikariPool-1"}' - value: "== 10" - matrix-condition: default -``` - -`matrix-condition` is a regex that is applied to the matrix name. This field is -available for all assertion types (traces, logs, metrics, profiles). - -## Docker Compose - -Describes the docker-compose file(s) to use for the test. -The files typically define the instrumented application you want to test and optionally some dependencies, -e.g. a database server to send requests to. -You don't need (and shouldn't have) to define the observability stack (e.g. Prometheus, Grafana, etc.), -because this is provided by the test framework (and may test different versions of the observability stack, -e.g. OTel Collector and Grafana Alloy). - -This docker-compose file is relative to the `oats.yaml` file. - -## Kubernetes - -A local Kubernetes cluster can be used to test the application in a Kubernetes environment rather than in docker-compose. -This is useful to test the application in a more realistic environment - and when you want to test Kubernetes specific features. - -Describes the Kubernetes manifest(s) to use for the test. - -```yaml -kubernetes: - dir: k8s - app-service: dice - app-docker-file: Dockerfile - app-docker-context: .. - app-docker-tag: dice:1.1-SNAPSHOT - app-docker-port: 8080 -``` +- custom checks +- best-effort migration from legacy OATS yaml via: -[Tempo]: https://github.com/grafana/tempo -[OpenTelemetry Collector]: https://opentelemetry.io/docs/collector/ -[Prometheus]: https://prometheus.io/ -[Grafana]: https://grafana.com/ -[Loki]: https://github.com/grafana/loki -[docker-otel-lgtm]: https://github.com/grafana/docker-otel-lgtm/ -[k3d]: https://k3d.io/ + ```sh + oats --migrate path/to/legacy.yaml + ``` diff --git a/UPGRADING.md b/UPGRADING.md index a8b5eed1..58f99747 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -5,6 +5,24 @@ The changelog is available in the [releases section](https://github.com/grafana/ This file only contains upgrade notes for breaking changes that require you to modify your existing YAML test files. +## Unreleased / next major + +⚠️ Breaking Changes — legacy root runner removed + +The root `oats` binary now runs the gcx-driven current CLI only. + +That means upgrades now require migrating to the current: + +- `oats.toml` suite/discovery file +- current case yaml shape documented in [CURRENT.md](CURRENT.md) + +The legacy “pass one or more old yaml files directly to `oats`” runner has been +removed. For one-off help migrating old cases, use: + +```sh +oats --migrate path/to/legacy.yaml +``` + ## 0.6.0 ⚠️ Breaking Changes - Migration Required: File Version Tag diff --git a/assert/assert.go b/assert/assert.go index d32eabf4..cbfd29d1 100644 --- a/assert/assert.go +++ b/assert/assert.go @@ -16,7 +16,7 @@ import ( "strconv" "strings" - "github.com/grafana/oats/v2case" + "github.com/grafana/oats/casefile" ) // Failure carries enough context to render a compact "FAIL " @@ -133,7 +133,7 @@ func Absent(actual int) []Failure { // MatchRows checks that each collector-style match entry is satisfied by at // least one row. Each entry is independent: one entry may match one row and // the next entry a different row. -func MatchRows(rows []Row, entries []v2case.MatchEntry) []Failure { +func MatchRows(rows []Row, entries []casefile.MatchEntry) []Failure { var fails []Failure for _, entry := range entries { if !anyRowMatches(rows, entry) { @@ -146,7 +146,7 @@ func MatchRows(rows []Row, entries []v2case.MatchEntry) []Failure { return fails } -func anyRowMatches(rows []Row, entry v2case.MatchEntry) bool { +func anyRowMatches(rows []Row, entry casefile.MatchEntry) bool { for _, row := range rows { if rowMatches(row, entry) { return true @@ -155,7 +155,7 @@ func anyRowMatches(rows []Row, entry v2case.MatchEntry) bool { return false } -func rowMatches(row Row, entry v2case.MatchEntry) bool { +func rowMatches(row Row, entry casefile.MatchEntry) bool { matchType := entry.EffectiveMatchType() if entry.Name != nil { if !matchesValue(row.Name, *entry.Name, matchType) { @@ -180,9 +180,9 @@ func rowMatches(row Row, entry v2case.MatchEntry) bool { return true } -func matchesValue(actual, expected string, matchType v2case.MatchType) bool { +func matchesValue(actual, expected string, matchType casefile.MatchType) bool { switch matchType { - case v2case.MatchTypeRegexp: + case casefile.MatchTypeRegexp: re, err := regexp.Compile(expected) if err != nil { return false @@ -193,7 +193,7 @@ func matchesValue(actual, expected string, matchType v2case.MatchType) bool { } } -func describeMatch(entry v2case.MatchEntry) string { +func describeMatch(entry casefile.MatchEntry) string { var parts []string if entry.MatchType != "" { parts = append(parts, fmt.Sprintf("match_type=%s", entry.MatchType)) diff --git a/assert/assert_test.go b/assert/assert_test.go index 84ad1262..d3d83831 100644 --- a/assert/assert_test.go +++ b/assert/assert_test.go @@ -3,7 +3,7 @@ package assert import ( "testing" - "github.com/grafana/oats/v2case" + "github.com/grafana/oats/casefile" ) func TestContains(t *testing.T) { @@ -109,10 +109,10 @@ func TestMatchRows(t *testing.T) { }, } - got := MatchRows(rows, []v2case.MatchEntry{ + got := MatchRows(rows, []casefile.MatchEntry{ { Name: strPtr("seed-operation"), - Attributes: v2case.AttributeMatchers{ + Attributes: casefile.AttributeMatchers{ {Key: "service.name", Value: strPtr("gcx-e2e-seed")}, {Key: "trace_id"}, }, @@ -122,20 +122,20 @@ func TestMatchRows(t *testing.T) { t.Fatalf("expected match to pass, got %v", got) } - got = MatchRows(rows, []v2case.MatchEntry{ + got = MatchRows(rows, []casefile.MatchEntry{ { - MatchType: v2case.MatchTypeRegexp, + MatchType: casefile.MatchTypeRegexp, Name: strPtr("^seed-.*$"), - Attributes: v2case.AttributeMatchers{{Key: "trace_id", Value: strPtr("^abc")}}, + Attributes: casefile.AttributeMatchers{{Key: "trace_id", Value: strPtr("^abc")}}, }, }) if len(got) != 0 { t.Fatalf("expected regexp match to pass, got %v", got) } - got = MatchRows(rows, []v2case.MatchEntry{ + got = MatchRows(rows, []casefile.MatchEntry{ { - Attributes: v2case.AttributeMatchers{{Key: "missing"}}, + Attributes: casefile.AttributeMatchers{{Key: "missing"}}, }, }) if len(got) != 1 || got[0].Rule != "match" { diff --git a/v2case/case.go b/casefile/case.go similarity index 98% rename from v2case/case.go rename to casefile/case.go index 2799c8f8..17834481 100644 --- a/v2case/case.go +++ b/casefile/case.go @@ -1,4 +1,4 @@ -// Package v2case is the in-memory representation of an OATS v2 case yaml. +// Package casefile is the in-memory representation of an OATS case yaml. // // A case yaml describes one observable behaviour to verify: how to seed data // into the stack, and what gcx queries should return. Cases declare what @@ -8,7 +8,7 @@ // The struct shape mirrors the case yaml documented in the OATS v2 // implementation plan (internal-docs #14). Field tags follow the yaml.v3 // convention. -package v2case +package casefile import ( "bytes" @@ -295,11 +295,11 @@ type ProfileAssertion struct { func Load(path string) (*Case, error) { data, err := os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("v2case load %s: %w", path, err) + return nil, fmt.Errorf("casefile load %s: %w", path, err) } c, err := Parse(data) if err != nil { - return nil, fmt.Errorf("v2case parse %s: %w", path, err) + return nil, fmt.Errorf("casefile parse %s: %w", path, err) } c.SourcePath = path return c, nil diff --git a/v2case/case_test.go b/casefile/case_test.go similarity index 99% rename from v2case/case_test.go rename to casefile/case_test.go index d4e35557..0f16af8b 100644 --- a/v2case/case_test.go +++ b/casefile/case_test.go @@ -1,4 +1,4 @@ -package v2case +package casefile import ( "strings" diff --git a/discovery/discovery.go b/discovery/discovery.go index c2a1e5b2..1302f996 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -2,7 +2,7 @@ // concrete run plan. // // In OATS v1, the runner walked the file system for any yaml carrying -// "oats-schema-version" and ran whatever it found. v2 declares the plan up +// "oats-schema-version" and ran whatever it found. the current format declares the plan up // front: oats.toml lists suites, each suite lists cases (path globs) and the // fixture they share. "oats list" prints the plan before "oats run" executes // it. @@ -17,7 +17,7 @@ import ( "strings" "github.com/BurntSushi/toml" - "github.com/grafana/oats/v2case" + "github.com/grafana/oats/casefile" ) // RootConfig is the parsed oats.toml file. Field names mirror the TOML keys @@ -160,7 +160,7 @@ type Plan struct { Suite SuiteConfig Fixture FixtureConfig FixtureSourceDir string - Cases []*v2case.Case + Cases []*casefile.Case } func (c *RootConfig) effectiveSuites() []SuiteConfig { @@ -231,9 +231,9 @@ func (c *RootConfig) PlanRun(f Filter) ([]Plan, error) { return plans, nil } -func (c *RootConfig) loadSuiteCases(suite SuiteConfig) ([]*v2case.Case, error) { +func (c *RootConfig) loadSuiteCases(suite SuiteConfig) ([]*casefile.Case, error) { seen := make(map[string]struct{}) // dedupe overlapping globs - var cases []*v2case.Case + var cases []*casefile.Case for _, pattern := range suite.Cases { abs := filepath.Join(c.SourceDir, pattern) matches, err := filepath.Glob(abs) @@ -248,7 +248,7 @@ func (c *RootConfig) loadSuiteCases(suite SuiteConfig) ([]*v2case.Case, error) { continue } seen[m] = struct{}{} - tc, loadErr := v2case.Load(m) + tc, loadErr := casefile.Load(m) if loadErr != nil { return nil, loadErr } @@ -275,7 +275,7 @@ func suiteLabel(s SuiteConfig) string { return fmt.Sprintf("suite[%d cases]", len(s.Cases)) } -func materializeSuite(s SuiteConfig, cases []*v2case.Case) SuiteConfig { +func materializeSuite(s SuiteConfig, cases []*casefile.Case) SuiteConfig { if s.Name == "" { s.Name = deriveSuiteName(s, cases) } @@ -295,7 +295,7 @@ func materializeSuite(s SuiteConfig, cases []*v2case.Case) SuiteConfig { return s } -func deriveSuiteName(s SuiteConfig, cases []*v2case.Case) string { +func deriveSuiteName(s SuiteConfig, cases []*casefile.Case) string { if len(cases) == 1 { if strings.TrimSpace(cases[0].Name) != "" { return cases[0].Name @@ -328,7 +328,7 @@ func dirLabel(dir string) string { return base } -func (c *RootConfig) resolveSuiteFixture(suite SuiteConfig, cases []*v2case.Case) (FixtureConfig, string, error) { +func (c *RootConfig) resolveSuiteFixture(suite SuiteConfig, cases []*casefile.Case) (FixtureConfig, string, error) { if suite.Fixture != "" { return c.Fixture[suite.Fixture], c.SourceDir, nil } @@ -357,7 +357,7 @@ func (c *RootConfig) resolveSuiteFixture(suite SuiteConfig, cases []*v2case.Case return fixture, sourceDir, nil } -func fixtureConfigFromCase(f v2case.FixtureConfig) FixtureConfig { +func fixtureConfigFromCase(f casefile.FixtureConfig) FixtureConfig { return FixtureConfig{ Type: f.Type, Template: f.Template, diff --git a/cmd/v2/integration_test.go b/internal/cli/integration_test.go similarity index 94% rename from cmd/v2/integration_test.go rename to internal/cli/integration_test.go index d206b497..23067b0c 100644 --- a/cmd/v2/integration_test.go +++ b/internal/cli/integration_test.go @@ -1,4 +1,4 @@ -package main +package cli import ( "bytes" @@ -200,6 +200,50 @@ func TestStartFixture_K3DLifecycle(t *testing.T) { } } +func TestStartFixture_K3DStartFailure(t *testing.T) { + oldFactory := newKubernetesEndpoint + defer func() { newKubernetesEndpoint = oldFactory }() + + newKubernetesEndpoint = func(plan discovery.Plan) *remote.Endpoint { + return remote.NewEndpoint("localhost", remote.PortsConfig{}, func(ctx context.Context) error { + return fmt.Errorf("cluster boom") + }, func(ctx context.Context) error { + return nil + }, nil) + } + + _, err := startFixture(context.Background(), discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, + Fixture: discovery.FixtureConfig{ + Type: "k3d", + K8sDir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerContext: ".", + AppDockerTag: "dice:test", + AppPort: 18080, + }, + FixtureSourceDir: "/tmp/work", + }) + if err == nil || !strings.Contains(err.Error(), "cluster boom") { + t.Fatalf("expected k3d startup error, got %v", err) + } +} + +func TestResolveComposeFiles_UnsupportedTemplate(t *testing.T) { + _, _, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "weird"}) + if err == nil || !strings.Contains(err.Error(), `unsupported compose fixture template "weird"`) { + t.Fatalf("expected unsupported template error, got %v", err) + } +} + +func TestResolveComposeFiles_MissingConfig(t *testing.T) { + _, _, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose"}) + if err == nil || !strings.Contains(err.Error(), "compose fixture requires compose_file, compose_files, or supported template") { + t.Fatalf("expected missing compose config error, got %v", err) + } +} + func TestCloseFixture_EmitsTeardownEvent(t *testing.T) { rep := &recordingReporter{} fix := &fakeSuiteFixture{} diff --git a/cmd/v2/main.go b/internal/cli/main.go similarity index 98% rename from cmd/v2/main.go rename to internal/cli/main.go index 112eed05..f86c9aa8 100644 --- a/cmd/v2/main.go +++ b/internal/cli/main.go @@ -1,8 +1,7 @@ // Command oats is the OATS binary entry point. // -// This command currently lives under cmd/v2 while the repository still keeps -// the legacy root entrypoint around for existing tests. The gcx-driven CLI is -// intended to replace that legacy path. +// This package contains the gcx-driven CLI implementation used by the root +// oats binary. // // Usage: // @@ -17,7 +16,7 @@ // -v / -vv / -vvv Progressive verbosity (passes / commands / lifecycle) // --suite Comma-separated suite names to include // --tags Comma-separated tag any-match filter -package main +package cli import ( "context" @@ -72,11 +71,15 @@ var ( ) func main() { - code := run() + code := Run() os.Exit(code) } -func run() int { +func Main() { + os.Exit(Run()) +} + +func Run() int { configPath := flag.String("config", "oats.toml", "path to oats.toml") gcxBin := flag.String("gcx", "gcx", "path to gcx binary (PATH-resolved if a bare name)") listOnly := flag.Bool("list", false, "print the run plan and exit (no execution)") diff --git a/cmd/v2/testdata/fake-gcx.sh b/internal/cli/testdata/fake-gcx.sh similarity index 64% rename from cmd/v2/testdata/fake-gcx.sh rename to internal/cli/testdata/fake-gcx.sh index 7ea00ca9..7b39ad15 100755 --- a/cmd/v2/testdata/fake-gcx.sh +++ b/internal/cli/testdata/fake-gcx.sh @@ -7,11 +7,18 @@ set -euo pipefail -# Drop a leading "--context X" pair so the rest of the args mirror what -# a case yaml would produce via signalcmd. -if [[ "${1:-}" == "--context" ]]; then - shift 2 -fi +# Drop leading global flags so the rest of the args mirror what a case yaml +# would produce via signalcmd. +while [[ $# -gt 0 ]]; do + case "${1:-}" in + --context|--config) + shift 2 + ;; + *) + break + ;; + esac +done json=false for arg in "$@"; do @@ -20,8 +27,22 @@ for arg in "$@"; do fi done +query="${*: -1}" +is_missing=false +if [[ "$query" == *"missing"* ]]; then + is_missing=true +fi + case "${1:-}.${2:-}" in traces.search) + if [[ "$is_missing" == true ]]; then + if [[ "$json" == true ]]; then + cat <<'EOF' +{"status":"success","data":{"result":[]}} +EOF + fi + exit 0 + fi if [[ "$json" == true ]]; then cat <<'EOF' {"status":"success","data":{"result":[{"name":"seed-operation","attributes":{"service.name":"gcx-e2e-seed","trace_id":"abc123def456"}}]}} @@ -34,6 +55,14 @@ EOF fi ;; logs.query) + if [[ "$is_missing" == true ]]; then + if [[ "$json" == true ]]; then + cat <<'EOF' +{"status":"success","data":{"resultType":"streams","result":[]}} +EOF + fi + exit 0 + fi if [[ "$json" == true ]]; then cat <<'EOF' {"status":"success","data":{"resultType":"streams","result":[{"stream":{"service_name":"gcx-e2e-seed","trace_id":"abc123def456"},"values":[["1700000000000000000","seed-log-line"]]}]}} @@ -46,12 +75,24 @@ EOF fi ;; metrics.query) + if [[ "$is_missing" == true ]]; then + cat <<'EOF' +{"status":"success","data":{"resultType":"vector","result":[]}} +EOF + exit 0 + fi # Static JSON shaped like Prometheus instant query output. cat <<'EOF' {"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"seed_counter_total","service_name":"gcx-e2e-seed"},"value":[1700000000,"42"]}]}} EOF ;; profiles.query) + if [[ "$is_missing" == true ]]; then + cat <<'EOF' +{"flamebearer":{"names":[]}} +EOF + exit 0 + fi cat <<'EOF' {"flamebearer":{"names":["main","worker"]}} EOF diff --git a/main.go b/main.go index a6b70f98..fcaa5631 100644 --- a/main.go +++ b/main.go @@ -1,94 +1,7 @@ package main -import ( - "errors" - "flag" - "fmt" - "log/slog" - "strings" - "time" - - "github.com/grafana/oats/model" - "github.com/grafana/oats/yaml" - "github.com/onsi/gomega" -) +import cli "github.com/grafana/oats/internal/cli" func main() { - err := run() - if err != nil { - panic(err) - } -} - -func run() error { - settings, err := parseSettings() - if err != nil { - return err - } - - gomega.RegisterFailHandler(func(message string, callerSkip ...int) { - panic(message) - }) - - if flag.NArg() < 1 { - return errors.New("you must pass at least one path to a test case yaml file or directory") - } - - inputs := flag.Args() - cases, err := yaml.ReadTestCases(inputs, true) - if err != nil { - return fmt.Errorf("failed to read test cases: %w", err) - } - - if len(cases) == 0 { - return fmt.Errorf("no cases found in %s - see migration notes at "+ - "https://github.com/grafana/oats/blob/main/UPGRADING.md", strings.Join(inputs, ", ")) - } - for _, testCase := range cases { - slog.Info("test case found", "test", testCase.Name) - } - for _, c := range cases { - yaml.RunTestCase(&c, settings) - } - - slog.Info("all test cases passed") - return nil -} - -func parseSettings() (model.Settings, error) { - host := flag.String("host", "localhost", "host to run the test cases against") - lgtmVersion := flag.String("lgtm-version", "latest", "version of https://github.com/grafana/docker-otel-lgtm") - - logAll := flag.Bool("lgtm-log-all", false, "enable logging for all LGTM components") - logGrafana := flag.Bool("lgtm-log-grafana", false, "enable logging for Grafana") - logPrometheus := flag.Bool("lgtm-log-prometheus", false, "enable logging for Prometheus") - logLoki := flag.Bool("lgtm-log-loki", false, "enable logging for Loki") - logTempo := flag.Bool("lgtm-log-tempo", false, "enable logging for Tempo") - logPyroscope := flag.Bool("lgtm-log-pyroscope", false, "enable logging for Pyroscope") - logCollector := flag.Bool("lgtm-log-collector", false, "enable logging for OTel Collector") - - timeout := flag.Duration("timeout", 30*time.Second, "timeout for each test case") - absentTimeout := flag.Duration("absent-timeout", 10*time.Second, "timeout for tests that assert absence") - manualDebug := flag.Bool("manual-debug", false, "debug mode") - logLimit := flag.Int("log-limit", 1000, "maximum log output length per log entry") - flag.Parse() - - logSettings := make(map[string]bool) - logSettings["ENABLE_LOGS_ALL"] = *logAll - logSettings["ENABLE_LOGS_GRAFANA"] = *logGrafana - logSettings["ENABLE_LOGS_PROMETHEUS"] = *logPrometheus - logSettings["ENABLE_LOGS_LOKI"] = *logLoki - logSettings["ENABLE_LOGS_TEMPO"] = *logTempo - logSettings["ENABLE_LOGS_PYROSCOPE"] = *logPyroscope - logSettings["ENABLE_LOGS_OTELCOL"] = *logCollector - - return model.Settings{ - Host: *host, - Timeout: *timeout, - AbsentTimeout: *absentTimeout, - LgtmVersion: *lgtmVersion, - LgtmLogSettings: logSettings, - ManualDebug: *manualDebug, - LogLimit: *logLimit, - }, nil + cli.Main() } diff --git a/migrate/migrate.go b/migrate/migrate.go index 50be02d0..6f0b1268 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -6,14 +6,14 @@ import ( "regexp" "strings" + "github.com/grafana/oats/casefile" "github.com/grafana/oats/model" - "github.com/grafana/oats/v2case" legacyyaml "github.com/grafana/oats/yaml" goyaml "go.yaml.in/yaml/v3" ) -// ConvertFile reads one legacy OATS yaml file and returns a best-effort v2 -// case yaml plus any warnings about dropped or lossy fields. +// ConvertFile reads one legacy OATS yaml file and returns a best-effort +// current-format case yaml plus any warnings about dropped or lossy fields. func ConvertFile(path string) ([]byte, []string, error) { def, err := legacyyaml.LoadTestCaseDefinition(path) if err != nil { @@ -34,11 +34,11 @@ func ConvertFile(path string) ([]byte, []string, error) { } // ConvertDefinition maps a legacy v1/v1.5-style OATS definition into the -// current v2 case shape. Unsupported fields are dropped with warnings. -func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, []string, error) { +// current case yaml shape. Unsupported fields are dropped with warnings. +func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Case, []string, error) { var warnings []string - c := &v2case.Case{ - OatsVersion: v2case.SchemaVersion, + c := &casefile.Case{ + OatsVersion: casefile.SchemaVersion, Name: name, Interval: def.Interval, } @@ -90,11 +90,11 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, } else if def.Kubernetes == nil && selectedMatrix == nil { warnings = append(warnings, "no docker-compose fixture found; defaulting seed.type to inline-otlp placeholder") c.Seed.Type = "inline-otlp" - c.Seed.Traces = []v2case.SeedTrace{{Service: "migrated-service", Spans: []v2case.SeedSpan{{Name: "replace-me"}}}} + c.Seed.Traces = []casefile.SeedTrace{{Service: "migrated-service", Spans: []casefile.SeedSpan{{Name: "replace-me"}}}} } for _, in := range def.Input { - c.Input = append(c.Input, v2case.Input{ + c.Input = append(c.Input, casefile.Input{ Scheme: in.Scheme, Host: in.Host, Method: in.Method, @@ -111,7 +111,7 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, } assertion, ws := convertSignal(tr.TraceQL, tr.Signal) warnings = append(warnings, ws...) - c.Expected.Traces = append(c.Expected.Traces, v2case.TraceAssertion{TraceQL: tr.TraceQL, MatchSpans: assertion.Match, AssertionCommon: withoutMatch(assertion)}) + c.Expected.Traces = append(c.Expected.Traces, casefile.TraceAssertion{TraceQL: tr.TraceQL, MatchSpans: assertion.Match, AssertionCommon: withoutMatch(assertion)}) } for _, lg := range def.Expected.Logs { if !keepForMatrix(lg.Signal.MatrixCondition, selectedMatrix) { @@ -119,13 +119,13 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, } assertion, ws := convertSignal(lg.LogQL, lg.Signal) warnings = append(warnings, ws...) - c.Expected.Logs = append(c.Expected.Logs, v2case.LogAssertion{LogQL: lg.LogQL, AssertionCommon: assertion}) + c.Expected.Logs = append(c.Expected.Logs, casefile.LogAssertion{LogQL: lg.LogQL, AssertionCommon: assertion}) } for _, m := range def.Expected.Metrics { if !keepForMatrix(m.MatrixCondition, selectedMatrix) { continue } - c.Expected.Metrics = append(c.Expected.Metrics, v2case.MetricAssertion{PromQL: m.PromQL, Value: m.Value}) + c.Expected.Metrics = append(c.Expected.Metrics, casefile.MetricAssertion{PromQL: m.PromQL, Value: m.Value}) if m.MatrixCondition != "" { warnings = append(warnings, fmt.Sprintf("metric %q matrix-condition dropped", m.PromQL)) } @@ -134,16 +134,16 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, if !keepForMatrix(p.MatrixCondition, selectedMatrix) { continue } - var matches []v2case.MatchEntry + var matches []casefile.MatchEntry if p.Flamebearers.NameEquals != "" { - matches = append(matches, v2case.MatchEntry{Name: strPtr(p.Flamebearers.NameEquals)}) + matches = append(matches, casefile.MatchEntry{Name: strPtr(p.Flamebearers.NameEquals)}) } if p.Flamebearers.NameRegexp != "" { - matches = append(matches, v2case.MatchEntry{MatchType: v2case.MatchTypeRegexp, Name: strPtr(p.Flamebearers.NameRegexp)}) + matches = append(matches, casefile.MatchEntry{MatchType: casefile.MatchTypeRegexp, Name: strPtr(p.Flamebearers.NameRegexp)}) } - c.Expected.Profiles = append(c.Expected.Profiles, v2case.ProfileAssertion{ + c.Expected.Profiles = append(c.Expected.Profiles, casefile.ProfileAssertion{ Query: p.Query, - AssertionCommon: v2case.AssertionCommon{Match: matches}, + AssertionCommon: casefile.AssertionCommon{Match: matches}, }) if p.MatrixCondition != "" { warnings = append(warnings, fmt.Sprintf("profile %q matrix-condition dropped", p.Query)) @@ -153,28 +153,28 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*v2case.Case, if !keepForMatrix(cc.MatrixCondition, selectedMatrix) { continue } - c.Expected.Custom = append(c.Expected.Custom, v2case.CustomCheck{Script: cc.Script}) + c.Expected.Custom = append(c.Expected.Custom, casefile.CustomCheck{Script: cc.Script}) if cc.MatrixCondition != "" { warnings = append(warnings, "custom-check matrix-condition dropped after filtering selected matrix") } } if err := c.Validate(); err != nil { - return nil, warnings, fmt.Errorf("migrated v2 case failed validation: %w", err) + return nil, warnings, fmt.Errorf("migrated case failed validation: %w", err) } return c, warnings, nil } -func convertSignal(label string, s model.ExpectedSignal) (v2case.AssertionCommon, []string) { +func convertSignal(label string, s model.ExpectedSignal) (casefile.AssertionCommon, []string) { var warnings []string - out := v2case.AssertionCommon{} + out := casefile.AssertionCommon{} if s.Count != nil { switch { case s.Count.Min == 0 && s.Count.Max == 0: out.Absent = true case s.Count.Max > 0 && s.Count.Min > 0 && s.Count.Max != s.Count.Min: out.Count = fmt.Sprintf(">= %d", s.Count.Min) - warnings = append(warnings, fmt.Sprintf("%s count max=%d dropped; v2 scalar count keeps only min bound", label, s.Count.Max)) + warnings = append(warnings, fmt.Sprintf("%s count max=%d dropped; scalar count keeps only min bound", label, s.Count.Max)) case s.Count.Max > 0 && s.Count.Min == s.Count.Max: out.Count = fmt.Sprintf("== %d", s.Count.Min) case s.Count.Min > 0: @@ -184,31 +184,31 @@ func convertSignal(label string, s model.ExpectedSignal) (v2case.AssertionCommon } } if s.NoExtraAttributes { - warnings = append(warnings, fmt.Sprintf("%s no-extra-attributes is not supported in v2 and was dropped", label)) + warnings = append(warnings, fmt.Sprintf("%s no-extra-attributes is not supported in the current schema and was dropped", label)) } if s.MatrixCondition != "" { warnings = append(warnings, fmt.Sprintf("%s matrix-condition dropped", label)) } if s.NameEquals != "" || len(s.Attributes) > 0 { - entry := v2case.MatchEntry{} + entry := casefile.MatchEntry{} if s.NameEquals != "" { entry.Name = strPtr(s.NameEquals) } for k, v := range s.Attributes { - entry.Attributes = append(entry.Attributes, v2case.AttributeMatcher{Key: k, Value: strPtr(v)}) + entry.Attributes = append(entry.Attributes, casefile.AttributeMatcher{Key: k, Value: strPtr(v)}) } out.Match = append(out.Match, entry) } if s.NameRegexp != "" || len(s.AttributeRegexp) > 0 { - entry := v2case.MatchEntry{MatchType: v2case.MatchTypeRegexp} + entry := casefile.MatchEntry{MatchType: casefile.MatchTypeRegexp} if s.NameRegexp != "" { entry.Name = strPtr(s.NameRegexp) } for k, v := range s.AttributeRegexp { if v == ".*" { - entry.Attributes = append(entry.Attributes, v2case.AttributeMatcher{Key: k}) + entry.Attributes = append(entry.Attributes, casefile.AttributeMatcher{Key: k}) } else { - entry.Attributes = append(entry.Attributes, v2case.AttributeMatcher{Key: k, Value: strPtr(v)}) + entry.Attributes = append(entry.Attributes, casefile.AttributeMatcher{Key: k, Value: strPtr(v)}) } } if entry.Name != nil || len(entry.Attributes) > 0 { @@ -221,7 +221,7 @@ func convertSignal(label string, s model.ExpectedSignal) (v2case.AssertionCommon return out, warnings } -func withoutMatch(a v2case.AssertionCommon) v2case.AssertionCommon { +func withoutMatch(a casefile.AssertionCommon) casefile.AssertionCommon { a.Match = nil return a } diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index c7c6c57b..e68c7f44 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -7,9 +7,9 @@ import ( "testing" "time" + "github.com/grafana/oats/casefile" "github.com/grafana/oats/model" "github.com/grafana/oats/testhelpers/kubernetes" - "github.com/grafana/oats/v2case" ) func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { @@ -114,7 +114,7 @@ func TestConvertFile_MatrixSampleIsParseable(t *testing.T) { fatalf(t, "expected matrix warning to contain %q:\n%s", want, joined) } } - c, err := v2case.Parse(out) + c, err := casefile.Parse(out) if err != nil { fatalf(t, "migrated matrix yaml should parse as v2: %v\n%s", err, string(out)) } @@ -151,7 +151,7 @@ expected: if len(warnings) != 0 { fatalf(t, "expected no warnings for straightforward custom checks, got %v", warnings) } - c, err := v2case.Parse(out) + c, err := casefile.Parse(out) if err != nil { fatalf(t, "migrated custom-check yaml should parse as v2: %v\n%s", err, string(out)) } diff --git a/mise.toml b/mise.toml index d4958073..9a52ab01 100644 --- a/mise.toml +++ b/mise.toml @@ -34,14 +34,13 @@ run = "flint run --fix" description = "Run tests" run = "go test $(go list ./...)" -[tasks.integration-test] -description = "Run integration tests" -env.INTEGRATION_TESTS = "true" -run = "go test github.com/grafana/oats/tests" +[tasks.e2e-test] +description = "Run acceptance case suite" +run = "go test ./tests/e2e -run TestCases" [tasks.build] description = "Build the project" -run = "go build" +run = "go build -buildvcs=false ." [tasks.check] description = "Run all checks" diff --git a/runner/cache_integration_test.go b/runner/cache_integration_test.go index 77a39e72..fd8893c9 100644 --- a/runner/cache_integration_test.go +++ b/runner/cache_integration_test.go @@ -10,11 +10,11 @@ import ( "time" "github.com/grafana/oats/cache" + "github.com/grafana/oats/casefile" "github.com/grafana/oats/report" - "github.com/grafana/oats/v2case" ) -func cachedRunnerCase(t *testing.T) (*v2case.Case, []byte) { +func cachedRunnerCase(t *testing.T) (*casefile.Case, []byte) { t.Helper() src := []byte(`oats: 2 name: cached @@ -26,12 +26,12 @@ expected: - traceql: '{}' contains: ["svc"] `) - // v2case.Load requires SourcePath, so persist to a temp file. + // casefile.Load requires SourcePath, so persist to a temp file. tmp := filepath.Join(t.TempDir(), "case.yaml") if err := os.WriteFile(tmp, src, 0o644); err != nil { t.Fatal(err) } - c, err := v2case.Load(tmp) + c, err := casefile.Load(tmp) if err != nil { t.Fatal(err) } diff --git a/runner/runner.go b/runner/runner.go index 9df298f5..4eb9ae46 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -1,4 +1,4 @@ -// Package runner orchestrates one v2 case end-to-end: seed → poll-and-assert +// Package runner orchestrates one case end-to-end: seed → poll-and-assert // → report. It is intentionally agnostic about fixtures — the caller hands // in already-resolved endpoints, so this layer ships before the // fixture-lifecycle layer exists. @@ -22,12 +22,12 @@ import ( "github.com/grafana/oats/assert" "github.com/grafana/oats/cache" + "github.com/grafana/oats/casefile" "github.com/grafana/oats/engine" "github.com/grafana/oats/report" "github.com/grafana/oats/seed" "github.com/grafana/oats/signalcmd" "github.com/grafana/oats/testhelpers/requests" - "github.com/grafana/oats/v2case" "github.com/grafana/oats/wait" ) @@ -118,7 +118,7 @@ type Runner struct { } // CacheContext describes the per-run inputs that must contribute to the -// cache key. They are passed in by the caller (typically cmd/v2) rather +// cache key. They are passed in by the caller (typically the CLI package) rather // than discovered by the Runner because they are global to the whole run: // the gcx version doesn't change between cases, and a fresh "gcx --version" // per case is wasted work. @@ -153,7 +153,7 @@ func (r *Runner) WithCache(store *cache.Store, ctx CacheContext) *Runner { return r } -func (r *Runner) cacheKey(c *v2case.Case) cache.Key { +func (r *Runner) cacheKey(c *casefile.Case) cache.Key { yamlBytes, _ := os.ReadFile(c.SourcePath) // best-effort; nil on error return cache.Key{ CaseYAML: yamlBytes, @@ -173,7 +173,7 @@ func (r *Runner) cacheKey(c *v2case.Case) cache.Key { // case.skip and returns true without running the case at all. A miss // runs the case as usual; passes are recorded, failures evict any stale // entry so a regression is never masked. -func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { +func (r *Runner) RunCase(ctx context.Context, c *casefile.Case) bool { caseStart := time.Now() r.reporter.Emit(report.Event{ Type: report.EventCaseStart, @@ -267,7 +267,7 @@ func (r *Runner) RunCase(ctx context.Context, c *v2case.Case) bool { return ok } -func (r *Runner) runComposeLogCheck(ctx context.Context, c *v2case.Case, msg string) bool { +func (r *Runner) runComposeLogCheck(ctx context.Context, c *casefile.Case, msg string) bool { run := func() []assert.Failure { if err := r.driveInputs(c); err != nil { return []assert.Failure{{Rule: "input", Detail: err.Error()}} @@ -298,7 +298,7 @@ func (r *Runner) runComposeLogCheck(ctx context.Context, c *v2case.Case, msg str return false } -func (r *Runner) runCustomCheck(ctx context.Context, c *v2case.Case, chk *v2case.CustomCheck) bool { +func (r *Runner) runCustomCheck(ctx context.Context, c *casefile.Case, chk *casefile.CustomCheck) bool { dir := "." if c.SourcePath != "" { dir = filepath.Dir(c.SourcePath) @@ -432,7 +432,7 @@ func trimOutput(s string) string { return s } -func (r *Runner) seedCase(c *v2case.Case) error { +func (r *Runner) seedCase(c *casefile.Case) error { switch c.Seed.Type { case "app": // External fixture is responsible for booting the app. Runner @@ -447,7 +447,7 @@ func (r *Runner) seedCase(c *v2case.Case) error { return fmt.Errorf("unknown seed type %q", c.Seed.Type) } -func toSeedPayload(s v2case.Seed) seed.Payload { +func toSeedPayload(s casefile.Seed) seed.Payload { p := seed.Payload{} for _, t := range s.Traces { for _, sp := range t.Spans { @@ -485,7 +485,7 @@ func toSeedPayload(s v2case.Seed) seed.Payload { // wait.Until / wait.While accordingly. func (r *Runner) pollAssert( ctx context.Context, - c *v2case.Case, + c *casefile.Case, args []string, absent bool, evalFn func(stdout, stderr string, exit int) []assert.Failure, @@ -532,14 +532,14 @@ func (r *Runner) pollAssert( return false } -func (r *Runner) caseInterval(c *v2case.Case) time.Duration { +func (r *Runner) caseInterval(c *casefile.Case) time.Duration { if c.Interval > 0 { return c.Interval } return r.opts.Interval } -func (r *Runner) runTrace(ctx context.Context, c *v2case.Case, a *v2case.TraceAssertion) bool { +func (r *Runner) runTrace(ctx context.Context, c *casefile.Case, a *casefile.TraceAssertion) bool { if len(a.MatchSpans) > 0 { return r.runTraceStructured(ctx, c, a) } @@ -549,7 +549,7 @@ func (r *Runner) runTrace(ctx context.Context, c *v2case.Case, a *v2case.TraceAs }) } -func (r *Runner) runTraceStructured(ctx context.Context, c *v2case.Case, a *v2case.TraceAssertion) bool { +func (r *Runner) runTraceStructured(ctx context.Context, c *casefile.Case, a *casefile.TraceAssertion) bool { run := func() []assert.Failure { if err := r.driveInputs(c); err != nil { return []assert.Failure{{Rule: "input", Detail: err.Error()}} @@ -589,7 +589,7 @@ func (r *Runner) runTraceStructured(ctx context.Context, c *v2case.Case, a *v2ca return false } -func (r *Runner) fetchTraceRows(ctx context.Context, c *v2case.Case, searchStdout string) ([]assert.Row, int, error) { +func (r *Runner) fetchTraceRows(ctx context.Context, c *casefile.Case, searchStdout string) ([]assert.Row, int, error) { traceIDs, count, err := extractTraceIDs(searchStdout) if err != nil { return nil, 0, err @@ -628,7 +628,7 @@ func (r *Runner) fetchTraceRows(ctx context.Context, c *v2case.Case, searchStdou return rows, count, nil } -func (r *Runner) runLog(ctx context.Context, c *v2case.Case, a *v2case.LogAssertion) bool { +func (r *Runner) runLog(ctx context.Context, c *casefile.Case, a *casefile.LogAssertion) bool { args := signalcmd.Logs(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { if len(a.Match) == 0 { @@ -639,7 +639,7 @@ func (r *Runner) runLog(ctx context.Context, c *v2case.Case, a *v2case.LogAssert }) } -func (r *Runner) runMetric(ctx context.Context, c *v2case.Case, a *v2case.MetricAssertion) bool { +func (r *Runner) runMetric(ctx context.Context, c *casefile.Case, a *casefile.MetricAssertion) bool { args := signalcmd.Metrics(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { if a.Value == "" && len(a.Match) == 0 { @@ -658,7 +658,7 @@ func (r *Runner) runMetric(ctx context.Context, c *v2case.Case, a *v2case.Metric }) } -func (r *Runner) runProfile(ctx context.Context, c *v2case.Case, a *v2case.ProfileAssertion) bool { +func (r *Runner) runProfile(ctx context.Context, c *casefile.Case, a *casefile.ProfileAssertion) bool { args := signalcmd.Profiles(*a, r.opts.Timeout) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { if len(a.Match) == 0 { @@ -671,7 +671,7 @@ func (r *Runner) runProfile(ctx context.Context, c *v2case.Case, a *v2case.Profi // evalCommonText runs the assertions that every signal type shares when gcx // output is plain text rather than JSON. -func evalCommonText(stdout string, c v2case.AssertionCommon) []assert.Failure { +func evalCommonText(stdout string, c casefile.AssertionCommon) []assert.Failure { var fails []assert.Failure fails = append(fails, assert.Contains(stdout, c.Contains)...) fails = append(fails, assert.NotContains(stdout, c.NotContains)...) @@ -685,7 +685,7 @@ func evalCommonText(stdout string, c v2case.AssertionCommon) []assert.Failure { return fails } -func evalTraceStructured(stdout string, a v2case.TraceAssertion, rows []assert.Row, count int, parseErr error) []assert.Failure { +func evalTraceStructured(stdout string, a casefile.TraceAssertion, rows []assert.Row, count int, parseErr error) []assert.Failure { var fails []assert.Failure fails = append(fails, assert.Contains(stdout, a.Contains)...) fails = append(fails, assert.NotContains(stdout, a.NotContains)...) @@ -710,7 +710,7 @@ func evalTraceStructured(stdout string, a v2case.TraceAssertion, rows []assert.R return fails } -func evalCommonStructured(stdout string, c v2case.AssertionCommon, rows []assert.Row, count int, parseErr error) []assert.Failure { +func evalCommonStructured(stdout string, c casefile.AssertionCommon, rows []assert.Row, count int, parseErr error) []assert.Failure { var fails []assert.Failure fails = append(fails, assert.Contains(stdout, c.Contains)...) fails = append(fails, assert.NotContains(stdout, c.NotContains)...) @@ -742,6 +742,12 @@ func approxRowCount(stdout string) int { if t == "" { continue } + if strings.HasPrefix(t, "hint: use --json") { + continue + } + if t == "No data" { + continue + } // Skip lines that look like table-headers / dividers in gcx text mode. if strings.HasPrefix(t, "─") || strings.HasPrefix(t, "═") || strings.HasPrefix(t, "+") || looksLikeGCXHeader(t) { continue @@ -820,7 +826,7 @@ func extractMetricRows(stdout string) ([]assert.Row, int, float64, error) { return rows, len(generic.Data.Result), f, nil } -func (r *Runner) failCase(c *v2case.Case, msg, cmd string) { +func (r *Runner) failCase(c *casefile.Case, msg, cmd string) { r.reporter.Emit(report.Event{ Type: report.EventAssertFail, Case: c.Name, @@ -830,7 +836,7 @@ func (r *Runner) failCase(c *v2case.Case, msg, cmd string) { }) } -func (r *Runner) driveInputs(c *v2case.Case) error { +func (r *Runner) driveInputs(c *casefile.Case) error { for _, in := range c.Input { if err := r.doInput(in); err != nil { return err @@ -839,7 +845,7 @@ func (r *Runner) driveInputs(c *v2case.Case) error { return nil } -func (r *Runner) doInput(in v2case.Input) error { +func (r *Runner) doInput(in casefile.Input) error { if in.Path == "" { return nil } @@ -1011,7 +1017,7 @@ func collectNamedRows(v any) []assert.Row { func maybeRow(m map[string]any) (assert.Row, bool) { row := assert.Row{Attributes: map[string]string{}} - for _, key := range []string{"name", "spanName", "span_name", "body"} { + for _, key := range []string{"name", "spanName", "span_name", "body", "rootTraceName"} { if v, ok := m[key]; ok { row.Name = fmt.Sprint(v) break @@ -1031,6 +1037,12 @@ func maybeRow(m map[string]any) (assert.Row, bool) { } } } + if v, ok := m["rootServiceName"]; ok { + row.Attributes["service.name"] = fmt.Sprint(v) + } + if v, ok := m["traceID"]; ok { + row.Attributes["trace_id"] = fmt.Sprint(v) + } if row.Name == "" && len(row.Attributes) == 0 { return assert.Row{}, false } diff --git a/runner/runner_test.go b/runner/runner_test.go index a709c16e..b405cb2d 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -13,9 +13,9 @@ import ( "testing" "time" + "github.com/grafana/oats/casefile" "github.com/grafana/oats/engine" "github.com/grafana/oats/report" - "github.com/grafana/oats/v2case" ) // stubExec is a deterministic Executor that returns the configured output @@ -49,9 +49,9 @@ func newRunner(t *testing.T, exec engine.Executor, opts Options) (*Runner, *byte return New(exec, rep, Endpoint{GCXContext: "test"}, opts), &buf } -func mustParse(t *testing.T, src string) *v2case.Case { +func mustParse(t *testing.T, src string) *casefile.Case { t.Helper() - c, err := v2case.Parse([]byte(src)) + c, err := casefile.Parse([]byte(src)) if err != nil { t.Fatalf("Parse: %v", err) } @@ -391,6 +391,131 @@ expected: } } +func TestRunCase_ComposeLogsPass(t *testing.T) { + dir := t.TempDir() + binDir := filepath.Join(dir, "bin") + if err := os.MkdirAll(binDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + docker := filepath.Join(binDir, "docker") + if err := os.WriteFile(docker, []byte("#!/bin/sh\necho 'service boot complete'\n"), 0o755); err != nil { + t.Fatalf("WriteFile: %v", err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + c := mustParse(t, ` +oats: 2 +name: compose logs pass +seed: + type: app +expected: + logs: + - logql: '{service_name="gcx-e2e-seed"}' + contains: seed-log-line + compose-logs: + - service boot complete +`) + + exec := &stubExec{stdout: "seed-log-line"} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + r := New(exec, rep, Endpoint{ + GCXContext: "test", + CustomCheckEnv: []string{ + "OATS_FIXTURE_TYPE=compose", + "PATH=" + binDir + string(os.PathListSeparator) + os.Getenv("PATH"), + }, + }, Options{Timeout: 200 * time.Millisecond, SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + if !ok { + t.Fatalf("expected compose-logs case to pass:\n%s", buf.String()) + } +} + +func TestRunCase_ComposeLogsMissingSurfaced(t *testing.T) { + dir := t.TempDir() + binDir := filepath.Join(dir, "bin") + if err := os.MkdirAll(binDir, 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + docker := filepath.Join(binDir, "docker") + if err := os.WriteFile(docker, []byte("#!/bin/sh\necho 'different output'\n"), 0o755); err != nil { + t.Fatalf("WriteFile: %v", err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + c := mustParse(t, ` +oats: 2 +name: compose logs missing +seed: + type: app +expected: + logs: + - logql: '{service_name="gcx-e2e-seed"}' + contains: seed-log-line + compose-logs: + - expected line +`) + + exec := &stubExec{stdout: "seed-log-line"} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + r := New(exec, rep, Endpoint{ + GCXContext: "test", + CustomCheckEnv: []string{ + "OATS_FIXTURE_TYPE=compose", + "PATH=" + binDir + string(os.PathListSeparator) + os.Getenv("PATH"), + }, + }, Options{Timeout: 200 * time.Millisecond, Interval: 20 * time.Millisecond, SeedSettleDelay: 1}) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + if ok { + t.Fatalf("expected compose-logs case to fail") + } + if !strings.Contains(buf.String(), `compose-logs: missing "expected line"`) { + t.Fatalf("expected compose-logs failure output, got:\n%s", buf.String()) + } +} + +func TestRunCase_ComposeLogsRequireComposeFixture(t *testing.T) { + c := mustParse(t, ` +oats: 2 +name: compose logs wrong fixture +seed: + type: app +expected: + logs: + - logql: '{service_name="gcx-e2e-seed"}' + contains: seed-log-line + compose-logs: + - expected line +`) + + exec := &stubExec{stdout: "seed-log-line"} + var buf bytes.Buffer + rep := report.NewTextReporter(&buf, report.VerboseDefault) + r := New(exec, rep, Endpoint{GCXContext: "test", CustomCheckEnv: []string{"OATS_FIXTURE_TYPE=remote"}}, Options{ + Timeout: 200 * time.Millisecond, + Interval: 20 * time.Millisecond, + SeedSettleDelay: 1, + }) + + r.reporter.Emit(report.Event{Type: report.EventRunStart}) + ok := r.RunCase(context.Background(), c) + r.reporter.Emit(report.Event{Type: report.EventRunEnd}) + if ok { + t.Fatalf("expected compose-logs case to fail") + } + if !strings.Contains(buf.String(), "compose-logs requires a compose fixture") { + t.Fatalf("expected compose fixture error, got:\n%s", buf.String()) + } +} + func TestResolveCustomCheckPath(t *testing.T) { dir := "/tmp/cases" cases := []struct { diff --git a/scripts/build-local-tools.sh b/scripts/build-local-tools.sh index 2431caf7..fb4587af 100755 --- a/scripts/build-local-tools.sh +++ b/scripts/build-local-tools.sh @@ -9,7 +9,7 @@ mkdir -p "$bin_dir" # fetch/build OATS itself. : "${GCX_VERSION:=v0.4.0}" -GOWORK=off go -C "$root" build -buildvcs=false -o "$bin_dir/oats" ./cmd/v2 +GOWORK=off go -C "$root" build -buildvcs=false -o "$bin_dir/oats" . GOBIN="$bin_dir" GOWORK=off go install "github.com/grafana/gcx/cmd/gcx@${GCX_VERSION}" printf 'built %s/oats and %s/gcx\n' "$bin_dir" "$bin_dir" diff --git a/signalcmd/signalcmd.go b/signalcmd/signalcmd.go index f68a4a04..990713d9 100644 --- a/signalcmd/signalcmd.go +++ b/signalcmd/signalcmd.go @@ -1,4 +1,4 @@ -// Package signalcmd translates a single v2case assertion into the gcx +// Package signalcmd translates a single case assertion into the gcx // command line that will execute the corresponding query. // // One function per signal type. Each returns []string suitable for handing @@ -14,7 +14,7 @@ import ( "strings" "time" - "github.com/grafana/oats/v2case" + "github.com/grafana/oats/casefile" ) // Defaults applied when a case does not pin its own values. @@ -23,7 +23,7 @@ const ( ) // Traces builds the gcx args for a TraceAssertion. -func Traces(a v2case.TraceAssertion, since time.Duration) []string { +func Traces(a casefile.TraceAssertion, since time.Duration) []string { if since <= 0 { since = DefaultSince } @@ -52,7 +52,7 @@ func TraceGet(traceID string, since time.Duration) []string { } // Logs builds the gcx args for a LogAssertion. -func Logs(a v2case.LogAssertion, since time.Duration) []string { +func Logs(a casefile.LogAssertion, since time.Duration) []string { if since <= 0 { since = DefaultSince } @@ -71,7 +71,7 @@ func Logs(a v2case.LogAssertion, since time.Duration) []string { // declares a numeric `value` comparison we ask gcx for JSON so the runner // can parse the actual value out; otherwise the default agent text format // is enough for substring matching. -func Metrics(a v2case.MetricAssertion, since time.Duration) []string { +func Metrics(a casefile.MetricAssertion, since time.Duration) []string { if since <= 0 { since = DefaultSince } @@ -87,7 +87,7 @@ func Metrics(a v2case.MetricAssertion, since time.Duration) []string { } // Profiles builds the gcx args for a ProfileAssertion. -func Profiles(a v2case.ProfileAssertion, since time.Duration) []string { +func Profiles(a casefile.ProfileAssertion, since time.Duration) []string { if since <= 0 { since = DefaultSince } diff --git a/signalcmd/signalcmd_test.go b/signalcmd/signalcmd_test.go index 9484482c..6f9503a8 100644 --- a/signalcmd/signalcmd_test.go +++ b/signalcmd/signalcmd_test.go @@ -5,11 +5,11 @@ import ( "testing" "time" - "github.com/grafana/oats/v2case" + "github.com/grafana/oats/casefile" ) func TestTraces(t *testing.T) { - got := Traces(v2case.TraceAssertion{TraceQL: `{ span.http.route = "/x" }`}, 0) + got := Traces(casefile.TraceAssertion{TraceQL: `{ span.http.route = "/x" }`}, 0) want := []string{"traces", "search", "--since", "10m0s", `{ span.http.route = "/x" }`} if !equal(got, want) { t.Errorf("got %v\nwant %v", got, want) @@ -17,9 +17,9 @@ func TestTraces(t *testing.T) { } func TestTraces_WithMatchAsksForJSON(t *testing.T) { - got := Traces(v2case.TraceAssertion{ + got := Traces(casefile.TraceAssertion{ TraceQL: `{ span.http.route = "/x" }`, - MatchSpans: []v2case.MatchEntry{{Name: strPtr("GET /x")}}, + MatchSpans: []casefile.MatchEntry{{Name: strPtr("GET /x")}}, }, 0) if !contains(got, "-o", "json") { t.Errorf("expected -o json in: %v", got) @@ -27,7 +27,7 @@ func TestTraces_WithMatchAsksForJSON(t *testing.T) { } func TestLogs(t *testing.T) { - got := Logs(v2case.LogAssertion{LogQL: `{service_name="x"}`}, 5*time.Minute) + got := Logs(casefile.LogAssertion{LogQL: `{service_name="x"}`}, 5*time.Minute) want := []string{"logs", "query", "--since", "5m0s", `{service_name="x"}`} if !equal(got, want) { t.Errorf("got %v\nwant %v", got, want) @@ -35,10 +35,10 @@ func TestLogs(t *testing.T) { } func TestLogs_WithMatchAsksForJSON(t *testing.T) { - got := Logs(v2case.LogAssertion{ + got := Logs(casefile.LogAssertion{ LogQL: `{service_name="x"}`, - AssertionCommon: v2case.AssertionCommon{ - Match: []v2case.MatchEntry{{Name: strPtr("line")}}, + AssertionCommon: casefile.AssertionCommon{ + Match: []casefile.MatchEntry{{Name: strPtr("line")}}, }, }, 5*time.Minute) if !contains(got, "-o", "json") { @@ -47,7 +47,7 @@ func TestLogs_WithMatchAsksForJSON(t *testing.T) { } func TestMetrics_PromQLOnly(t *testing.T) { - got := Metrics(v2case.MetricAssertion{PromQL: "up"}, time.Minute) + got := Metrics(casefile.MetricAssertion{PromQL: "up"}, time.Minute) want := []string{"metrics", "query", "--since", "1m0s", "up"} if !equal(got, want) { t.Errorf("got %v\nwant %v", got, want) @@ -55,7 +55,7 @@ func TestMetrics_PromQLOnly(t *testing.T) { } func TestMetrics_WithValueAsksForJSON(t *testing.T) { - got := Metrics(v2case.MetricAssertion{PromQL: "rate(x[1m])", Value: ">= 0"}, time.Minute) + got := Metrics(casefile.MetricAssertion{PromQL: "rate(x[1m])", Value: ">= 0"}, time.Minute) // JSON output flag must appear before the positional PromQL. if !contains(got, "-o", "json") { t.Errorf("expected -o json in: %v", got) @@ -66,7 +66,7 @@ func TestMetrics_WithValueAsksForJSON(t *testing.T) { } func TestProfiles(t *testing.T) { - got := Profiles(v2case.ProfileAssertion{Query: "process_cpu:cpu:nanoseconds:cpu:nanoseconds{}"}, 0) + got := Profiles(casefile.ProfileAssertion{Query: "process_cpu:cpu:nanoseconds:cpu:nanoseconds{}"}, 0) want := []string{"profiles", "query", "--since", "10m0s", "--profile-type", "process_cpu:cpu:nanoseconds:cpu:nanoseconds", "{}"} if !equal(got, want) { t.Errorf("got %v\nwant %v", got, want) @@ -74,7 +74,7 @@ func TestProfiles(t *testing.T) { } func TestProfiles_QueryWithoutSelectorDefaultsToEmptySelector(t *testing.T) { - got := Profiles(v2case.ProfileAssertion{Query: "process_cpu:cpu:nanoseconds:cpu:nanoseconds"}, 0) + got := Profiles(casefile.ProfileAssertion{Query: "process_cpu:cpu:nanoseconds:cpu:nanoseconds"}, 0) want := []string{"profiles", "query", "--since", "10m0s", "--profile-type", "process_cpu:cpu:nanoseconds:cpu:nanoseconds", "{}"} if !equal(got, want) { t.Errorf("got %v\nwant %v", got, want) @@ -82,10 +82,10 @@ func TestProfiles_QueryWithoutSelectorDefaultsToEmptySelector(t *testing.T) { } func TestProfiles_WithMatchAsksForJSON(t *testing.T) { - got := Profiles(v2case.ProfileAssertion{ + got := Profiles(casefile.ProfileAssertion{ Query: "process_cpu:cpu:nanoseconds:cpu:nanoseconds{}", - AssertionCommon: v2case.AssertionCommon{ - Match: []v2case.MatchEntry{{Name: strPtr("main")}}, + AssertionCommon: casefile.AssertionCommon{ + Match: []casefile.MatchEntry{{Name: strPtr("main")}}, }, }, 0) if !contains(got, "-o", "json") { diff --git a/testhelpers/kubernetes/kubernetes.go b/testhelpers/kubernetes/kubernetes.go index 37cd1913..0184033d 100644 --- a/testhelpers/kubernetes/kubernetes.go +++ b/testhelpers/kubernetes/kubernetes.go @@ -101,7 +101,16 @@ func start(model *Kubernetes, ports remote.PortsConfig, testName string, run fun if err != nil { return err } - err = run(exec.Command("kubectl", "wait", "--timeout=5m", "--for=condition=ready", "pod", "-l", "app=lgtm"), false) + err = run( + exec.Command( + "kubectl", + "wait", + "--timeout=5m", + "--for=condition=available", + "deployment/lgtm", + ), + false, + ) if err != nil { return err } diff --git a/testhelpers/kubernetes/kubernetes_test.go b/testhelpers/kubernetes/kubernetes_test.go index 87e5fc11..596bd6f9 100644 --- a/testhelpers/kubernetes/kubernetes_test.go +++ b/testhelpers/kubernetes/kubernetes_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/grafana/oats/testhelpers/remote" + "github.com/stretchr/testify/require" ) func TestClusterName_TruncatesFromEnd(t *testing.T) { @@ -70,7 +71,7 @@ func TestStart_DefaultDockerContextAndCommandSequence(t *testing.T) { "fg: k3d image import -c smoke-suite dice:test", "fg: k3d image import -c smoke-suite busybox:latest", "fg: kubectl apply -f k8s", - "fg: kubectl wait --timeout=5m --for=condition=ready pod -l app=lgtm", + "fg: kubectl wait --timeout=5m --for=condition=available deployment/lgtm", "bg: kubectl port-forward service/dice 18080:8080", "bg: kubectl port-forward service/lgtm 13100:3100", "bg: kubectl port-forward service/lgtm 19090:9090", @@ -81,3 +82,39 @@ func TestStart_DefaultDockerContextAndCommandSequence(t *testing.T) { t.Fatalf("unexpected command sequence:\n got: %#v\nwant: %#v", calls, want) } } + +func TestStartWaitsForLgtmDeploymentAvailability(t *testing.T) { + t.Parallel() + + model := &Kubernetes{ + Dir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerContext: ".", + AppDockerTag: "dice:1.1-SNAPSHOT", + AppDockerPort: 8080, + } + ports := remote.PortsConfig{ + LokiHttpPort: 3100, + PrometheusHTTPPort: 9090, + TempoHTTPPort: 3200, + PyroscopeHttpPort: 4040, + } + + var commands [][]string + run := func(cmd *exec.Cmd, background bool) error { + commands = append(commands, append([]string(nil), cmd.Args...)) + return nil + } + + err := start(model, ports, "run-oats", run) + require.NoError(t, err) + + require.Contains(t, commands, []string{ + "kubectl", + "wait", + "--timeout=5m", + "--for=condition=available", + "deployment/lgtm", + }) +} diff --git a/tests/e2e/README.md b/tests/e2e/README.md new file mode 100644 index 00000000..b9c04cd3 --- /dev/null +++ b/tests/e2e/README.md @@ -0,0 +1,54 @@ +# End-to-end cases + +This directory uses a Flint-style case layout: + +```text +tests/e2e/cases/// + test.yaml + files/ + ... +``` + +## Defaults + +- `files/oats.yaml` is the default OATS input. +- When `files/oats.toml` is absent, the test runner synthesizes one that points + at `files/oats.yaml`. +- The harness boots a shared local LGTM stack plus real `oats` and real `gcx`. +- The default run command is: + + ```bash + --config .generated-oats.toml --gcx --gcx-context local --no-cache --timeout 10s --interval 1s + ``` + +- Expected exit code defaults to `0`. +- Cases run in parallel by default. Set `execution.mode: serial` to opt out. + +## Fixture coverage note + +The matrix includes explicit remote, compose (`compose-logs`), and k3d cases. +Pure matcher edge cases that proved flaky or not meaningfully end-to-end stay in +unit tests instead of this directory. + +## Placeholder expansion + +The runner expands these placeholders inside `test.yaml`, `files/*`, env values, +and shell commands: + +- `{{REPO_ROOT}}` +- `{{CASE_DIR}}` +- `{{CASE_NAME}}` +- `{{REMOTE_OTLP_HTTP}}` +- `{{GCX}}` +- `{{GCX_CONFIG}}` +- `{{OATS}}` + +## Filtering + +Use `OATS_E2E_FILTER` to run a subset of cases by relative path: + +```bash +OATS_E2E_FILTER=assert/ go test ./tests/e2e -run TestCases +OATS_E2E_FILTER=fixture/remote go test ./tests/e2e -run TestCases +OATS_E2E_FILTER=assert/contains,assert/regex go test ./tests/e2e -run TestCases +``` diff --git a/tests/e2e/cases/assert/absent-fail/files/oats.yaml b/tests/e2e/cases/assert/absent-fail/files/oats.yaml new file mode 100644 index 00000000..c8ccaa19 --- /dev/null +++ b/tests/e2e/cases/assert/absent-fail/files/oats.yaml @@ -0,0 +1,14 @@ +oats: 2 +name: absent-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: present-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + absent: true diff --git a/tests/e2e/cases/assert/absent-fail/test.yaml b/tests/e2e/cases/assert/absent-fail/test.yaml new file mode 100644 index 00000000..fd88efe4 --- /dev/null +++ b/tests/e2e/cases/assert/absent-fail/test.yaml @@ -0,0 +1,17 @@ +name: absent-fail +tags: [assert, absent, fail] + +run: + args: + - --timeout + - 3s + - --interval + - 500ms + - --absent-timeout + - 1s + +expect: + exit_code: 1 + stdout_contains: + - FAIL + - "absent: expected zero results, got 1" diff --git a/tests/e2e/cases/assert/absent/files/oats.yaml b/tests/e2e/cases/assert/absent/files/oats.yaml new file mode 100644 index 00000000..502856dd --- /dev/null +++ b/tests/e2e/cases/assert/absent/files/oats.yaml @@ -0,0 +1,14 @@ +oats: 2 +name: absent +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: present-log-line +expected: + logs: + - logql: '{service_name="missing-{{CASE_NAME}}"}' + absent: true diff --git a/tests/e2e/cases/assert/absent/test.yaml b/tests/e2e/cases/assert/absent/test.yaml new file mode 100644 index 00000000..409b11a8 --- /dev/null +++ b/tests/e2e/cases/assert/absent/test.yaml @@ -0,0 +1,6 @@ +name: absent +tags: [assert, absent] + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/assert/contains-fail/files/oats.yaml b/tests/e2e/cases/assert/contains-fail/files/oats.yaml new file mode 100644 index 00000000..a8722819 --- /dev/null +++ b/tests/e2e/cases/assert/contains-fail/files/oats.yaml @@ -0,0 +1,14 @@ +oats: 2 +name: contains-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + contains: needle diff --git a/tests/e2e/cases/assert/contains-fail/test.yaml b/tests/e2e/cases/assert/contains-fail/test.yaml new file mode 100644 index 00000000..60c4df58 --- /dev/null +++ b/tests/e2e/cases/assert/contains-fail/test.yaml @@ -0,0 +1,15 @@ +name: contains-fail +tags: [assert, contains, fail] + +run: + args: + - --timeout + - 3s + - --interval + - 500ms + +expect: + exit_code: 1 + stdout_contains: + - FAIL + - 'contains: substring "needle" not found in stdout' diff --git a/tests/e2e/cases/assert/contains/files/oats.yaml b/tests/e2e/cases/assert/contains/files/oats.yaml new file mode 100644 index 00000000..613d1412 --- /dev/null +++ b/tests/e2e/cases/assert/contains/files/oats.yaml @@ -0,0 +1,14 @@ +oats: 2 +name: contains +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + contains: seed-log-line diff --git a/tests/e2e/cases/assert/contains/test.yaml b/tests/e2e/cases/assert/contains/test.yaml new file mode 100644 index 00000000..565b92fa --- /dev/null +++ b/tests/e2e/cases/assert/contains/test.yaml @@ -0,0 +1,6 @@ +name: contains +tags: [assert, contains] + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/assert/count/files/oats.yaml b/tests/e2e/cases/assert/count/files/oats.yaml new file mode 100644 index 00000000..4332f937 --- /dev/null +++ b/tests/e2e/cases/assert/count/files/oats.yaml @@ -0,0 +1,16 @@ +oats: 2 +name: count +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + count: '== 1' + match: + - name: seed-log-line diff --git a/tests/e2e/cases/assert/count/test.yaml b/tests/e2e/cases/assert/count/test.yaml new file mode 100644 index 00000000..065070b3 --- /dev/null +++ b/tests/e2e/cases/assert/count/test.yaml @@ -0,0 +1,6 @@ +name: count +tags: [assert, count] + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/assert/match-fail/files/oats.yaml b/tests/e2e/cases/assert/match-fail/files/oats.yaml new file mode 100644 index 00000000..d01489bb --- /dev/null +++ b/tests/e2e/cases/assert/match-fail/files/oats.yaml @@ -0,0 +1,18 @@ +oats: 2 +name: match-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + match: + - name: seed-log-line + attributes: + - key: service_name + value: other-service diff --git a/tests/e2e/cases/assert/match-fail/test.yaml b/tests/e2e/cases/assert/match-fail/test.yaml new file mode 100644 index 00000000..37d2874a --- /dev/null +++ b/tests/e2e/cases/assert/match-fail/test.yaml @@ -0,0 +1,15 @@ +name: match-fail +tags: [assert, match, fail] + +run: + args: + - --timeout + - 3s + - --interval + - 500ms + +expect: + exit_code: 1 + stdout_contains: + - FAIL + - 'match: no row matched name="seed-log-line", attribute service_name="other-service"' diff --git a/tests/e2e/cases/assert/match-regexp/files/oats.yaml b/tests/e2e/cases/assert/match-regexp/files/oats.yaml new file mode 100644 index 00000000..f287a5da --- /dev/null +++ b/tests/e2e/cases/assert/match-regexp/files/oats.yaml @@ -0,0 +1,19 @@ +oats: 2 +name: match-regexp +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + match: + - match_type: regexp + name: seed-.* + attributes: + - key: service_name + value: '{{CASE_NAME}}' diff --git a/tests/e2e/cases/assert/match-regexp/test.yaml b/tests/e2e/cases/assert/match-regexp/test.yaml new file mode 100644 index 00000000..9365f0b4 --- /dev/null +++ b/tests/e2e/cases/assert/match-regexp/test.yaml @@ -0,0 +1,6 @@ +name: match-regexp +tags: [assert, match, regexp] + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml b/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml new file mode 100644 index 00000000..f9d0fa3a --- /dev/null +++ b/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml @@ -0,0 +1,16 @@ +oats: 2 +name: match-spans-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + traces: + - service: {{CASE_NAME}} + spans: + - name: seed-operation +expected: + traces: + - traceql: '{ resource.service.name = "{{CASE_NAME}}" }' + match_spans: + - name: wrong-operation diff --git a/tests/e2e/cases/assert/match-spans-fail/test.yaml b/tests/e2e/cases/assert/match-spans-fail/test.yaml new file mode 100644 index 00000000..382c6aae --- /dev/null +++ b/tests/e2e/cases/assert/match-spans-fail/test.yaml @@ -0,0 +1,15 @@ +name: match-spans-fail +tags: [assert, match-spans, fail] + +run: + args: + - --timeout + - 3s + - --interval + - 500ms + +expect: + exit_code: 1 + stdout_contains: + - FAIL + - 'match_spans: no row matched name="wrong-operation"' diff --git a/tests/e2e/cases/assert/not-contains/files/oats.yaml b/tests/e2e/cases/assert/not-contains/files/oats.yaml new file mode 100644 index 00000000..0d47121a --- /dev/null +++ b/tests/e2e/cases/assert/not-contains/files/oats.yaml @@ -0,0 +1,14 @@ +oats: 2 +name: not-contains +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + not_contains: definitely-not-present diff --git a/tests/e2e/cases/assert/not-contains/test.yaml b/tests/e2e/cases/assert/not-contains/test.yaml new file mode 100644 index 00000000..a5bd7169 --- /dev/null +++ b/tests/e2e/cases/assert/not-contains/test.yaml @@ -0,0 +1,6 @@ +name: not-contains +tags: [assert, not-contains] + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/assert/regex-fail/files/oats.yaml b/tests/e2e/cases/assert/regex-fail/files/oats.yaml new file mode 100644 index 00000000..4580fbb2 --- /dev/null +++ b/tests/e2e/cases/assert/regex-fail/files/oats.yaml @@ -0,0 +1,14 @@ +oats: 2 +name: regex-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + regex: needle.* diff --git a/tests/e2e/cases/assert/regex-fail/test.yaml b/tests/e2e/cases/assert/regex-fail/test.yaml new file mode 100644 index 00000000..6d8f5c94 --- /dev/null +++ b/tests/e2e/cases/assert/regex-fail/test.yaml @@ -0,0 +1,15 @@ +name: regex-fail +tags: [assert, regex, fail] + +run: + args: + - --timeout + - 3s + - --interval + - 500ms + +expect: + exit_code: 1 + stdout_contains: + - FAIL + - 'regex: pattern "needle.*" did not match stdout' diff --git a/tests/e2e/cases/assert/regex/files/oats.yaml b/tests/e2e/cases/assert/regex/files/oats.yaml new file mode 100644 index 00000000..1946fc66 --- /dev/null +++ b/tests/e2e/cases/assert/regex/files/oats.yaml @@ -0,0 +1,14 @@ +oats: 2 +name: regex +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + regex: seed-log-line diff --git a/tests/e2e/cases/assert/regex/test.yaml b/tests/e2e/cases/assert/regex/test.yaml new file mode 100644 index 00000000..802d399a --- /dev/null +++ b/tests/e2e/cases/assert/regex/test.yaml @@ -0,0 +1,6 @@ +name: regex +tags: [assert, regex] + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml b/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml new file mode 100644 index 00000000..12ce8c48 --- /dev/null +++ b/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml @@ -0,0 +1,13 @@ +oats: 2 +name: custom-check-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: custom-check-line +expected: + custom-checks: + - script: ./verify.sh diff --git a/tests/e2e/cases/custom/custom-check-fail/files/verify.sh b/tests/e2e/cases/custom/custom-check-fail/files/verify.sh new file mode 100644 index 00000000..07588ac2 --- /dev/null +++ b/tests/e2e/cases/custom/custom-check-fail/files/verify.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +echo "intentional custom check failure" >&2 +exit 1 diff --git a/tests/e2e/cases/custom/custom-check-fail/test.yaml b/tests/e2e/cases/custom/custom-check-fail/test.yaml new file mode 100644 index 00000000..6dba82d2 --- /dev/null +++ b/tests/e2e/cases/custom/custom-check-fail/test.yaml @@ -0,0 +1,15 @@ +name: custom-check-fail +tags: [custom, check, fail] + +run: + args: + - --timeout + - 200ms + - --interval + - 20ms + +expect: + exit_code: 1 + stdout_contains: + - FAIL + - custom-check diff --git a/tests/e2e/cases/custom/custom-check/files/oats.yaml b/tests/e2e/cases/custom/custom-check/files/oats.yaml new file mode 100644 index 00000000..9250609c --- /dev/null +++ b/tests/e2e/cases/custom/custom-check/files/oats.yaml @@ -0,0 +1,13 @@ +oats: 2 +name: custom-check +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: custom-check-line +expected: + custom-checks: + - script: ./verify.sh diff --git a/tests/e2e/cases/custom/custom-check/files/verify.sh b/tests/e2e/cases/custom/custom-check/files/verify.sh new file mode 100644 index 00000000..e2c140ff --- /dev/null +++ b/tests/e2e/cases/custom/custom-check/files/verify.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +python3 - <<'PY' +import os, urllib.request +urllib.request.urlopen(os.environ['OATS_GRAFANA_URL'] + '/api/health').read() +PY +echo "custom check ok" diff --git a/tests/e2e/cases/custom/custom-check/test.yaml b/tests/e2e/cases/custom/custom-check/test.yaml new file mode 100644 index 00000000..ff1415ac --- /dev/null +++ b/tests/e2e/cases/custom/custom-check/test.yaml @@ -0,0 +1,6 @@ +name: custom-check +tags: [custom, check] + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/fixture/compose-logs-fail/files/docker-compose.oats.yml b/tests/e2e/cases/fixture/compose-logs-fail/files/docker-compose.oats.yml new file mode 100644 index 00000000..f63c06dc --- /dev/null +++ b/tests/e2e/cases/fixture/compose-logs-fail/files/docker-compose.oats.yml @@ -0,0 +1,4 @@ +services: + app: + image: alpine:3.22 + command: ["sh", "-c", "echo app started ok && sleep 1"] diff --git a/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml b/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml new file mode 100644 index 00000000..f4a85a75 --- /dev/null +++ b/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml @@ -0,0 +1,17 @@ +oats: 2 +name: compose-logs-fail +fixture: + type: compose + template: lgtm + compose_file: docker-compose.oats.yml +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + contains: seed-log-line + compose-logs: + - missing log line diff --git a/tests/e2e/cases/fixture/compose-logs-fail/test.yaml b/tests/e2e/cases/fixture/compose-logs-fail/test.yaml new file mode 100644 index 00000000..7b81c114 --- /dev/null +++ b/tests/e2e/cases/fixture/compose-logs-fail/test.yaml @@ -0,0 +1,18 @@ +name: compose-logs-fail +tags: [fixture, compose, compose-logs, fail] + +execution: + mode: serial + +run: + args: + - --timeout + - 200ms + - --interval + - 20ms + +expect: + exit_code: 1 + stdout_contains: + - FAIL + - 'compose-logs: missing "missing log line"' diff --git a/tests/e2e/cases/fixture/compose-logs/files/docker-compose.oats.yml b/tests/e2e/cases/fixture/compose-logs/files/docker-compose.oats.yml new file mode 100644 index 00000000..f63c06dc --- /dev/null +++ b/tests/e2e/cases/fixture/compose-logs/files/docker-compose.oats.yml @@ -0,0 +1,4 @@ +services: + app: + image: alpine:3.22 + command: ["sh", "-c", "echo app started ok && sleep 1"] diff --git a/tests/e2e/cases/fixture/compose-logs/files/oats.yaml b/tests/e2e/cases/fixture/compose-logs/files/oats.yaml new file mode 100644 index 00000000..61c541ee --- /dev/null +++ b/tests/e2e/cases/fixture/compose-logs/files/oats.yaml @@ -0,0 +1,17 @@ +oats: 2 +name: compose-logs +fixture: + type: compose + template: lgtm + compose_file: docker-compose.oats.yml +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + contains: seed-log-line + compose-logs: + - app started ok diff --git a/tests/e2e/cases/fixture/compose-logs/test.yaml b/tests/e2e/cases/fixture/compose-logs/test.yaml new file mode 100644 index 00000000..8d64ed70 --- /dev/null +++ b/tests/e2e/cases/fixture/compose-logs/test.yaml @@ -0,0 +1,9 @@ +name: compose-logs +tags: [fixture, compose, compose-logs] + +execution: + mode: serial + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/fixture/k3d-fail/files/Dockerfile b/tests/e2e/cases/fixture/k3d-fail/files/Dockerfile new file mode 100644 index 00000000..2dd6ce68 --- /dev/null +++ b/tests/e2e/cases/fixture/k3d-fail/files/Dockerfile @@ -0,0 +1,3 @@ +FROM docker.io/library/alpine:3.22 +EXPOSE 8080 +CMD ["sh", "-c", "sleep infinity"] diff --git a/tests/e2e/cases/fixture/k3d-fail/files/k8s/invalid.yaml b/tests/e2e/cases/fixture/k3d-fail/files/k8s/invalid.yaml new file mode 100644 index 00000000..f8a2a656 --- /dev/null +++ b/tests/e2e/cases/fixture/k3d-fail/files/k8s/invalid.yaml @@ -0,0 +1,3 @@ +this is not valid kubernetes yaml: + - because + - it: breaks diff --git a/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml b/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml new file mode 100644 index 00000000..a43daaf0 --- /dev/null +++ b/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml @@ -0,0 +1,19 @@ +oats: 2 +name: k3d-fail +fixture: + type: k3d + k8s_dir: k8s + app_service: dice + app_docker_file: Dockerfile + app_docker_context: . + app_docker_tag: dice:test + app_port: 18080 +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + contains: seed-log-line diff --git a/tests/e2e/cases/fixture/k3d-fail/test.yaml b/tests/e2e/cases/fixture/k3d-fail/test.yaml new file mode 100644 index 00000000..ce72e6af --- /dev/null +++ b/tests/e2e/cases/fixture/k3d-fail/test.yaml @@ -0,0 +1,11 @@ +name: k3d-fail +tags: [fixture, k3d, fail] + +execution: + mode: serial + +expect: + exit_code: 2 + stderr_contains: + - "error parsing k8s/invalid.yaml" + - 'suite "k3d-fail": exit status 1' diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/Dockerfile b/tests/e2e/cases/fixture/k3d-smoke/files/Dockerfile new file mode 100644 index 00000000..2dd6ce68 --- /dev/null +++ b/tests/e2e/cases/fixture/k3d-smoke/files/Dockerfile @@ -0,0 +1,3 @@ +FROM docker.io/library/alpine:3.22 +EXPOSE 8080 +CMD ["sh", "-c", "sleep infinity"] diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/k8s/dice-pod.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/dice-pod.yaml new file mode 100644 index 00000000..b5b8b934 --- /dev/null +++ b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/dice-pod.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: dice + labels: + app: dice +spec: + containers: + - name: dice + image: dice:test + imagePullPolicy: IfNotPresent + command: ["sh", "-c", "sleep infinity"] +--- +apiVersion: v1 +kind: Service +metadata: + name: dice +spec: + selector: + app: dice + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml new file mode 100644 index 00000000..0f321d64 --- /dev/null +++ b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: lgtm +spec: + replicas: 1 + selector: + matchLabels: + app: lgtm + template: + metadata: + labels: + app: lgtm + spec: + containers: + - name: lgtm + image: docker.io/library/alpine:3.22 + command: ["sh", "-c", "sleep infinity"] +--- +apiVersion: v1 +kind: Service +metadata: + name: lgtm +spec: + selector: + app: lgtm + ports: + - name: loki + port: 3100 + targetPort: 3100 + - name: prom + port: 9090 + targetPort: 9090 + - name: tempo + port: 3200 + targetPort: 3200 + - name: pyro + port: 4040 + targetPort: 4040 diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml new file mode 100644 index 00000000..f2863634 --- /dev/null +++ b/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml @@ -0,0 +1,19 @@ +oats: 2 +name: k3d-smoke +fixture: + type: k3d + k8s_dir: k8s + app_service: dice + app_docker_file: Dockerfile + app_docker_context: . + app_docker_tag: dice:test + app_port: 18080 +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + contains: seed-log-line diff --git a/tests/e2e/cases/fixture/k3d-smoke/test.yaml b/tests/e2e/cases/fixture/k3d-smoke/test.yaml new file mode 100644 index 00000000..aafcf874 --- /dev/null +++ b/tests/e2e/cases/fixture/k3d-smoke/test.yaml @@ -0,0 +1,9 @@ +name: k3d-smoke +tags: [fixture, k3d] + +execution: + mode: serial + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/fixture/remote-basic/files/docker-compose.remote.yml b/tests/e2e/cases/fixture/remote-basic/files/docker-compose.remote.yml new file mode 100644 index 00000000..c024466b --- /dev/null +++ b/tests/e2e/cases/fixture/remote-basic/files/docker-compose.remote.yml @@ -0,0 +1,6 @@ +services: + lgtm: + image: docker.io/grafana/otel-lgtm:latest + ports: + - "3001:3000" + - "4319:4318" diff --git a/tests/e2e/cases/fixture/remote-basic/files/oats.yaml b/tests/e2e/cases/fixture/remote-basic/files/oats.yaml new file mode 100644 index 00000000..d982b537 --- /dev/null +++ b/tests/e2e/cases/fixture/remote-basic/files/oats.yaml @@ -0,0 +1,18 @@ +oats: 2 +name: remote-basic +fixture: + type: remote + endpoint: http://localhost:4319 +seed: + type: inline-otlp + logs: + - service: remote-basic + body: seed-log-line +expected: + logs: + - logql: '{service_name="remote-basic"}' + match: + - name: seed-log-line + attributes: + - key: service_name + value: remote-basic diff --git a/tests/e2e/cases/fixture/remote-basic/test.yaml b/tests/e2e/cases/fixture/remote-basic/test.yaml new file mode 100644 index 00000000..e07957b9 --- /dev/null +++ b/tests/e2e/cases/fixture/remote-basic/test.yaml @@ -0,0 +1,37 @@ +name: remote-basic +tags: [fixture, remote] +execution: + mode: serial +run: + shell: | + set -euo pipefail + workdir="$PWD" + project="oats-remote-basic-$$" + trap 'docker compose -p "$project" -f "$workdir/files/docker-compose.remote.yml" down -v --remove-orphans >/dev/null 2>&1 || true' EXIT + docker compose -p "$project" -f "$workdir/files/docker-compose.remote.yml" up -d >/dev/null + for i in $(seq 1 60); do + if curl -fsS http://localhost:3001/api/health >/dev/null; then + break + fi + sleep 2 + done + cat > "$workdir/files/gcx.remote.yaml" < 0 && !matchesAnyFilter(rel, filters) { + continue + } + spec := loadCaseFile(t, filepath.Join(dir, "test.yaml")) + if spec.Execution.Mode == "serial" { + serial = append(serial, dir) + continue + } + parallel = append(parallel, dir) + } + + for _, dir := range parallel { + dir := dir + t.Run(caseName(root, dir), func(t *testing.T) { + t.Parallel() + runCase(t, root, dir) + }) + } + + for _, dir := range serial { + dir := dir + t.Run(caseName(root, dir), func(t *testing.T) { + runCase(t, root, dir) + }) + } +} + +func setupSharedEnv(env *sharedEnv) error { + if _, err := exec.LookPath("docker"); err != nil { + return err + } + tmp, err := os.MkdirTemp("", "oats-e2e-") + if err != nil { + return err + } + env.TempDir = tmp + env.Project = fmt.Sprintf("oatse2e%d", os.Getpid()) + env.ComposeFile = filepath.Join(tmp, "docker-compose.yml") + const composeBody = `services: + lgtm: + image: docker.io/grafana/otel-lgtm:latest + ports: + - "3000:3000" + - "4317:4317" + - "4318:4318" + - "3200:3200" + - "4040:4040" + - "9090:9090" +` + if err := os.WriteFile(env.ComposeFile, []byte(composeBody), 0o644); err != nil { + return err + } + binDir := filepath.Join(tmp, "bin") + build := exec.Command("bash", "-lc", fmt.Sprintf("./scripts/build-local-tools.sh %q", binDir)) + build.Dir = env.RepoRoot + build.Stdout = os.Stdout + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + return fmt.Errorf("build local tools: %w", err) + } + env.OATS = filepath.Join(binDir, "oats") + env.GCX = filepath.Join(binDir, "gcx") + env.GCXConfig = filepath.Join(tmp, "gcx.yaml") + const gcxConfig = `current-context: local +contexts: + local: + grafana: + server: http://localhost:3000 + user: admin + password: admin + org-id: 1 + auth-method: basic + datasources: + prometheus: prometheus + loki: loki + tempo: tempo + pyroscope: pyroscope +` + if err := os.WriteFile(env.GCXConfig, []byte(gcxConfig), 0o600); err != nil { + return err + } + up := exec.Command("docker", "compose", "-p", env.Project, "-f", env.ComposeFile, "up", "-d") + up.Stdout = os.Stdout + up.Stderr = os.Stderr + if err := up.Run(); err != nil { + return fmt.Errorf("start shared lgtm: %w", err) + } + env.RemoteOTLPHTTP = "http://localhost:4318" + if err := waitForHTTP("http://localhost:3000/api/health", 2*time.Minute); err != nil { + return err + } + return nil +} + +func teardownSharedEnv(env *sharedEnv) error { + var errs []string + if env.ComposeFile != "" { + cmd := exec.Command("docker", "compose", "-p", env.Project, "-f", env.ComposeFile, "down", "-v", "--remove-orphans") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + errs = append(errs, err.Error()) + } + } + if env.TempDir != "" { + if err := os.RemoveAll(env.TempDir); err != nil { + errs = append(errs, err.Error()) + } + } + if len(errs) > 0 { + return errors.New(strings.Join(errs, "; ")) + } + return nil +} + +func waitForHTTP(url string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + resp, err := http.Get(url) //nolint:gosec + if err == nil { + _ = resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 500 { + return nil + } + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("timed out waiting for %s", url) +} + +func repoRoot(t *testing.T) string { + t.Helper() + return shared.RepoRoot +} + +func caseName(root, dir string) string { + rel, err := filepath.Rel(filepath.Join(root, "tests", "e2e", "cases"), dir) + if err != nil { + return filepath.Base(dir) + } + return filepath.ToSlash(rel) +} + +func writeFile(t *testing.T, path, label string, data []byte) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir for %s: %v", label, err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatalf("write %s: %v", label, err) + } +} + +func splitFilter(raw string) []string { + if strings.TrimSpace(raw) == "" { + return nil + } + var out []string + for _, part := range strings.Split(raw, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + out = append(out, part) + } + return out +} + +func matchesAnyFilter(path string, filters []string) bool { + for _, f := range filters { + if strings.Contains(path, f) { + return true + } + } + return false +} + +func runCase(t *testing.T, root, dir string) { + t.Helper() + + spec := loadCaseFile(t, filepath.Join(dir, "test.yaml")) + tmp := t.TempDir() + filesDir := filepath.Join(tmp, "files") + if err := os.MkdirAll(filesDir, 0o755); err != nil { + t.Fatalf("mkdir files dir: %v", err) + } + + ph := placeholders{ + RepoRoot: root, + CaseDir: dir, + CaseName: spec.Name, + RemoteOTLPHTTP: shared.RemoteOTLPHTTP, + GCX: shared.GCX, + GCXConfig: shared.GCXConfig, + OATS: shared.OATS, + } + + copyFiles(t, filepath.Join(dir, "files"), filesDir, ph) + + configPath := filepath.Join(tmp, ".generated-oats.toml") + if _, err := os.Stat(filepath.Join(filesDir, "oats.toml")); err == nil { + configPath = filepath.Join(filesDir, "oats.toml") + } else { + writeFile(t, configPath, "generated oats.toml", []byte("cases = [\"files/oats.yaml\"]\n\n[meta]\nversion = 2\n")) + } + + markExecutables(t, filesDir) + + stdout, stderr, exitCode := runCommand(t, spec.Run, ph, tmp, filesDir, configPath) + assertOutput(t, spec.Expect, stdout, stderr, exitCode) +} + +func runCommand(t *testing.T, run runSpec, ph placeholders, cwd, filesDir, configPath string) (string, string, int) { + t.Helper() + + if strings.TrimSpace(run.Shell) != "" { + cmd := exec.Command("bash", "-lc", expand(run.Shell, ph)) + cmd.Dir = cwd + cmd.Env = baseEnv(filesDir, ph) + cmd.Env = append(cmd.Env, renderEnv(run.Env, ph)...) + return captureCommand(t, cmd) + } + + command := strings.TrimSpace(run.Command) + extraArgs := append([]string(nil), run.Args...) + args := []string(nil) + if command == "" { + command = ph.OATS + args = []string{ + "--config", configPath, + "--gcx", ph.GCX, + "--gcx-context", "local", + "--no-cache", + "--timeout", "10s", + "--interval", "1s", + } + args = append(args, extraArgs...) + } else { + args = extraArgs + } + command = expand(command, ph) + for i := range args { + args[i] = expand(args[i], ph) + } + cmd := exec.Command(command, args...) + cmd.Dir = ph.RepoRoot + cmd.Env = baseEnv(filesDir, ph) + cmd.Env = append(cmd.Env, renderEnv(run.Env, ph)...) + return captureCommand(t, cmd) +} + +func captureCommand(t *testing.T, cmd *exec.Cmd) (string, string, int) { + t.Helper() + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + if err == nil { + return stdout.String(), stderr.String(), 0 + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return stdout.String(), stderr.String(), exitErr.ExitCode() + } + t.Fatalf("run command: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + return "", "", 0 +} + +func discoverCases(t *testing.T, root string) []string { + t.Helper() + var dirs []string + err := filepath.WalkDir(root, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + if d.Name() == "test.yaml" { + dirs = append(dirs, filepath.Dir(path)) + } + return nil + }) + if err != nil { + t.Fatalf("discover cases: %v", err) + } + return dirs +} + +func loadCaseFile(t *testing.T, path string) caseFile { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var spec caseFile + if err := yaml.Unmarshal(data, &spec); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + if spec.Name == "" { + t.Fatalf("%s: name is required", path) + } + return spec +} + +func copyFiles(t *testing.T, srcDir, dstDir string, ph placeholders) { + t.Helper() + if _, err := os.Stat(srcDir); err != nil { + if os.IsNotExist(err) { + return + } + t.Fatalf("stat %s: %v", srcDir, err) + } + err := filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(srcDir, path) + if err != nil { + return err + } + out := filepath.Join(dstDir, rel) + data, err := os.ReadFile(path) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil { + return err + } + return os.WriteFile(out, []byte(expand(string(data), ph)), 0o644) + }) + if err != nil { + t.Fatalf("copy files: %v", err) + } +} + +func markExecutables(t *testing.T, filesDir string) { + t.Helper() + for _, rel := range []string{"verify.sh", "setup.sh"} { + path := filepath.Join(filesDir, rel) + if _, err := os.Stat(path); err == nil { + if chmodErr := os.Chmod(path, 0o755); chmodErr != nil { + t.Fatalf("chmod %s: %v", path, chmodErr) + } + } + } + binDir := filepath.Join(filesDir, "bin") + _ = filepath.WalkDir(binDir, func(path string, d os.DirEntry, err error) error { + if err != nil || d == nil || d.IsDir() { + return err + } + if chmodErr := os.Chmod(path, 0o755); chmodErr != nil { + t.Fatalf("chmod %s: %v", path, chmodErr) + } + return nil + }) +} + +func assertOutput(t *testing.T, expect expectSpec, stdout, stderr string, exitCode int) { + t.Helper() + wantExit := 0 + if expect.ExitCode != nil { + wantExit = *expect.ExitCode + } + if exitCode != wantExit { + t.Fatalf("exit code: got %d want %d\nstdout:\n%s\nstderr:\n%s", exitCode, wantExit, stdout, stderr) + } + for _, want := range expect.StdoutContains { + if !strings.Contains(stdout, want) { + t.Fatalf("stdout missing %q\nstdout:\n%s\nstderr:\n%s", want, stdout, stderr) + } + } + for _, want := range expect.StdoutNotContains { + if strings.Contains(stdout, want) { + t.Fatalf("stdout unexpectedly contains %q\nstdout:\n%s\nstderr:\n%s", want, stdout, stderr) + } + } + for _, want := range expect.StderrContains { + if !strings.Contains(stderr, want) { + t.Fatalf("stderr missing %q\nstdout:\n%s\nstderr:\n%s", want, stdout, stderr) + } + } +} + +func renderEnv(env map[string]string, ph placeholders) []string { + if len(env) == 0 { + return nil + } + out := make([]string, 0, len(env)) + for k, v := range env { + out = append(out, k+"="+expand(v, ph)) + } + return out +} + +func baseEnv(filesDir string, ph placeholders) []string { + env := append([]string(nil), os.Environ()...) + env = replaceEnv(env, "GCX_CONFIG", ph.GCXConfig) + binDir := filepath.Join(filesDir, "bin") + if info, err := os.Stat(binDir); err == nil && info.IsDir() { + pathValue := os.Getenv("PATH") + if pathValue == "" { + pathValue = binDir + } else { + pathValue = binDir + string(os.PathListSeparator) + pathValue + } + env = replaceEnv(env, "PATH", pathValue) + } + return env +} + +func replaceEnv(env []string, key, value string) []string { + prefix := key + "=" + out := make([]string, 0, len(env)+1) + for _, kv := range env { + if strings.HasPrefix(kv, prefix) { + continue + } + out = append(out, kv) + } + return append(out, prefix+value) +} + +func expand(text string, ph placeholders) string { + return strings.NewReplacer( + "{{REPO_ROOT}}", ph.RepoRoot, + "{{CASE_DIR}}", ph.CaseDir, + "{{CASE_NAME}}", ph.CaseName, + "{{REMOTE_OTLP_HTTP}}", ph.RemoteOTLPHTTP, + "{{GCX}}", ph.GCX, + "{{GCX_CONFIG}}", ph.GCXConfig, + "{{OATS}}", ph.OATS, + ).Replace(text) +} diff --git a/tests/e2e/oats.yaml b/tests/e2e/oats.yaml deleted file mode 100644 index 7ba6fcc2..00000000 --- a/tests/e2e/oats.yaml +++ /dev/null @@ -1,26 +0,0 @@ -oats-schema-version: 2 -docker-compose: - files: - - docker-compose.oats.yml -input: - - path: /vets.html -expected: - traces: - - traceql: '{name="GET /vets.html"}' - equals: "GET /vets.html" - attributes: - otel.library.name: io.opentelemetry.tomcat-7.0 - - traceql: '{name="GET /foo"}' - count: - max: 0 - metrics: - - promql: 'histogram_quantile(0.95, sum by(le) (rate(http_server_request_duration_seconds_bucket{http_route="/vets.html"}[5m])))' - value: "> 0" - logs: - - logql: '{service_name="spring-pet-clinic"} |= `Completed initialization`' - count: - min: 1 - max: 1 - - logql: '{service_name="spring-pet-clinic"} |= `Crashed during startup`' - count: - max: 0 diff --git a/tests/integration_test.go b/tests/integration_test.go deleted file mode 100644 index 7ffc2d77..00000000 --- a/tests/integration_test.go +++ /dev/null @@ -1,319 +0,0 @@ -package tests - -import ( - "os" - "testing" - "time" - - "github.com/grafana/oats/model" - "github.com/grafana/oats/yaml" - "github.com/onsi/gomega" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestIntegration(t *testing.T) { - key := "INTEGRATION_TESTS" - if os.Getenv(key) != "true" { - t.Skipf("skipping integration test; set %s=true to run", key) - } - - // tests must not run in parallel because they share the same docker compose environment - - promTest := model.Expected{ - Metrics: []model.ExpectedMetrics{ - { - PromQL: "histogram_quantile(0.95, sum by(le) (rate(http_server_request_duration_seconds_bucket{http_route=\"/vets.html\"}[5m])))", - Value: "> 0", - }, - }, - } - - tests := []struct { - name string - shouldPanic bool - expected model.Expected - }{ - { - name: "prometheus metrics pass", - shouldPanic: false, - expected: promTest, - }, - { - name: "multiple metrics pass", - shouldPanic: false, - expected: model.Expected{ - Metrics: []model.ExpectedMetrics{ - { - PromQL: "histogram_quantile(0.95, sum by(le) (rate(http_server_request_duration_seconds_bucket{http_route=\"/vets.html\"}[5m])))", - Value: "> 0", - }, - { - PromQL: "rate(http_server_request_duration_seconds_count{http_route=\"/vets.html\"}[5m])", - Value: "> 0", - }, - }, - }, - }, - { - name: "logs check pass", - shouldPanic: false, - expected: model.Expected{ - Logs: []model.ExpectedLogs{ - { - LogQL: "{service_name=\"spring-pet-clinic\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 1, Max: 1000}, - }, - }, - }, - }, - }, - { - name: "traces check pass", - shouldPanic: false, - expected: model.Expected{ - Traces: []model.ExpectedTraces{ - { - TraceQL: "{name=\"GET /vets.html\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 1, Max: 100}, - }, - }, - }, - }, - }, - { - name: "composed checks with metrics and logs", - shouldPanic: false, - expected: model.Expected{ - Metrics: []model.ExpectedMetrics{ - { - PromQL: "histogram_quantile(0.95, sum by(le) (rate(http_server_request_duration_seconds_bucket{http_route=\"/vets.html\"}[5m])))", - Value: "> 0", - }, - }, - Logs: []model.ExpectedLogs{ - { - LogQL: "{service_name=\"spring-pet-clinic\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 1, Max: 1000}, - }, - }, - }, - }, - }, - { - name: "successfully check for absence - non-existent logs", - shouldPanic: false, - expected: model.Expected{ - Logs: []model.ExpectedLogs{ - { - LogQL: "{job=\"nonexistent-job\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 0, Max: 0}, - }, - }, - }, - }, - }, - { - name: "successfully check for absence - non-existent traces", - shouldPanic: false, - expected: model.Expected{ - Traces: []model.ExpectedTraces{ - { - TraceQL: "{name=\"/nonexistent-endpoint\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 0, Max: 0}, - }, - }, - }, - }, - }, - { - name: "successfully check for absence - non-existent service logs", - shouldPanic: false, - expected: model.Expected{ - Logs: []model.ExpectedLogs{ - { - LogQL: "{service_name=\"service-that-does-not-exist\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 0, Max: 0}, - }, - }, - }, - }, - }, - { - name: "successfully check for absence - traces with specific attribute", - shouldPanic: false, - expected: model.Expected{ - Traces: []model.ExpectedTraces{ - { - TraceQL: "{.http.status_code=999}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 0, Max: 0}, - }, - }, - }, - }, - }, - { - name: "should fail - checking for absence when logs exist", - shouldPanic: true, - expected: model.Expected{ - Logs: []model.ExpectedLogs{ - { - LogQL: "{service_name=\"spring-pet-clinic\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 0, Max: 0}, - }, - }, - }, - }, - }, - { - name: "should fail - checking for absence when traces exist", - shouldPanic: true, - expected: model.Expected{ - Traces: []model.ExpectedTraces{ - { - TraceQL: "{name=\"GET /vets.html\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 0, Max: 0}, - }, - }, - }, - }, - }, - { - name: "should fail - metric value doesn't match condition", - shouldPanic: true, - expected: model.Expected{ - Metrics: []model.ExpectedMetrics{ - { - PromQL: "histogram_quantile(0.95, sum by(le) (rate(http_server_request_duration_seconds_bucket{http_route=\"/vets.html\"}[5m])))", - Value: "< 0", - }, - }, - }, - }, - { - name: "should fail - looking for non-existent logs", - shouldPanic: true, - expected: model.Expected{ - Logs: []model.ExpectedLogs{ - { - LogQL: "{job=\"nonexistent-job\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 1, Max: 1000}, - }, - }, - }, - }, - }, - { - name: "should fail - looking for non-existent traces", - shouldPanic: true, - expected: model.Expected{ - Traces: []model.ExpectedTraces{ - { - TraceQL: "{name=\"/nonexistent-endpoint\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 1, Max: 100}, - }, - }, - }, - }, - }, - { - name: "should fail - count range too restrictive", - shouldPanic: true, - expected: model.Expected{ - Logs: []model.ExpectedLogs{ - { - LogQL: "{service_name=\"spring-pet-clinic\"}", - Signal: model.ExpectedSignal{ - Count: &model.ExpectedRange{Min: 10000, Max: 20000}, - }, - }, - }, - }, - }, - } - - failed := "" - failFast := true - gomega.RegisterFailHandler(func(message string, callerSkip ...int) { - if failFast { - panic(message) - } - failed = message - t.Log("test failed:", message) - }) - - buildDir := yaml.PrepareBuildDir("integration-test") - c := &model.TestCase{ - Name: "integration-test", - OutputDir: buildDir, - Definition: model.TestCaseDefinition{ - DockerCompose: &model.DockerCompose{Files: []string{ - "e2e/docker-compose.oats.yml", - }}, - Input: []model.Input{{ - Path: "/vets.html", - }}, - Expected: promTest, - }, - } - c.ValidateAndSetVariables(gomega.Default) - - checkTimeout := 2 * time.Second - startupTimeout := 2 * time.Minute - - settings := model.Settings{ - Host: "localhost", - Timeout: startupTimeout, - PresentTimeout: startupTimeout, // first test needs more time to allow docker compose to start - AbsentTimeout: checkTimeout, - LogLimit: 1000, - LgtmLogSettings: map[string]bool{ - "ENABLE_LOGS_ALL": false, - }, - LgtmVersion: "latest", - } - - r := yaml.NewRunner(c, settings) - - t.Log("start docker compose") - - end, err := r.StartEndpoint() - assert.NoError(t, err, "expected no error starting an observability endpoint") - defer end() - - t.Log("finished starting docker compose") - - assert.Empty(t, failed, "expected no failure starting docker compose") - - failFast = false - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c.Name = tt.name - c.Definition.Expected = tt.expected - - failed = "" - r.ExecuteChecks() - - // docker compose is already started, so speed up the checks for all subsequent tests - r.Settings.PresentTimeout = checkTimeout - - if tt.shouldPanic { - require.NotEmpty(t, failed) - } else { - require.Empty(t, failed) - } - }) - } -} From fbb33d3912587a9bdd64d43a5dfbd046ea036c39 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 2 Jul 2026 17:44:01 +0200 Subject: [PATCH 066/124] fix lint and k3d e2e failure case Signed-off-by: Gregor Zeitlinger --- internal/cli/main.go | 5 ----- internal/cli/testdata/fake-gcx.sh | 2 +- tests/e2e/cases/fixture/k3d-fail/files/k8s/invalid.yaml | 9 ++++++--- tests/e2e/cases/fixture/k3d-fail/test.yaml | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/internal/cli/main.go b/internal/cli/main.go index f86c9aa8..aff96cab 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -70,11 +70,6 @@ var ( waitForGrafanaToken = waitForGrafanaTokenImpl ) -func main() { - code := Run() - os.Exit(code) -} - func Main() { os.Exit(Run()) } diff --git a/internal/cli/testdata/fake-gcx.sh b/internal/cli/testdata/fake-gcx.sh index 7b39ad15..9d99d84f 100755 --- a/internal/cli/testdata/fake-gcx.sh +++ b/internal/cli/testdata/fake-gcx.sh @@ -11,7 +11,7 @@ set -euo pipefail # would produce via signalcmd. while [[ $# -gt 0 ]]; do case "${1:-}" in - --context|--config) + --context | --config) shift 2 ;; *) diff --git a/tests/e2e/cases/fixture/k3d-fail/files/k8s/invalid.yaml b/tests/e2e/cases/fixture/k3d-fail/files/k8s/invalid.yaml index f8a2a656..2b414e40 100644 --- a/tests/e2e/cases/fixture/k3d-fail/files/k8s/invalid.yaml +++ b/tests/e2e/cases/fixture/k3d-fail/files/k8s/invalid.yaml @@ -1,3 +1,6 @@ -this is not valid kubernetes yaml: - - because - - it: breaks +apiVersion: v1 +kind: Pod +metadata: + name: invalid +spec: + containers: definitely-not-a-list diff --git a/tests/e2e/cases/fixture/k3d-fail/test.yaml b/tests/e2e/cases/fixture/k3d-fail/test.yaml index ce72e6af..132c87c3 100644 --- a/tests/e2e/cases/fixture/k3d-fail/test.yaml +++ b/tests/e2e/cases/fixture/k3d-fail/test.yaml @@ -7,5 +7,5 @@ execution: expect: exit_code: 2 stderr_contains: - - "error parsing k8s/invalid.yaml" + - 'error when creating "k8s/invalid.yaml"' - 'suite "k3d-fail": exit status 1' From 6f510a4c0002c28b6e8ea47d5e4e779734d15066 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 2 Jul 2026 17:53:20 +0200 Subject: [PATCH 067/124] fix ci test split and install k3d tools Signed-off-by: Gregor Zeitlinger --- mise.toml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mise.toml b/mise.toml index 9a52ab01..7c668af2 100644 --- a/mise.toml +++ b/mise.toml @@ -2,6 +2,8 @@ go = "1.26.4" goreleaser = "2.16.0" node = "24.18.0" +k3d = "5.8.3" +kubectl = "1.34.1" # Linters actionlint = "1.7.12" @@ -32,11 +34,11 @@ run = "flint run --fix" [tasks.test] description = "Run tests" -run = "go test $(go list ./...)" +run = 'GOFLAGS=-buildvcs=false go test $(GOFLAGS=-buildvcs=false go list ./... | grep -v "github.com/grafana/oats/tests/e2e")' [tasks.e2e-test] -description = "Run acceptance case suite" -run = "go test ./tests/e2e -run TestCases" +description = "Run e2e case suite" +run = "go test -buildvcs=false ./tests/e2e -run TestCases" [tasks.build] description = "Build the project" From 59c18a104b3a3248e07d6dd8da55b6b56840423b Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 2 Jul 2026 17:54:06 +0200 Subject: [PATCH 068/124] track e2e tools in mise and renovate Signed-off-by: Gregor Zeitlinger --- .github/renovate-tracked-deps.json | 2 ++ AGENTS.md | 2 +- mise.toml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/renovate-tracked-deps.json b/.github/renovate-tracked-deps.json index 8e140593..9b62f9ba 100644 --- a/.github/renovate-tracked-deps.json +++ b/.github/renovate-tracked-deps.json @@ -79,6 +79,8 @@ "golangci-lint", "goreleaser", "hadolint", + "k3d", + "kubectl", "lychee", "node", "npm:renovate", diff --git a/AGENTS.md b/AGENTS.md index 495d6b28..f3de6cac 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ validate traces (TraceQL), logs (LogQL), metrics (PromQL), and profiles # Build mise run build -# Run unit tests +# Run non-e2e tests mise run test # Run e2e tests diff --git a/mise.toml b/mise.toml index 7c668af2..3b9627e8 100644 --- a/mise.toml +++ b/mise.toml @@ -1,9 +1,9 @@ [tools] go = "1.26.4" goreleaser = "2.16.0" -node = "24.18.0" k3d = "5.8.3" kubectl = "1.34.1" +node = "24.18.0" # Linters actionlint = "1.7.12" From f7945316fb3895cbf052d8e6d64ba3f53da7f38a Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Thu, 2 Jul 2026 18:27:39 +0200 Subject: [PATCH 069/124] split e2e matrix and address review comments Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 32 +++++++++++++------------------- internal/cli/main.go | 8 +++++--- report/text.go | 8 ++++++++ runner/runner.go | 31 +++++++++++++++++++++++-------- seed/seed.go | 23 ++++++++++++++++++----- testhelpers/compose/compose.go | 3 ++- testhelpers/requests/http.go | 6 +++++- wait/wait.go | 6 +++--- 8 files changed, 77 insertions(+), 40 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 7ffb76e6..f4f49eb8 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -6,8 +6,18 @@ on: [pull_request] permissions: {} jobs: - test-core-and-compose: + test-e2e: runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - name: core + filter: "assert/,custom/,seed/" + - name: compose-and-remote + filter: "fixture/compose,fixture/remote" + - name: k3d + filter: "fixture/k3d" steps: - name: Check out uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -17,23 +27,7 @@ jobs: with: version: v2026.6.14 sha256: 96ae1ef7b00a6ebbbec23ba1016d6e722f5e904966272f621d15326429e90d53 - - name: Run core e2e tests + - name: Run ${{ matrix.name }} e2e tests env: - OATS_E2E_FILTER: "assert/,custom/,seed/,fixture/compose,fixture/remote" - run: mise run e2e-test - - test-k3d: - runs-on: ubuntu-24.04 - steps: - - name: Check out - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 - with: - version: v2026.6.14 - sha256: 96ae1ef7b00a6ebbbec23ba1016d6e722f5e904966272f621d15326429e90d53 - - name: Run k3d e2e tests - env: - OATS_E2E_FILTER: "fixture/k3d" + OATS_E2E_FILTER: ${{ matrix.filter }} run: mise run e2e-test diff --git a/internal/cli/main.go b/internal/cli/main.go index aff96cab..664d056b 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -321,7 +321,7 @@ type startableSuiteFixture interface { Up() error } -func startFixture(_ context.Context, plan discovery.Plan) (suiteFixture, error) { +func startFixture(ctx context.Context, plan discovery.Plan) (suiteFixture, error) { switch plan.Fixture.Type { case "", "remote": return nil, nil @@ -346,7 +346,7 @@ func startFixture(_ context.Context, plan discovery.Plan) (suiteFixture, error) return composeFixture{suite: suite, cleanup: cleanup}, nil case "k3d": ep := newKubernetesEndpoint(plan) - if err := ep.Start(context.Background()); err != nil { + if err := ep.Start(ctx); err != nil { return nil, err } return endpointFixture{ep: ep}, nil @@ -635,7 +635,9 @@ func splitCSV(s string) []string { // output, or "" if gcx is unreachable. The version contributes to the // cache key so an upgrade to gcx invalidates all green records. func gcxVersion(bin string) string { - out, err := exec.Command(bin, "--version").Output() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, bin, "--version").Output() if err != nil { return "" } diff --git a/report/text.go b/report/text.go index 3d253d27..10057808 100644 --- a/report/text.go +++ b/report/text.go @@ -115,6 +115,7 @@ func (r *TextReporter) emitGHAAnnotation(e Event) { if msg == "" { msg = "OATS assertion failed" } + msg = ghaEscape(msg) if line > 0 { r.write("::error file=%s,line=%d::%s\n", file, line, msg) } else { @@ -122,6 +123,13 @@ func (r *TextReporter) emitGHAAnnotation(e Event) { } } +func ghaEscape(s string) string { + s = strings.ReplaceAll(s, "%", "%25") + s = strings.ReplaceAll(s, "\r", "%0D") + s = strings.ReplaceAll(s, "\n", "%0A") + return s +} + func (r *TextReporter) flushRunEnd(e Event) { // Failure blocks first so the summary line is the final thing the reader // sees — useful for both humans (scroll to bottom) and CI logs (tail). diff --git a/runner/runner.go b/runner/runner.go index 4eb9ae46..3e602234 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -93,9 +93,6 @@ func (o Options) withDefaults() Options { if o.AbsentTimeout <= 0 { o.AbsentTimeout = 10 * time.Second } - if o.SeedSettleDelay < 0 { - o.SeedSettleDelay = 2 * time.Second - } if o.SeedSettleDelay == 0 { o.SeedSettleDelay = 2 * time.Second } @@ -365,7 +362,7 @@ func customCheckCommand(ctx context.Context, dir, script string, extraEnv []stri return nil, cleanup, err } cleanup = func() { _ = os.Remove(f.Name()) } - cmd := exec.CommandContext(ctx, "sh", f.Name()) + cmd := exec.CommandContext(ctx, f.Name()) cmd.Dir = dir cmd.Env = append(cmd.Environ(), extraEnv...) return cmd, cleanup, nil @@ -442,16 +439,27 @@ func (r *Runner) seedCase(c *casefile.Case) error { if r.seeder.OTLPEndpoint == "" { return fmt.Errorf("inline-otlp seed requires Endpoint.OTLPHTTP") } - return r.seeder.Send(toSeedPayload(c.Seed)) + payload, err := toSeedPayload(c.Seed) + if err != nil { + return err + } + return r.seeder.Send(payload) } return fmt.Errorf("unknown seed type %q", c.Seed.Type) } -func toSeedPayload(s casefile.Seed) seed.Payload { +func toSeedPayload(s casefile.Seed) (seed.Payload, error) { p := seed.Payload{} for _, t := range s.Traces { for _, sp := range t.Spans { - dur, _ := time.ParseDuration(sp.Duration) + dur := time.Duration(0) + if sp.Duration != "" { + parsed, err := time.ParseDuration(sp.Duration) + if err != nil { + return seed.Payload{}, fmt.Errorf("seed trace span %q: invalid duration %q: %w", sp.Name, sp.Duration, err) + } + dur = parsed + } p.Traces = append(p.Traces, seed.Trace{ Service: t.Service, Span: seed.SpanFields{ @@ -477,7 +485,7 @@ func toSeedPayload(s casefile.Seed) seed.Payload { Value: m.Value, }) } - return p + return p, nil } // pollAssert handles the polling loop common to all signal types. The @@ -505,6 +513,13 @@ func (r *Runner) pollAssert( Case: c.Name, Cmd: cmdStr, }) + if res.ExitCode != 0 { + detail := trimOutput(strings.TrimSpace(res.Stderr)) + if detail == "" { + detail = fmt.Sprintf("gcx exit code %d", res.ExitCode) + } + return []assert.Failure{{Rule: "exec", Detail: detail}} + } return evalFn(res.Stdout, res.Stderr, res.ExitCode) } diff --git a/seed/seed.go b/seed/seed.go index 51af6758..67d64b6d 100644 --- a/seed/seed.go +++ b/seed/seed.go @@ -20,9 +20,12 @@ import ( "fmt" "io" "net/http" + "strings" "time" ) +var defaultHTTPClient = &http.Client{Timeout: 10 * time.Second} + // Payload describes one inline-otlp seed declaration as it arrives from a // case yaml. All three signal slices are optional; emitting only the ones // the case cares about keeps inline payloads short. @@ -70,7 +73,7 @@ func (s *Sender) httpClient() *http.Client { if s.Client != nil { return s.Client } - return http.DefaultClient + return defaultHTTPClient } // Send pushes all signals declared in p. Returns the first error encountered, @@ -128,7 +131,7 @@ func (s *Sender) sendTrace(t Trace, now time.Time) error { }] }] }] -}`, t.Service, randHex(16), randHex(8), t.Span.Name, kind, start, end) +}`, t.Service, mustRandHex(16), mustRandHex(8), t.Span.Name, kind, start, end) return s.post("/v1/traces", body) } @@ -201,10 +204,20 @@ func (s *Sender) post(path, body string) error { return nil } -func randHex(nBytes int) string { +func randHex(nBytes int) (string, error) { buf := make([]byte, nBytes) if _, err := rand.Read(buf); err != nil { - panic(err) + return "", err + } + return hex.EncodeToString(buf), nil +} + +func mustRandHex(nBytes int) string { + s, err := randHex(nBytes) + if err != nil { + // Extremely rare; fall back to a stable all-zero ID rather than panic so + // the run fails, if at all, through normal backend/query behavior. + return strings.Repeat("0", nBytes*2) } - return hex.EncodeToString(buf) + return s } diff --git a/testhelpers/compose/compose.go b/testhelpers/compose/compose.go index b966d872..3fea4863 100644 --- a/testhelpers/compose/compose.go +++ b/testhelpers/compose/compose.go @@ -80,7 +80,7 @@ func (c *Compose) Remove() error { func (c *Compose) runDocker(cc command) error { var cmdArgs []string if cc.compose { - cmdArgs = c.DefaultArgs + cmdArgs = append([]string(nil), c.DefaultArgs...) } cmdArgs = append(cmdArgs, cc.args...) cmd := exec.Command(c.Command, cmdArgs...) @@ -114,6 +114,7 @@ func (c *Compose) runDocker(cc command) error { if err != nil { return fmt.Errorf("failed to start docker command: %w", err) } + go func() { _ = cmd.Wait() }() } else { slog.Info("Running", "command", cmd.String(), "dir", strings.Join(c.Paths, ",")) cmd.Stdout = os.Stdout diff --git a/testhelpers/requests/http.go b/testhelpers/requests/http.go index 2af3887c..e3244321 100644 --- a/testhelpers/requests/http.go +++ b/testhelpers/requests/http.go @@ -6,12 +6,16 @@ import ( "io" "net/http" "strings" + "time" ) var tr = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, } -var testHTTPClient = &http.Client{Transport: tr} +var testHTTPClient = &http.Client{ + Transport: tr, + Timeout: 10 * time.Second, +} func doRequest(req *http.Request, statusCode int) error { r, err := testHTTPClient.Do(req) diff --git a/wait/wait.go b/wait/wait.go index 6a1e2065..267396e4 100644 --- a/wait/wait.go +++ b/wait/wait.go @@ -63,7 +63,7 @@ func Until[F any](ctx context.Context, opts Options, asserter Asserter[F]) Resul if len(last) == 0 { return Result[F]{OK: true, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} } - if time.Now().After(deadline) { + if !time.Now().Before(deadline) { return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: last} } if ctx.Err() != nil { @@ -97,12 +97,12 @@ func While[F any](ctx context.Context, opts Options, asserter Asserter[F]) Resul return Result[F]{OK: true, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} } if ctx.Err() != nil { - return Result[F]{OK: true, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} + return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} } select { case <-time.After(opts.Interval): case <-ctx.Done(): - return Result[F]{OK: true, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} + return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} } } } From 20666986bc695bc9b7b87ca40aad8fa8e0d4ac39 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 11:25:29 +0200 Subject: [PATCH 070/124] fix: finalize schema v3 and stabilize e2e Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 20 +++- AGENTS.md | 2 +- CURRENT.md | 4 +- UPGRADING.md | 8 +- casefile/case.go | 35 ++++-- casefile/case_test.go | 44 +++---- discovery/discovery_test.go | 6 +- examples/fixtures/cases/compose/rolldice.yaml | 2 +- examples/fixtures/cases/k3d/rolldice.yaml | 2 +- examples/smoke/cases/custom-check.yaml | 2 +- examples/smoke/cases/inline-seed.yaml | 2 +- examples/smoke/cases/profile.yaml | 2 +- examples/smoke/cases/rolldice.yaml | 2 +- internal/cli/integration_test.go | 6 +- migrate/migrate.go | 6 +- migrate/migrate_test.go | 2 +- report/event.go | 6 +- runner/cache_integration_test.go | 2 +- runner/runner.go | 46 ++++++-- runner/runner_test.go | 26 ++--- testhelpers/compose/compose.go | 4 +- .../cases/assert/absent-fail/files/oats.yaml | 2 +- tests/e2e/cases/assert/absent/files/oats.yaml | 2 +- .../assert/contains-fail/files/oats.yaml | 2 +- .../e2e/cases/assert/contains/files/oats.yaml | 2 +- tests/e2e/cases/assert/count/files/oats.yaml | 2 +- .../cases/assert/match-fail/files/oats.yaml | 2 +- .../cases/assert/match-regexp/files/oats.yaml | 2 +- .../assert/match-spans-fail/files/oats.yaml | 2 +- .../cases/assert/not-contains/files/oats.yaml | 2 +- .../cases/assert/regex-fail/files/oats.yaml | 2 +- tests/e2e/cases/assert/regex/files/oats.yaml | 2 +- .../custom/custom-check-fail/files/oats.yaml | 2 +- .../cases/custom/custom-check/files/oats.yaml | 2 +- .../fixture/compose-logs-fail/files/oats.yaml | 2 +- .../fixture/compose-logs/files/oats.yaml | 2 +- .../cases/fixture/k3d-fail/files/oats.yaml | 2 +- .../cases/fixture/k3d-smoke/files/oats.yaml | 2 +- .../files/docker-compose.remote.yml | 4 +- .../fixture/remote-basic/files/oats.yaml | 2 +- .../e2e/cases/fixture/remote-basic/test.yaml | 26 ++++- .../seed/inline-otlp-invalid/files/oats.yaml | 2 +- .../cases/seed/inline-otlp/files/oats.yaml | 2 +- tests/e2e/e2e_test.go | 109 +++++++++++++----- 44 files changed, 265 insertions(+), 143 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index f4f49eb8..7bd09b48 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -12,12 +12,20 @@ jobs: fail-fast: false matrix: include: - - name: core - filter: "assert/,custom/,seed/" - - name: compose-and-remote - filter: "fixture/compose,fixture/remote" - - name: k3d - filter: "fixture/k3d" + - name: assert-a + filter: "assert/absent,assert/contains,assert/count" + - name: assert-b + filter: "assert/match,assert/not-contains,assert/regex" + - name: custom-and-seed + filter: "custom/,seed/" + - name: compose + filter: "fixture/compose" + - name: remote + filter: "fixture/remote" + - name: k3d-fail + filter: "fixture/k3d-fail" + - name: k3d-smoke + filter: "fixture/k3d-smoke" steps: - name: Check out uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 diff --git a/AGENTS.md b/AGENTS.md index f3de6cac..8ce1783e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,7 @@ OpenTelemetry written in Go. It enables full round-trip testing from instrumented applications to the observability stack (LGTM: Loki, Grafana, Tempo, Prometheus, Mimir, OpenTelemetry Collector). -Test cases are defined in YAML files with `oats-schema-version: 2` and can +Test cases are defined in YAML files with `oats-schema-version: 3` and can validate traces (TraceQL), logs (LogQL), metrics (PromQL), and profiles (Pyroscope queries) using Docker Compose or Kubernetes backends. diff --git a/CURRENT.md b/CURRENT.md index 0add1d7d..1c3f0b27 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -123,7 +123,7 @@ ttl_days = 7 # default ## Case yaml shape ```yaml -oats: 2 +oats-schema-version: 3 name: rolldice traces have route attribute fixture: @@ -167,7 +167,7 @@ but the runner does not require it when a fixture already provides the app. Inline-OTLP seed (no example app required): ```yaml -oats: 2 +oats-schema-version: 3 name: gcx returns seeded trace seed: diff --git a/UPGRADING.md b/UPGRADING.md index 58f99747..f43a1f60 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -32,14 +32,14 @@ All test files must now include an `oats-schema-version` tag. Full release notes: -### Add `oats-schema-version: 2` to all test files +### Add `oats-schema-version: 3` to all test files All OATS test files must now include the `oats-schema-version` field at the top level. The current version is `2`. ```yaml # ✅ Required in all test files -oats-schema-version: 2 +oats-schema-version: 3 docker-compose: files: @@ -70,7 +70,7 @@ points must now use the `oats-template: true` flag: ```yaml # Template file (e.g., oats-base.yaml) -oats-schema-version: 2 +oats-schema-version: 3 oats-template: true docker-compose: @@ -97,7 +97,7 @@ and want to avoid parsing all of them. ### Migration Steps -1. Add `oats-schema-version: 2` to all your test files +1. Add `oats-schema-version: 3` to all your test files 2. Add `oats-template: true` to any template files (files that are included but not entry points) 3. (Optional) Consider passing specific file paths instead of directories for diff --git a/casefile/case.go b/casefile/case.go index 17834481..84cb0d22 100644 --- a/casefile/case.go +++ b/casefile/case.go @@ -21,19 +21,20 @@ import ( "go.yaml.in/yaml/v3" ) -// SchemaVersion is the value of the top-level `oats:` field that this loader -// understands. Cases with any other value are rejected at parse time. -const SchemaVersion = 2 +// SchemaVersion is the value of the top-level `oats-schema-version` field +// that this loader understands. Cases with any other value are rejected at +// parse time. +const SchemaVersion = 3 // Case is one entry point yaml file. Cases are independently runnable; // suites group them via oats.toml. type Case struct { - OatsVersion int `yaml:"oats"` - Name string `yaml:"name"` - Tags []string `yaml:"tags,omitempty"` - Hermetic *bool `yaml:"hermetic,omitempty"` // pointer: distinguish unset vs explicit false - Interval time.Duration `yaml:"interval,omitempty"` - Fixture *FixtureConfig `yaml:"fixture,omitempty"` + OatsSchemaVersion int `yaml:"oats-schema-version"` + Name string `yaml:"name"` + Tags []string `yaml:"tags,omitempty"` + Hermetic *bool `yaml:"hermetic,omitempty"` // pointer: distinguish unset vs explicit false + Interval time.Duration `yaml:"interval,omitempty"` + Fixture *FixtureConfig `yaml:"fixture,omitempty"` Seed Seed `yaml:"seed"` Input []Input `yaml:"input,omitempty"` @@ -325,9 +326,9 @@ func Parse(data []byte) (*Case, error) { // Called automatically by Parse; exported for tests that construct Cases // programmatically. func (c *Case) Validate() error { - if c.OatsVersion != SchemaVersion { - return fmt.Errorf("oats: expected %d, got %d (this binary parses v%d only)", - SchemaVersion, c.OatsVersion, SchemaVersion) + if c.OatsSchemaVersion != SchemaVersion { + return fmt.Errorf("unsupported oats-schema-version '%d' required version is '%d'", + c.OatsSchemaVersion, SchemaVersion) } if c.Name == "" { return fmt.Errorf("name: required, non-empty") @@ -349,6 +350,16 @@ func (c *Case) Validate() error { if len(c.Seed.Traces)+len(c.Seed.Logs)+len(c.Seed.Metrics) == 0 { return fmt.Errorf("seed: inline-otlp must declare at least one trace, log, or metric") } + for i, tr := range c.Seed.Traces { + for j, sp := range tr.Spans { + if sp.Duration == "" { + continue + } + if _, err := time.ParseDuration(sp.Duration); err != nil { + return fmt.Errorf("seed.traces[%d].spans[%d].duration: invalid duration %q: %v", i, j, sp.Duration, err) + } + } + } case "": return fmt.Errorf("seed.type: required (app | inline-otlp)") default: diff --git a/casefile/case_test.go b/casefile/case_test.go index 0f16af8b..a06034a8 100644 --- a/casefile/case_test.go +++ b/casefile/case_test.go @@ -8,7 +8,7 @@ import ( func TestParse_AppSeed(t *testing.T) { src := []byte(` -oats: 2 +oats-schema-version: 3 name: rolldice traces have route attribute interval: 250ms seed: @@ -50,7 +50,7 @@ expected: func TestParse_InlineOTLPSeed(t *testing.T) { src := []byte(` -oats: 2 +oats-schema-version: 3 name: gcx returns seeded trace seed: type: inline-otlp @@ -77,7 +77,7 @@ expected: func TestParse_MatchAssertions(t *testing.T) { src := []byte(` -oats: 2 +oats-schema-version: 3 name: structured match seed: type: app @@ -116,7 +116,7 @@ expected: func TestParse_StringListScalarOrList(t *testing.T) { src := []byte(` -oats: 2 +oats-schema-version: 3 name: text assertions seed: type: app @@ -146,7 +146,7 @@ expected: func TestParse_LegacyAttributeMapStillAccepted(t *testing.T) { src := []byte(` -oats: 2 +oats-schema-version: 3 name: legacy attrs seed: type: app @@ -177,7 +177,7 @@ expected: func TestParse_CustomChecks(t *testing.T) { src := []byte(` -oats: 2 +oats-schema-version: 3 name: custom checks seed: type: app @@ -196,7 +196,7 @@ expected: func TestParse_RejectsUnknownFields(t *testing.T) { src := []byte(` -oats: 2 +oats-schema-version: 3 name: typo seed: type: app @@ -216,13 +216,13 @@ expected: func TestValidate_MissingOats(t *testing.T) { c := &Case{Name: "x", Seed: Seed{Type: "app", Compose: "y"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() - if err == nil || !strings.Contains(err.Error(), "oats:") { + if err == nil || !strings.Contains(err.Error(), "oats-schema-version") { t.Errorf("expected oats version error, got %v", err) } } func TestValidate_MissingName(t *testing.T) { - c := &Case{OatsVersion: 2, Seed: Seed{Type: "app", Compose: "y"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + c := &Case{OatsSchemaVersion: 3, Seed: Seed{Type: "app", Compose: "y"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() if err == nil || !strings.Contains(err.Error(), "name:") { t.Errorf("expected name error, got %v", err) @@ -230,7 +230,7 @@ func TestValidate_MissingName(t *testing.T) { } func TestValidate_UnknownSeedType(t *testing.T) { - c := &Case{OatsVersion: 2, Name: "x", Seed: Seed{Type: "weird"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + c := &Case{OatsSchemaVersion: 3, Name: "x", Seed: Seed{Type: "weird"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() if err == nil || !strings.Contains(err.Error(), "seed.type") { t.Errorf("expected seed type error, got %v", err) @@ -238,7 +238,7 @@ func TestValidate_UnknownSeedType(t *testing.T) { } func TestValidate_AppSeedAllowsFixtureManagedBoot(t *testing.T) { - c := &Case{OatsVersion: 2, Name: "x", Seed: Seed{Type: "app"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + c := &Case{OatsSchemaVersion: 3, Name: "x", Seed: Seed{Type: "app"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() if err != nil { t.Errorf("expected app seed without compose to validate, got %v", err) @@ -246,7 +246,7 @@ func TestValidate_AppSeedAllowsFixtureManagedBoot(t *testing.T) { } func TestValidate_InlineOTLPNeedsPayload(t *testing.T) { - c := &Case{OatsVersion: 2, Name: "x", Seed: Seed{Type: "inline-otlp"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + c := &Case{OatsSchemaVersion: 3, Name: "x", Seed: Seed{Type: "inline-otlp"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() if err == nil || !strings.Contains(err.Error(), "at least one trace") { t.Errorf("expected payload error, got %v", err) @@ -254,7 +254,7 @@ func TestValidate_InlineOTLPNeedsPayload(t *testing.T) { } func TestValidate_NoExpectations(t *testing.T) { - c := &Case{OatsVersion: 2, Name: "x", Seed: Seed{Type: "app", Compose: "y"}} + c := &Case{OatsSchemaVersion: 3, Name: "x", Seed: Seed{Type: "app", Compose: "y"}} err := c.Validate() if err == nil || !strings.Contains(err.Error(), "expected:") { t.Errorf("expected 'no expectations' error, got %v", err) @@ -263,10 +263,10 @@ func TestValidate_NoExpectations(t *testing.T) { func TestValidate_CustomCheckOnlyCase(t *testing.T) { c := &Case{ - OatsVersion: 2, - Name: "x", - Seed: Seed{Type: "app"}, - Expected: Expected{Custom: []CustomCheck{{Script: "./verify.sh"}}}, + OatsSchemaVersion: 3, + Name: "x", + Seed: Seed{Type: "app"}, + Expected: Expected{Custom: []CustomCheck{{Script: "./verify.sh"}}}, } if err := c.Validate(); err != nil { t.Fatalf("expected custom-check-only case to validate, got %v", err) @@ -275,7 +275,7 @@ func TestValidate_CustomCheckOnlyCase(t *testing.T) { func TestValidate_RejectsEmptyCustomCheck(t *testing.T) { _, err := Parse([]byte(` -oats: 2 +oats-schema-version: 3 name: bad custom seed: type: app @@ -290,7 +290,7 @@ expected: func TestValidate_RejectsInputWithoutPath(t *testing.T) { _, err := Parse([]byte(` -oats: 2 +oats-schema-version: 3 name: bad input seed: type: app @@ -309,7 +309,7 @@ expected: func TestValidate_RejectsUnknownMatchType(t *testing.T) { _, err := Parse([]byte(` -oats: 2 +oats-schema-version: 3 name: bad match type seed: type: app @@ -328,7 +328,7 @@ expected: func TestValidate_RejectsPresentFalse(t *testing.T) { _, err := Parse([]byte(` -oats: 2 +oats-schema-version: 3 name: bad present seed: type: app @@ -350,7 +350,7 @@ expected: func TestValidate_RejectsInvalidMatchRegexp(t *testing.T) { _, err := Parse([]byte(` -oats: 2 +oats-schema-version: 3 name: bad regexp seed: type: app diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 8872cda9..e0eddb57 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -18,7 +18,7 @@ func writeFile(t *testing.T, dir, rel, body string) { } } -const validCaseYAML = `oats: 2 +const validCaseYAML = `oats-schema-version: 3 name: %s seed: type: app @@ -314,7 +314,7 @@ cases = ["cases/a.yaml", "cases/b.yaml"] version = 2 `) writeFile(t, dir, "cases/a.yaml", ` -oats: 2 +oats-schema-version: 3 name: a seed: { type: app } expected: @@ -323,7 +323,7 @@ expected: absent: true `) writeFile(t, dir, "cases/b.yaml", ` -oats: 2 +oats-schema-version: 3 name: b seed: { type: app } expected: diff --git a/examples/fixtures/cases/compose/rolldice.yaml b/examples/fixtures/cases/compose/rolldice.yaml index 3f800de6..8f9337fc 100644 --- a/examples/fixtures/cases/compose/rolldice.yaml +++ b/examples/fixtures/cases/compose/rolldice.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: compose fixture app smoke seed: type: app diff --git a/examples/fixtures/cases/k3d/rolldice.yaml b/examples/fixtures/cases/k3d/rolldice.yaml index a2cf6409..7aea5d75 100644 --- a/examples/fixtures/cases/k3d/rolldice.yaml +++ b/examples/fixtures/cases/k3d/rolldice.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: k3d fixture app smoke seed: type: app diff --git a/examples/smoke/cases/custom-check.yaml b/examples/smoke/cases/custom-check.yaml index eac350c1..6b3ba892 100644 --- a/examples/smoke/cases/custom-check.yaml +++ b/examples/smoke/cases/custom-check.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: custom check smoke seed: type: app diff --git a/examples/smoke/cases/inline-seed.yaml b/examples/smoke/cases/inline-seed.yaml index 26ca5653..d564bb4b 100644 --- a/examples/smoke/cases/inline-seed.yaml +++ b/examples/smoke/cases/inline-seed.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: inline seed smoke seed: type: inline-otlp diff --git a/examples/smoke/cases/profile.yaml b/examples/smoke/cases/profile.yaml index 257418b9..2f613447 100644 --- a/examples/smoke/cases/profile.yaml +++ b/examples/smoke/cases/profile.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: profile smoke seed: type: app diff --git a/examples/smoke/cases/rolldice.yaml b/examples/smoke/cases/rolldice.yaml index af2102e3..d03d6598 100644 --- a/examples/smoke/cases/rolldice.yaml +++ b/examples/smoke/cases/rolldice.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: rolldice smoke input: - path: /rolldice?rolls=5 diff --git a/internal/cli/integration_test.go b/internal/cli/integration_test.go index 23067b0c..1828a58b 100644 --- a/internal/cli/integration_test.go +++ b/internal/cli/integration_test.go @@ -350,7 +350,7 @@ fixture = "remote-lgtm" type = "remote" endpoint = "REPLACED_AT_RUNTIME" `) - writeFile(t, dir, "cases/inline.yaml", `oats: 2 + writeFile(t, dir, "cases/inline.yaml", `oats-schema-version: 3 name: inline seed end-to-end seed: type: inline-otlp @@ -473,7 +473,7 @@ fixture = "remote-lgtm" type = "remote" endpoint = "http://localhost:4318" `) - writeFile(t, dir, "cases/app.yaml", `oats: 2 + writeFile(t, dir, "cases/app.yaml", `oats-schema-version: 3 name: app seed end-to-end seed: type: app @@ -560,7 +560,7 @@ fixture = "remote-lgtm" type = "remote" endpoint = "http://localhost:4318" `) - writeFile(t, dir, "cases/profile.yaml", `oats: 2 + writeFile(t, dir, "cases/profile.yaml", `oats-schema-version: 3 name: profile query end-to-end seed: type: app diff --git a/migrate/migrate.go b/migrate/migrate.go index 6f0b1268..ffc060b0 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -38,9 +38,9 @@ func ConvertFile(path string) ([]byte, []string, error) { func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Case, []string, error) { var warnings []string c := &casefile.Case{ - OatsVersion: casefile.SchemaVersion, - Name: name, - Interval: def.Interval, + OatsSchemaVersion: casefile.SchemaVersion, + Name: name, + Interval: def.Interval, } selectedMatrix := (*model.Matrix)(nil) diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index e68c7f44..e141fea1 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -90,7 +90,7 @@ func TestConvertFile_RendersYAML(t *testing.T) { fatalf(t, "expected at least one warning for flattened include or fixture migration") } text := string(out) - for _, want := range []string{"oats: 2", "seed:", "input:", "path: /stock", "match_spans:", "match_type: regexp", "key: db.system", "value: h2", "promql: foo"} { + for _, want := range []string{"oats-schema-version: 3", "seed:", "input:", "path: /stock", "match_spans:", "match_type: regexp", "key: db.system", "value: h2", "promql: foo"} { if !strings.Contains(text, want) { fatalf(t, "expected migrated yaml to contain %q:\n%s", want, text) } diff --git a/report/event.go b/report/event.go index 20e290d6..20bb8f2e 100644 --- a/report/event.go +++ b/report/event.go @@ -81,9 +81,9 @@ type Reporter interface { Close() error } -// Verbosity controls how much detail TextReporter renders. NDJSON is -// unaffected — it emits everything except gcx.exec events, which are only -// included at VerboseAll because they can be very large. +// Verbosity controls how much detail the renderers emit. TextReporter uses +// it for pass/cmd/lifecycle chatter; NDJSON applies the same gates to +// case.pass, gcx.exec, and fixture lifecycle events. type Verbosity int const ( diff --git a/runner/cache_integration_test.go b/runner/cache_integration_test.go index fd8893c9..1e551f64 100644 --- a/runner/cache_integration_test.go +++ b/runner/cache_integration_test.go @@ -16,7 +16,7 @@ import ( func cachedRunnerCase(t *testing.T) (*casefile.Case, []byte) { t.Helper() - src := []byte(`oats: 2 + src := []byte(`oats-schema-version: 3 name: cached seed: type: app diff --git a/runner/runner.go b/runner/runner.go index 3e602234..2aa41468 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -151,7 +151,15 @@ func (r *Runner) WithCache(store *cache.Store, ctx CacheContext) *Runner { } func (r *Runner) cacheKey(c *casefile.Case) cache.Key { - yamlBytes, _ := os.ReadFile(c.SourcePath) // best-effort; nil on error + var yamlBytes []byte + if c.SourcePath != "" { + if data, err := os.ReadFile(c.SourcePath); err == nil { + yamlBytes = data + } + } + if len(yamlBytes) == 0 { + yamlBytes = []byte(fmt.Sprintf("case:%s\nsource:%s\n", c.Name, c.SourcePath)) + } return cache.Key{ CaseYAML: yamlBytes, FixtureBytes: r.cacheCtx.FixtureBytes, @@ -504,7 +512,9 @@ func (r *Runner) pollAssert( if err := r.driveInputs(c); err != nil { return []assert.Failure{{Rule: "input", Detail: err.Error()}} } - res, err := r.exec.Execute(ctx, args...) + execCtx, cancel := context.WithTimeout(ctx, r.opts.Timeout) + defer cancel() + res, err := r.exec.Execute(execCtx, args...) if err != nil { return []assert.Failure{{Rule: "exec", Detail: err.Error()}} } @@ -535,6 +545,16 @@ func (r *Runner) pollAssert( if result.OK { return true } + if len(result.LastFailures) == 0 { + r.reporter.Emit(report.Event{ + Type: report.EventAssertFail, + Case: c.Name, + Source: c.SourcePath, + Msg: "assertion polling stopped before any failure details were captured", + Cmd: cmdStr, + }) + return false + } for _, f := range result.LastFailures { r.reporter.Emit(report.Event{ Type: report.EventAssertFail, @@ -558,7 +578,7 @@ func (r *Runner) runTrace(ctx context.Context, c *casefile.Case, a *casefile.Tra if len(a.MatchSpans) > 0 { return r.runTraceStructured(ctx, c, a) } - args := signalcmd.Traces(*a, r.opts.Timeout) + args := signalcmd.Traces(*a, 0) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { return evalCommonText(stdout, a.AssertionCommon) }) @@ -569,9 +589,11 @@ func (r *Runner) runTraceStructured(ctx context.Context, c *casefile.Case, a *ca if err := r.driveInputs(c); err != nil { return []assert.Failure{{Rule: "input", Detail: err.Error()}} } - searchArgs := signalcmd.Traces(*a, r.opts.Timeout) + searchArgs := signalcmd.Traces(*a, 0) searchCmd := signalcmd.Render(searchArgs) - searchRes, err := r.exec.Execute(ctx, searchArgs...) + execCtx, cancel := context.WithTimeout(ctx, r.opts.Timeout) + defer cancel() + searchRes, err := r.exec.Execute(execCtx, searchArgs...) if err != nil { return []assert.Failure{{Rule: "exec", Detail: err.Error()}} } @@ -591,7 +613,7 @@ func (r *Runner) runTraceStructured(ctx context.Context, c *casefile.Case, a *ca if result.OK { return true } - cmdStr := signalcmd.Render(signalcmd.Traces(*a, r.opts.Timeout)) + cmdStr := signalcmd.Render(signalcmd.Traces(*a, 0)) for _, f := range result.LastFailures { r.reporter.Emit(report.Event{ Type: report.EventAssertFail, @@ -621,8 +643,10 @@ func (r *Runner) fetchTraceRows(ctx context.Context, c *casefile.Case, searchStd } var rows []assert.Row for _, traceID := range traceIDs { - args := signalcmd.TraceGet(traceID, r.opts.Timeout) - res, err := r.exec.Execute(ctx, args...) + args := signalcmd.TraceGet(traceID, 0) + execCtx, cancel := context.WithTimeout(ctx, r.opts.Timeout) + res, err := r.exec.Execute(execCtx, args...) + cancel() if err != nil { return nil, count, fmt.Errorf("trace %s fetch: %w", traceID, err) } @@ -644,7 +668,7 @@ func (r *Runner) fetchTraceRows(ctx context.Context, c *casefile.Case, searchStd } func (r *Runner) runLog(ctx context.Context, c *casefile.Case, a *casefile.LogAssertion) bool { - args := signalcmd.Logs(*a, r.opts.Timeout) + args := signalcmd.Logs(*a, 0) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { if len(a.Match) == 0 { return evalCommonText(stdout, a.AssertionCommon) @@ -655,7 +679,7 @@ func (r *Runner) runLog(ctx context.Context, c *casefile.Case, a *casefile.LogAs } func (r *Runner) runMetric(ctx context.Context, c *casefile.Case, a *casefile.MetricAssertion) bool { - args := signalcmd.Metrics(*a, r.opts.Timeout) + args := signalcmd.Metrics(*a, 0) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { if a.Value == "" && len(a.Match) == 0 { return evalCommonText(stdout, a.AssertionCommon) @@ -674,7 +698,7 @@ func (r *Runner) runMetric(ctx context.Context, c *casefile.Case, a *casefile.Me } func (r *Runner) runProfile(ctx context.Context, c *casefile.Case, a *casefile.ProfileAssertion) bool { - args := signalcmd.Profiles(*a, r.opts.Timeout) + args := signalcmd.Profiles(*a, 0) return r.pollAssert(ctx, c, args, a.Absent, func(stdout, _ string, _ int) []assert.Failure { if len(a.Match) == 0 { return evalCommonText(stdout, a.AssertionCommon) diff --git a/runner/runner_test.go b/runner/runner_test.go index b405cb2d..1cb88796 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -59,7 +59,7 @@ func mustParse(t *testing.T, src string) *casefile.Case { } const tracesCase = ` -oats: 2 +oats-schema-version: 3 name: traces pass seed: type: app @@ -94,7 +94,7 @@ func TestRunCase_TracesPass(t *testing.T) { } const containsMissingCase = ` -oats: 2 +oats-schema-version: 3 name: traces fail seed: type: app @@ -125,7 +125,7 @@ func TestRunCase_TracesFail_ContainsMissing(t *testing.T) { } const metricsValueCase = ` -oats: 2 +oats-schema-version: 3 name: metric value seed: type: app @@ -172,7 +172,7 @@ func TestRunCase_LogsStructuredMatchPass(t *testing.T) { r, buf := newRunner(t, exec, Options{Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: logs structured match seed: type: app @@ -205,7 +205,7 @@ func TestRunCase_MetricsStructuredMatchPass(t *testing.T) { r, _ := newRunner(t, exec, Options{Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: metrics structured match seed: type: app @@ -245,7 +245,7 @@ func TestRunCase_DrivesInputRequests(t *testing.T) { r.endpoint.AppPort = port c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: traces pass with input seed: type: app @@ -273,7 +273,7 @@ expected: func TestRunCase_InlineOTLPSeedRequiresEndpoint(t *testing.T) { c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: inline seed seed: type: inline-otlp @@ -312,7 +312,7 @@ func TestRunCase_CustomCheckScriptPath(t *testing.T) { } c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: custom check path seed: type: app @@ -337,7 +337,7 @@ expected: func TestRunCase_CustomCheckInlineScript(t *testing.T) { c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: custom check inline seed: type: app @@ -363,7 +363,7 @@ expected: func TestRunCase_CustomCheckFailureSurfaced(t *testing.T) { c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: custom check fail seed: type: app @@ -404,7 +404,7 @@ func TestRunCase_ComposeLogsPass(t *testing.T) { t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: compose logs pass seed: type: app @@ -448,7 +448,7 @@ func TestRunCase_ComposeLogsMissingSurfaced(t *testing.T) { t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: compose logs missing seed: type: app @@ -484,7 +484,7 @@ expected: func TestRunCase_ComposeLogsRequireComposeFixture(t *testing.T) { c := mustParse(t, ` -oats: 2 +oats-schema-version: 3 name: compose logs wrong fixture seed: type: app diff --git a/testhelpers/compose/compose.go b/testhelpers/compose/compose.go index 3fea4863..749c5487 100644 --- a/testhelpers/compose/compose.go +++ b/testhelpers/compose/compose.go @@ -98,7 +98,7 @@ func (c *Compose) runDocker(cc command) error { } wg.Wait() } else if cc.background { - slog.Info("Running", "command", cmd.String(), "dir", strings.Join(c.Paths, ",")) + slog.Info("Running", "command", cmd.String(), "compose_files", c.Paths) stdout, _ := cmd.StdoutPipe() cmd.Stderr = cmd.Stdout go func() { @@ -116,7 +116,7 @@ func (c *Compose) runDocker(cc command) error { } go func() { _ = cmd.Wait() }() } else { - slog.Info("Running", "command", cmd.String(), "dir", strings.Join(c.Paths, ",")) + slog.Info("Running", "command", cmd.String(), "compose_files", c.Paths) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err := cmd.Run() diff --git a/tests/e2e/cases/assert/absent-fail/files/oats.yaml b/tests/e2e/cases/assert/absent-fail/files/oats.yaml index c8ccaa19..754ac696 100644 --- a/tests/e2e/cases/assert/absent-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/absent-fail/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: absent-fail fixture: type: remote diff --git a/tests/e2e/cases/assert/absent/files/oats.yaml b/tests/e2e/cases/assert/absent/files/oats.yaml index 502856dd..4eb62d73 100644 --- a/tests/e2e/cases/assert/absent/files/oats.yaml +++ b/tests/e2e/cases/assert/absent/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: absent fixture: type: remote diff --git a/tests/e2e/cases/assert/contains-fail/files/oats.yaml b/tests/e2e/cases/assert/contains-fail/files/oats.yaml index a8722819..3efd76b8 100644 --- a/tests/e2e/cases/assert/contains-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/contains-fail/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: contains-fail fixture: type: remote diff --git a/tests/e2e/cases/assert/contains/files/oats.yaml b/tests/e2e/cases/assert/contains/files/oats.yaml index 613d1412..600e1615 100644 --- a/tests/e2e/cases/assert/contains/files/oats.yaml +++ b/tests/e2e/cases/assert/contains/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: contains fixture: type: remote diff --git a/tests/e2e/cases/assert/count/files/oats.yaml b/tests/e2e/cases/assert/count/files/oats.yaml index 4332f937..0bcf6370 100644 --- a/tests/e2e/cases/assert/count/files/oats.yaml +++ b/tests/e2e/cases/assert/count/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: count fixture: type: remote diff --git a/tests/e2e/cases/assert/match-fail/files/oats.yaml b/tests/e2e/cases/assert/match-fail/files/oats.yaml index d01489bb..a82c71c7 100644 --- a/tests/e2e/cases/assert/match-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/match-fail/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: match-fail fixture: type: remote diff --git a/tests/e2e/cases/assert/match-regexp/files/oats.yaml b/tests/e2e/cases/assert/match-regexp/files/oats.yaml index f287a5da..462b2b6a 100644 --- a/tests/e2e/cases/assert/match-regexp/files/oats.yaml +++ b/tests/e2e/cases/assert/match-regexp/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: match-regexp fixture: type: remote diff --git a/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml b/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml index f9d0fa3a..221ab8a3 100644 --- a/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: match-spans-fail fixture: type: remote diff --git a/tests/e2e/cases/assert/not-contains/files/oats.yaml b/tests/e2e/cases/assert/not-contains/files/oats.yaml index 0d47121a..514b00e3 100644 --- a/tests/e2e/cases/assert/not-contains/files/oats.yaml +++ b/tests/e2e/cases/assert/not-contains/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: not-contains fixture: type: remote diff --git a/tests/e2e/cases/assert/regex-fail/files/oats.yaml b/tests/e2e/cases/assert/regex-fail/files/oats.yaml index 4580fbb2..b43ed2ac 100644 --- a/tests/e2e/cases/assert/regex-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/regex-fail/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: regex-fail fixture: type: remote diff --git a/tests/e2e/cases/assert/regex/files/oats.yaml b/tests/e2e/cases/assert/regex/files/oats.yaml index 1946fc66..fb055975 100644 --- a/tests/e2e/cases/assert/regex/files/oats.yaml +++ b/tests/e2e/cases/assert/regex/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: regex fixture: type: remote diff --git a/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml b/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml index 12ce8c48..25ebc4d5 100644 --- a/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml +++ b/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: custom-check-fail fixture: type: remote diff --git a/tests/e2e/cases/custom/custom-check/files/oats.yaml b/tests/e2e/cases/custom/custom-check/files/oats.yaml index 9250609c..5656a8cc 100644 --- a/tests/e2e/cases/custom/custom-check/files/oats.yaml +++ b/tests/e2e/cases/custom/custom-check/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: custom-check fixture: type: remote diff --git a/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml b/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml index f4a85a75..d17a4e1f 100644 --- a/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml +++ b/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: compose-logs-fail fixture: type: compose diff --git a/tests/e2e/cases/fixture/compose-logs/files/oats.yaml b/tests/e2e/cases/fixture/compose-logs/files/oats.yaml index 61c541ee..e9bc4e7a 100644 --- a/tests/e2e/cases/fixture/compose-logs/files/oats.yaml +++ b/tests/e2e/cases/fixture/compose-logs/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: compose-logs fixture: type: compose diff --git a/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml b/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml index a43daaf0..6796f219 100644 --- a/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml +++ b/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: k3d-fail fixture: type: k3d diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml index f2863634..424b1e25 100644 --- a/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml +++ b/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: k3d-smoke fixture: type: k3d diff --git a/tests/e2e/cases/fixture/remote-basic/files/docker-compose.remote.yml b/tests/e2e/cases/fixture/remote-basic/files/docker-compose.remote.yml index c024466b..bc58ac88 100644 --- a/tests/e2e/cases/fixture/remote-basic/files/docker-compose.remote.yml +++ b/tests/e2e/cases/fixture/remote-basic/files/docker-compose.remote.yml @@ -2,5 +2,5 @@ services: lgtm: image: docker.io/grafana/otel-lgtm:latest ports: - - "3001:3000" - - "4319:4318" + - "127.0.0.1::3000" + - "127.0.0.1::4318" diff --git a/tests/e2e/cases/fixture/remote-basic/files/oats.yaml b/tests/e2e/cases/fixture/remote-basic/files/oats.yaml index d982b537..f4486137 100644 --- a/tests/e2e/cases/fixture/remote-basic/files/oats.yaml +++ b/tests/e2e/cases/fixture/remote-basic/files/oats.yaml @@ -1,4 +1,4 @@ -oats: 2 +oats-schema-version: 3 name: remote-basic fixture: type: remote diff --git a/tests/e2e/cases/fixture/remote-basic/test.yaml b/tests/e2e/cases/fixture/remote-basic/test.yaml index e07957b9..6362d876 100644 --- a/tests/e2e/cases/fixture/remote-basic/test.yaml +++ b/tests/e2e/cases/fixture/remote-basic/test.yaml @@ -9,8 +9,10 @@ run: project="oats-remote-basic-$$" trap 'docker compose -p "$project" -f "$workdir/files/docker-compose.remote.yml" down -v --remove-orphans >/dev/null 2>&1 || true' EXIT docker compose -p "$project" -f "$workdir/files/docker-compose.remote.yml" up -d >/dev/null + grafana_port="$(docker compose -p "$project" -f "$workdir/files/docker-compose.remote.yml" port lgtm 3000 | awk -F: '{print $NF}')" + otlp_port="$(docker compose -p "$project" -f "$workdir/files/docker-compose.remote.yml" port lgtm 4318 | awk -F: '{print $NF}')" for i in $(seq 1 60); do - if curl -fsS http://localhost:3001/api/health >/dev/null; then + if curl -fsS "http://127.0.0.1:${grafana_port}/api/health" >/dev/null; then break fi sleep 2 @@ -20,7 +22,7 @@ run: contexts: remote: grafana: - server: http://localhost:3001 + server: http://127.0.0.1:${grafana_port} user: admin password: admin org-id: 1 @@ -31,6 +33,26 @@ run: tempo: tempo pyroscope: pyroscope CFG + cat > "$workdir/files/oats.yaml" < len(addr) || addr[end+1] != ':' { + return "", "", fmt.Errorf("invalid address %q", addr) + } + return addr[1:end], addr[end+2:], nil + } + idx := strings.LastIndex(addr, ":") + if idx < 0 { + return "", "", fmt.Errorf("invalid address %q", addr) + } + return addr[:idx], addr[idx+1:], nil +} + func teardownSharedEnv(env *sharedEnv) error { var errs []string if env.ComposeFile != "" { From 96201f1b8136b4a428dea58c729d2a38131de7fc Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 11:28:02 +0200 Subject: [PATCH 071/124] test: clarify wait tight-deadline coverage Signed-off-by: Gregor Zeitlinger --- wait/wait_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wait/wait_test.go b/wait/wait_test.go index 385c1605..1703a8cc 100644 --- a/wait/wait_test.go +++ b/wait/wait_test.go @@ -47,9 +47,9 @@ func TestUntil_FailsAtDeadline(t *testing.T) { } } -func TestUntil_RunsAtLeastOnceEvenWithZeroTimeout(t *testing.T) { - // Zero timeout gets the default — we just want to verify the asserter - // is invoked, not that no poll happens. The contract is "at least once." +func TestUntil_RunsAtLeastOnceEvenWithTightDeadline(t *testing.T) { + // A tight deadline should still invoke the asserter at least once. The + // contract is "at least once," not "zero polls under short timeouts." called := false r := Until[string](context.Background(), Options{Timeout: 5 * time.Millisecond}, func() []string { called = true From e5d0c508562970fe72a11893decdf26a714524aa Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 12:51:56 +0200 Subject: [PATCH 072/124] test: make e2e fixture startup lazy and ready Signed-off-by: Gregor Zeitlinger --- internal/cli/main.go | 36 ++++++++ .../cases/custom/custom-check/files/verify.sh | 6 +- tests/e2e/e2e_test.go | 91 +++++++++++++++---- 3 files changed, 112 insertions(+), 21 deletions(-) diff --git a/internal/cli/main.go b/internal/cli/main.go index 664d056b..5c5e31c0 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -23,6 +23,7 @@ import ( "encoding/json" "flag" "fmt" + "net/http" "os" "os/exec" "os/signal" @@ -160,6 +161,13 @@ func Run() int { fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) return 2 } + if err := waitForFixtureReady(plan); err != nil { + if fix != nil { + _ = closeFixture(rep, plan, fix) + } + fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) + return 2 + } emitFixtureReady(rep, plan, fixtureStart) ep, err := resolveEndpoint(plan, *gcxContextOverride, *appHost, *appPort, *otlpHTTP) if err != nil { @@ -588,6 +596,34 @@ func waitForGrafanaTokenImpl(plan discovery.Plan) (string, error) { return "", fmt.Errorf("timed out waiting for Grafana service-account token") } +func waitForFixtureReady(plan discovery.Plan) error { + switch plan.Fixture.Type { + case "compose", "k3d": + if err := waitForHTTP(grafanaURL()+"/api/health", 2*time.Minute); err != nil { + return fmt.Errorf("wait for grafana: %w", err) + } + if err := waitForHTTP("http://localhost:4318", 2*time.Minute); err != nil { + return fmt.Errorf("wait for otlp-http: %w", err) + } + } + return nil +} + +func waitForHTTP(url string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + resp, err := http.Get(url) //nolint:gosec + if err == nil { + _ = resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 500 { + return nil + } + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("timed out waiting for %s", url) +} + func readComposeGrafanaToken(plan discovery.Plan) (string, error) { files, _, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) if err != nil { diff --git a/tests/e2e/cases/custom/custom-check/files/verify.sh b/tests/e2e/cases/custom/custom-check/files/verify.sh index e2c140ff..60a30c05 100644 --- a/tests/e2e/cases/custom/custom-check/files/verify.sh +++ b/tests/e2e/cases/custom/custom-check/files/verify.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash set -euo pipefail -python3 - <<'PY' -import os, urllib.request -urllib.request.urlopen(os.environ['OATS_GRAFANA_URL'] + '/api/health').read() -PY +grafana_url="$(awk '/server:/ {print $2; exit}' "${GCX_CONFIG:?}")" +curl -fsS "${grafana_url}/api/health" >/dev/null echo "custom check ok" diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 042c5a12..6d85c318 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -10,6 +10,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "testing" "time" @@ -64,7 +65,11 @@ type sharedEnv struct { OATS string } -var shared sharedEnv +var ( + shared sharedEnv + sharedOnce sync.Once + sharedErr error +) func TestMain(m *testing.M) { root, err := findRepoRoot() @@ -74,7 +79,7 @@ func TestMain(m *testing.M) { } shared.RepoRoot = root if runtime.GOOS != "windows" { - if err := setupSharedEnv(&shared); err != nil { + if err := prepareLocalTools(&shared); err != nil { fmt.Fprintf(os.Stderr, "e2e setup failed: %v\n", err) _ = teardownSharedEnv(&shared) os.Exit(1) @@ -160,13 +165,11 @@ func setupSharedEnv(env *sharedEnv) error { if _, err := exec.LookPath("docker"); err != nil { return err } - tmp, err := os.MkdirTemp("", "oats-e2e-") - if err != nil { + if err := prepareLocalTools(env); err != nil { return err } - env.TempDir = tmp env.Project = fmt.Sprintf("oatse2e%d", os.Getpid()) - env.ComposeFile = filepath.Join(tmp, "docker-compose.yml") + env.ComposeFile = filepath.Join(env.TempDir, "docker-compose.yml") const composeBody = `services: lgtm: image: docker.io/grafana/otel-lgtm:latest @@ -177,17 +180,6 @@ func setupSharedEnv(env *sharedEnv) error { if err := os.WriteFile(env.ComposeFile, []byte(composeBody), 0o644); err != nil { return err } - binDir := filepath.Join(tmp, "bin") - build := exec.Command("bash", "-lc", fmt.Sprintf("./scripts/build-local-tools.sh %q", binDir)) - build.Dir = env.RepoRoot - build.Stdout = os.Stdout - build.Stderr = os.Stderr - if err := build.Run(); err != nil { - return fmt.Errorf("build local tools: %w", err) - } - env.OATS = filepath.Join(binDir, "oats") - env.GCX = filepath.Join(binDir, "gcx") - env.GCXConfig = filepath.Join(tmp, "gcx.yaml") up := exec.Command("docker", "compose", "-p", env.Project, "-f", env.ComposeFile, "up", "-d") up.Stdout = os.Stdout up.Stderr = os.Stderr @@ -210,9 +202,71 @@ func setupSharedEnv(env *sharedEnv) error { if err := waitForHTTP(env.GrafanaURL+"/api/health", 2*time.Minute); err != nil { return err } + if err := waitForHTTP(env.RemoteOTLPHTTP, 2*time.Minute); err != nil { + return err + } return nil } +func prepareLocalTools(env *sharedEnv) error { + if env.TempDir != "" && env.OATS != "" && env.GCX != "" && env.GCXConfig != "" { + return nil + } + tmp, err := os.MkdirTemp("", "oats-e2e-") + if err != nil { + return err + } + env.TempDir = tmp + binDir := filepath.Join(tmp, "bin") + build := exec.Command("bash", "-lc", fmt.Sprintf("./scripts/build-local-tools.sh %q", binDir)) + build.Dir = env.RepoRoot + build.Stdout = os.Stdout + build.Stderr = os.Stderr + if err := build.Run(); err != nil { + return fmt.Errorf("build local tools: %w", err) + } + env.OATS = filepath.Join(binDir, "oats") + env.GCX = filepath.Join(binDir, "gcx") + env.GCXConfig = filepath.Join(tmp, "gcx.yaml") + return nil +} + +func ensureSharedEnv(t *testing.T) { + t.Helper() + sharedOnce.Do(func() { + sharedErr = setupSharedEnv(&shared) + }) + if sharedErr != nil { + t.Fatalf("e2e setup failed: %v", sharedErr) + } +} + +func caseNeedsSharedEnv(t *testing.T, dir string) bool { + t.Helper() + matches := false + err := filepath.WalkDir(dir, func(path string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if d.IsDir() { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + if strings.Contains(string(data), "{{REMOTE_OTLP_HTTP}}") { + matches = true + return filepath.SkipAll + } + return nil + }) + if err != nil && !errors.Is(err, filepath.SkipAll) { + t.Fatalf("scan %s: %v", dir, err) + } + return matches +} + func writeGCXConfig(path, grafanaURL string) error { cfg := map[string]any{ "current-context": "local", @@ -366,6 +420,9 @@ func runCase(t *testing.T, root, dir string) { t.Helper() spec := loadCaseFile(t, filepath.Join(dir, "test.yaml")) + if caseNeedsSharedEnv(t, dir) { + ensureSharedEnv(t) + } tmp := t.TempDir() filesDir := filepath.Join(tmp, "files") if err := os.MkdirAll(filesDir, 0o755); err != nil { From 6ba2d8c0625576f6152faad54c435a7082323326 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 13:03:40 +0200 Subject: [PATCH 073/124] test: use real lgtm for k3d smoke Signed-off-by: Gregor Zeitlinger --- testhelpers/kubernetes/kubernetes.go | 8 ++++++++ testhelpers/kubernetes/kubernetes_test.go | 2 ++ .../e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml | 9 +++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/testhelpers/kubernetes/kubernetes.go b/testhelpers/kubernetes/kubernetes.go index 0184033d..2b57d727 100644 --- a/testhelpers/kubernetes/kubernetes.go +++ b/testhelpers/kubernetes/kubernetes.go @@ -123,6 +123,14 @@ func start(model *Kubernetes, ports remote.PortsConfig, testName string, run fun if err != nil { return err } + err = portForward(3000, 3000) + if err != nil { + return err + } + err = portForward(4318, 4318) + if err != nil { + return err + } err = portForward(ports.PrometheusHTTPPort, 9090) if err != nil { return err diff --git a/testhelpers/kubernetes/kubernetes_test.go b/testhelpers/kubernetes/kubernetes_test.go index 596bd6f9..a6a8f9c7 100644 --- a/testhelpers/kubernetes/kubernetes_test.go +++ b/testhelpers/kubernetes/kubernetes_test.go @@ -74,6 +74,8 @@ func TestStart_DefaultDockerContextAndCommandSequence(t *testing.T) { "fg: kubectl wait --timeout=5m --for=condition=available deployment/lgtm", "bg: kubectl port-forward service/dice 18080:8080", "bg: kubectl port-forward service/lgtm 13100:3100", + "bg: kubectl port-forward service/lgtm 3000:3000", + "bg: kubectl port-forward service/lgtm 4318:4318", "bg: kubectl port-forward service/lgtm 19090:9090", "bg: kubectl port-forward service/lgtm 13200:3200", "bg: kubectl port-forward service/lgtm 14040:4040", diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml index 0f321d64..67f8f02a 100644 --- a/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml +++ b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml @@ -14,8 +14,7 @@ spec: spec: containers: - name: lgtm - image: docker.io/library/alpine:3.22 - command: ["sh", "-c", "sleep infinity"] + image: docker.io/grafana/otel-lgtm:latest --- apiVersion: v1 kind: Service @@ -25,6 +24,12 @@ spec: selector: app: lgtm ports: + - name: grafana + port: 3000 + targetPort: 3000 + - name: otlp-http + port: 4318 + targetPort: 4318 - name: loki port: 3100 targetPort: 3100 From d94ec5a4e60ca62609bcc5df23be4c7fd6c7c721 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 13:14:11 +0200 Subject: [PATCH 074/124] test: gate k3d smoke on grafana readiness Signed-off-by: Gregor Zeitlinger --- tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml index 67f8f02a..45da0d14 100644 --- a/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml +++ b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml @@ -15,6 +15,14 @@ spec: containers: - name: lgtm image: docker.io/grafana/otel-lgtm:latest + readinessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 2 + periodSeconds: 2 + timeoutSeconds: 1 + failureThreshold: 30 --- apiVersion: v1 kind: Service From 0c94f44bc6099858387e9c2e18c45d73024f1b66 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 14:24:38 +0200 Subject: [PATCH 075/124] test: align k3d smoke readiness probe Signed-off-by: Gregor Zeitlinger --- .../cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml index 45da0d14..5c088c95 100644 --- a/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml +++ b/tests/e2e/cases/fixture/k3d-smoke/files/k8s/lgtm-pod.yaml @@ -16,13 +16,10 @@ spec: - name: lgtm image: docker.io/grafana/otel-lgtm:latest readinessProbe: - httpGet: - path: /api/health - port: 3000 - initialDelaySeconds: 2 - periodSeconds: 2 - timeoutSeconds: 1 - failureThreshold: 30 + exec: + command: + - cat + - /tmp/ready --- apiVersion: v1 kind: Service From 828c58cbae255ea271f7b1b3d63fee855be3b72f Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 14:41:35 +0200 Subject: [PATCH 076/124] docs: fold current cli docs into readme Signed-off-by: Gregor Zeitlinger --- AGENTS.md | 2 +- CURRENT.md | 279 +---------------------------------- README.md | 100 ++++++++++++- UPGRADING.md | 4 +- internal/cli/main.go | 21 ++- scripts/build-local-tools.sh | 4 +- 6 files changed, 116 insertions(+), 294 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8ce1783e..c27e227c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,7 @@ EditorConfig rules live in `.editorconfig`. ### Test Case Schema -Current user-facing syntax is documented in `CURRENT.md`. Legacy yaml parsing +Current user-facing syntax is documented in `README.md`. Legacy yaml parsing still exists in-package for migration support, but the repo's CLI surface is the current `oats.toml` + case-yaml flow. diff --git a/CURRENT.md b/CURRENT.md index 1c3f0b27..d4c35ac5 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -1,278 +1 @@ -# Current OATS CLI - -This document covers the current gcx-driven OATS CLI, which replaces the bespoke TraceQL / -PromQL / LogQL HTTP query infrastructure with the [gcx](https://github.com/grafana/gcx) -CLI. The current `oats` binary runs the gcx-driven implementation. See -the internal design doc (grafana/internal-docs#14) -for the full design. - -## Quick start - -```sh -# Build oats plus the gcx binary it shells out to -./scripts/build-local-tools.sh - -# Print what would run, do not execute -bin/oats --config oats.toml --list - -# Run against an already-running stack -bin/oats --config oats.toml --gcx-context my-lgtm - -# Disable cache for this run -bin/oats --config oats.toml --no-cache - -# Verbose output -bin/oats -v # adds per-case PASS lines -bin/oats -v=2 # adds the gcx command behind each assertion -bin/oats -v=3 # adds fixture lifecycle and full gcx stdout - -# Machine-readable -bin/oats --format ndjson > events.jsonl -``` - -The repo includes a small reference config under `examples/smoke/` showing: - -- an app-backed case with `input` -- an inline-OTLP case -- a `custom-checks` case -- a profile assertion case - -For richer fixture examples, `examples/fixtures/` shows: - -- multi-file compose fixtures with env passthrough -- k3d fixture config with app build/import fields - -## Current implemented scope - -Today this branch already includes: - -- gcx-driven traces / logs / metrics / profiles querying -- collector-style structural assertions (`match_spans` for traces, `match` elsewhere) with `match_type: strict | regexp` -- app-backed and inline-OTLP seed modes -- input-driving HTTP requests -- custom checks -- remote / compose / k3d fixture flows -- best-effort legacy migration with runnable migrated outputs for core cases -- text and NDJSON reporting, including fixture lifecycle events at verbose levels - -## Architecture - -```text -discovery → seed → engine → assert → report - ↑ - fixture - ↑ - wait (polling) - ↑ - cache (skip-when-unchanged) -``` - -| Package | Responsibility | -|------------|---------------| -| `discovery` | Parse `oats.toml`, expand case globs, apply filters, and derive | -| | case-local fixtures when a suite omits one. | -| `casefile` | Parse and validate one current-format case yaml file. | -| `seed` | Push inline-OTLP payloads at an OTLP/HTTP endpoint. | -| `engine` | Execute a gcx command, capture stdout/stderr/exit. | -| `signalcmd` | Translate a `casefile` assertion into gcx args. | -| `assert` | The expectation vocabulary: `contains`, `not_contains`, `regex`, `value`, `count`, `absent`. | -| `wait` | `Until` / `While` polling primitives (replaces gomega.Eventually). | -| `report` | Compact-text and NDJSON renderers driven by an Event stream. | -| `cache` | Skip-when-unchanged store keyed by | -| | `(case yaml + fixture + gcx version + oats version)`. | -| `runner` | Orchestrates a suite: seed → poll-and-assert → report, with optional cache. | -| `internal/cli` | The gcx-driven CLI implementation package used by the root `oats` binary. | - -## Consumer-shape notes - -For simple consumer repos, keep `oats.toml` thin: a suite usually only needs -`cases = ["..."]`. Case-local `fixture:` blocks now cover the common -one-case-per-suite path. Shared root-level `[fixture.*]` blocks are still useful -when many suites intentionally reuse the same fixture or when one case is run -against multiple fixtures. - -Local LGTM compose boot plus Grafana auth bootstrap are now owned by OATS -itself: consumer repos should not need their own shared -`docker-compose.lgtm.yml` or a custom `gcx-wrapper.sh` just to talk to a local -LGTM stack. - -## `oats.toml` shape - -```toml -[meta] -version = 2 - -[[suite]] -cases = ["examples/nodejs/oats.yaml"] - -# Optional when many suites share one fixture: -# [[suite]] -# name = "lgtm-examples" -# cases = ["examples/*/oats.yaml"] -# fixture = "lgtm-shared" -# tags = ["traces", "metrics", "logs"] -# -# [fixture.lgtm-shared] -# type = "compose" -# template = "lgtm" - -[cache] -ttl_days = 7 # default -``` - -## Case yaml shape - -```yaml -oats-schema-version: 3 -name: rolldice traces have route attribute - -fixture: - type: compose - template: lgtm - compose_file: docker-compose.oats.yml - -seed: - type: app - compose: docker-compose.app.yml # optional legacy shorthand; suite fixture usually owns boot -input: - - path: /rolldice?rolls=5 - -expected: - traces: - - traceql: '{ span.http.route = "/rolldice" }' - match_spans: - - name: "GET /rolldice" - metrics: - - promql: 'dice_lib_rolls_counter_total{service_name="dice-server"}' - value: '>= 0' - match: - - attributes: - - key: service_name - value: dice-server - logs: - - logql: '{service_name="dice-server"} |~ `Received request`' - match: - - name: "Received request to roll dice" - attributes: - - key: service_name - value: dice-server - custom-checks: - - script: ./verify.sh -``` - -For `seed.type: app`, the application is normally started by the suite fixture in -`oats.toml`. `seed.compose` is still accepted as a migration/compatibility hint, -but the runner does not require it when a fixture already provides the app. - -Inline-OTLP seed (no example app required): - -```yaml -oats-schema-version: 3 -name: gcx returns seeded trace - -seed: - type: inline-otlp - traces: - - service: gcx-e2e-seed - spans: - - name: seed-operation - -expected: - traces: - - traceql: '{ resource.service.name = "gcx-e2e-seed" }' - match_spans: - - name: seed-operation - attributes: - - key: service.name - value: gcx-e2e-seed -``` - -### Assertion keys - -Per signal under `expected.[]`: - -| Key | Meaning | -|-----|---------| -| `traceql` / `promql` / `logql` / `query` | The query string. | -| `contains` | Substrings that must appear in gcx stdout. Accepts a string or list of strings. | -| `not_contains` | Substrings that must not appear. Accepts a string or list of strings. | -| `regex` | Patterns that must match. Accepts a string or list of strings. | -| `match_spans` | Trace-only structural assertions over returned spans using collector-style `match_type: strict | regexp`. | -| `match` | Structural assertions for logs / metrics / profiles using collector-style `match_type: strict | regexp`. | -| `value` | Metrics only — numeric comparison (`>= 0`, `== 42`). | -| `count` | Comparison against the number of result rows. | -| `absent` | If true, the query must return zero rows. | - -Per custom check under `expected.custom-checks[]`: - -| Key | Meaning | -|-----|---------| -| `script` | Either an executable path resolved relative to the case yaml, or an inline shell script block. | - -When a case declares `input`, the runner makes those HTTP requests before each -assertion poll, mirroring OATS v1's “drive the app until telemetry appears” -behavior. For remote fixtures, point those requests at a running app with -`--app-host` and `--app-port` (defaults: `localhost:8080`). - -`match_spans` (for traces) and `match` (for logs / metrics / profiles) -default to `match_type: strict`. Attributes follow collector-style -`[{ key, value? }]` entries; omitting `value` means “the key must be present”. -Text assertions keep list semantics internally, but author-facing YAML may use -either a scalar string or a list of strings. - -```yaml -match: - - name: seed-operation - attributes: - - key: service.name - value: gcx-e2e-seed - - key: trace_id - - match_type: regexp - attributes: - - key: http.route - value: "^/roll.*$" -``` - -## What's not here yet - -- Parallel cases within a suite. -- k3d pool warmup / reuse. -- Full-fidelity `oats migrate` for fixture/input semantics. A best-effort - `oats --migrate ` now converts expectation blocks into the - collector-style `match` schema and prints warnings for dropped/unsupported - fields (multi-entry matrix cases still expand manually). -- Hermeticity static-check (runtime check applies). - -## Migrating from v1 - -For the legacy → current migration story see the OATS implementation plan in -the internal design doc (grafana/internal-docs#14). -Today a best-effort converter exists: - -```bash -oats --migrate path/to/oats.yaml > migrated.yaml -``` - -It converts legacy `equals` / `regexp` / `attributes` / -`attribute-regexp` assertions into structural matcher entries -(`match_spans:` for traces, `match:` elsewhere) and prints warnings for -fields that still need manual follow-up. For richer legacy fixtures, the -warnings now include ready-to-paste `oats.toml` fixture snippets for -multi-file compose/env and kubernetes→k3d mappings. Single-entry legacy -`matrix:` cases are flattened automatically; multi-entry matrix cases emit -fixture-expansion hints for manual splitting. The v1 binary (`oats`) and the -current binary (`oats`) still reads a different file shape than the legacy YAML runner. - -## Verbosity contract - -| Flag | Adds to stdout | -|------|----------------| -| (default) | Failures + final summary only. | -| `-v` | One line per passing case. | -| `-v=2` | The gcx invocation behind each assertion. | -| `-v=3` | Fixture lifecycle, full gcx stdout, per-phase timing. | - -`--format ndjson` emits one JSON object per event regardless of verbosity. -Pass and fixture-lifecycle events are filtered at the same verbosity -thresholds. Under GitHub Actions, failures also emit `::error file=...,line=...::` -annotations for inline PR-diff visibility. +# Current OATS docs now live in [README.md](README.md) diff --git a/README.md b/README.md index 55ac19cf..cfe4a163 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,8 @@ OATs is a declarative acceptance-test framework for OpenTelemetry. -The current `oats` binary is the gcx-driven CLI. The legacy root runner has -been removed; existing users must migrate to the current `oats.toml` + -case-yaml flow when upgrading. +The `oats` binary is the gcx-driven CLI. Legacy direct-yaml invocation has been +removed; upgrades now use the current `oats.toml` + case-yaml flow. ## Install @@ -15,17 +14,25 @@ go install github.com/grafana/oats@latest ## Quick start ```sh +# Build local dev binaries (oats + gcx) into ./bin ./scripts/build-local-tools.sh + +# Print the CLI version +bin/oats version +bin/oats -version + +# Print what would run bin/oats --config examples/smoke/oats.toml --list + +# Run bin/oats --config examples/smoke/oats.toml ``` -## Key docs +## Layout -- current syntax and feature status: [CURRENT.md](CURRENT.md) -- migration guidance: [UPGRADING.md](UPGRADING.md) -- small runnable examples: [`examples/smoke/`](examples/smoke/) -- richer fixture examples: [`examples/fixtures/`](examples/fixtures/) +- `examples/smoke/` — small runnable examples +- `examples/fixtures/` — richer compose / k3d fixture examples +- `UPGRADING.md` — migration notes for older repos ## Current scope @@ -39,3 +46,80 @@ bin/oats --config examples/smoke/oats.toml ```sh oats --migrate path/to/legacy.yaml ``` + +## CLI + +```sh +oats --config oats.toml --list +oats --config oats.toml --suite smoke +oats --config oats.toml --tags traces,logs +oats --config oats.toml --gcx-context my-lgtm +oats --config oats.toml --no-cache +oats --format ndjson +oats -v +oats -v=2 +oats -v=3 +``` + +Key flags: + +- `--config` +- `--suite` +- `--tags` +- `--timeout` +- `--interval` +- `--absent-timeout` +- `--gcx` +- `--gcx-context` +- `--version` + +## Config shape + +```toml +[meta] +version = 2 + +[[suite]] +cases = ["examples/smoke/cases/*.yaml"] + +[cache] +ttl_days = 7 +``` + +Case yaml: + +```yaml +oats-schema-version: 3 +name: rolldice traces have route attribute + +fixture: + type: compose + template: lgtm + compose_file: docker-compose.oats.yml + +seed: + type: app +input: + - path: /rolldice?rolls=5 + +expected: + traces: + - traceql: '{ span.http.route = "/rolldice" }' + match_spans: + - name: "GET /rolldice" + metrics: + - promql: 'dice_lib_rolls_counter_total{service_name="dice-server"}' + value: '>= 0' + logs: + - logql: '{service_name="dice-server"}' + contains: Received request + custom-checks: + - script: ./verify.sh +``` + +## Notes + +- Cases currently run sequentially inside `oats`; there is no parallel case + execution yet. +- Case-local `fixture:` blocks cover the common one-case-per-suite shape. +- OATS owns local LGTM bootstrapping and gcx bootstrap for fixture-backed runs. diff --git a/UPGRADING.md b/UPGRADING.md index f43a1f60..00d8f395 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -14,7 +14,7 @@ The root `oats` binary now runs the gcx-driven current CLI only. That means upgrades now require migrating to the current: - `oats.toml` suite/discovery file -- current case yaml shape documented in [CURRENT.md](CURRENT.md) +- current case yaml shape documented in [README.md](README.md) The legacy “pass one or more old yaml files directly to `oats`” runner has been removed. For one-off help migrating old cases, use: @@ -35,7 +35,7 @@ Full release notes: ### Add `oats-schema-version: 3` to all test files All OATS test files must now include the `oats-schema-version` field at the -top level. The current version is `2`. +top level. The current version is `3`. ```yaml # ✅ Required in all test files diff --git a/internal/cli/main.go b/internal/cli/main.go index 5c5e31c0..b4e3a584 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -13,7 +13,7 @@ // --gcx Path to gcx binary (default "gcx" on PATH) // --list Print the run plan and exit (no execution) // --format Output format: "text" (default) or "ndjson" -// -v / -vv / -vvv Progressive verbosity (passes / commands / lifecycle) +// -v / -v=2 / -v=3 Progressive verbosity (passes / commands / lifecycle) // --suite Comma-separated suite names to include // --tags Comma-separated tag any-match filter package cli @@ -44,6 +44,10 @@ import ( ) var ( + // Version is the oats CLI version. Release builds can override this with + // -ldflags "-X github.com/grafana/oats/internal/cli.Version=vX.Y.Z". + Version = "dev" + newComposeSuite = func(files []string, env []string) (suiteFixture, error) { return compose.SuiteFiles(files, env) } @@ -76,9 +80,15 @@ func Main() { } func Run() int { + if len(os.Args) > 1 && os.Args[1] == "version" { + fmt.Println(Version) + return 0 + } + configPath := flag.String("config", "oats.toml", "path to oats.toml") gcxBin := flag.String("gcx", "gcx", "path to gcx binary (PATH-resolved if a bare name)") listOnly := flag.Bool("list", false, "print the run plan and exit (no execution)") + versionOnly := flag.Bool("version", false, "print the oats version and exit") migratePath := flag.String("migrate", "", "convert one legacy OATS yaml file and print the result to stdout") format := flag.String("format", "text", "output format: text | ndjson") suiteFilterStr := flag.String("suite", "", "comma-separated suite names") @@ -99,6 +109,11 @@ func Run() int { flag.Parse() + if *versionOnly { + fmt.Println(Version) + return 0 + } + if *migratePath != "" { out, warnings, err := migrate.ConvertFile(*migratePath) if err != nil { @@ -143,7 +158,7 @@ func Run() int { rep.Emit(report.Event{ Type: report.EventRunStart, - OatsVersion: "dev", + OatsVersion: Version, SchemaVersion: report.SchemaVersion, Ts: time.Now(), }) @@ -202,7 +217,7 @@ func Run() int { fixtureBytes, _ := json.Marshal(plan.Fixture) // stable across calls r = r.WithCache(store, runner.CacheContext{ GCXVersion: gcxVersion(*gcxBin), - OatsVersion: "dev", + OatsVersion: Version, FixtureBytes: fixtureBytes, }) } diff --git a/scripts/build-local-tools.sh b/scripts/build-local-tools.sh index fb4587af..ddaa2bb7 100755 --- a/scripts/build-local-tools.sh +++ b/scripts/build-local-tools.sh @@ -5,8 +5,8 @@ root="$(cd "$(dirname "$0")/.." && pwd)" bin_dir="${1:-$root/bin}" mkdir -p "$bin_dir" -# Keep the gcx bootstrap logic owned by OATS so consumer repos only need to -# fetch/build OATS itself. +# Local smoke/e2e runs need both binaries. Consumer repos should still only +# install `oats`; OATS owns the gcx bootstrap for fixture-backed runs. : "${GCX_VERSION:=v0.4.0}" GOWORK=off go -C "$root" build -buildvcs=false -o "$bin_dir/oats" . From a8e3ea44bcdd516ca30d062203e9f6882381eac5 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 14:47:38 +0200 Subject: [PATCH 077/124] fix: list top-level cases in summary Signed-off-by: Gregor Zeitlinger --- discovery/discovery.go | 2 +- discovery/discovery_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/discovery/discovery.go b/discovery/discovery.go index 1302f996..c964170c 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -380,7 +380,7 @@ func fixtureConfigFromCase(f casefile.FixtureConfig) FixtureConfig { // not load any cases — useful for a dry-run before deciding to expand globs. func (c *RootConfig) Summary() string { var out string - for _, s := range c.Suites { + for _, s := range c.effectiveSuites() { label := s.Name if label == "" { label = suiteLabel(s) diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index e0eddb57..62944cd7 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -119,6 +119,30 @@ tags = ["logs"] } } +func TestSummary_TopLevelCases(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +cases = ["cases/*.yaml"] + +[meta] +version = 2 +`) + writeFile(t, dir, "cases/a.yaml", strings.Replace(validCaseYAML, "%s", "case-a", 1)) + + cfg, err := Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + + got := cfg.Summary() + if !strings.Contains(got, `suite=cases`) { + t.Fatalf("Summary missing synthesized suite label: %q", got) + } + if !strings.Contains(got, `cases=[cases/*.yaml]`) { + t.Fatalf("Summary missing cases glob: %q", got) + } +} + func TestPlanRun_FilterBySuiteName(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats.toml", ` From af89fb95134ffbedbdddb33cd112ed79acf74970 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 15:09:28 +0200 Subject: [PATCH 078/124] feat: run safe suites in parallel Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 12 +- AGENTS.md | 2 +- README.md | 7 +- internal/cli/integration_test.go | 128 +++- internal/cli/main.go | 643 ++++++++++++++---- testhelpers/compose/compose.go | 13 +- .../fixture/parallel-compose/files/alpha.yaml | 15 + .../fixture/parallel-compose/files/beta.yaml | 15 + .../files/docker-compose.oats.yml | 4 + .../fixture/parallel-compose/files/oats.toml | 17 + .../parallel-compose/files/verify-alpha.sh | 14 + .../parallel-compose/files/verify-beta.sh | 14 + .../cases/fixture/parallel-compose/test.yaml | 7 + tests/e2e/e2e_test.go | 11 +- 14 files changed, 736 insertions(+), 166 deletions(-) create mode 100644 tests/e2e/cases/fixture/parallel-compose/files/alpha.yaml create mode 100644 tests/e2e/cases/fixture/parallel-compose/files/beta.yaml create mode 100644 tests/e2e/cases/fixture/parallel-compose/files/docker-compose.oats.yml create mode 100644 tests/e2e/cases/fixture/parallel-compose/files/oats.toml create mode 100644 tests/e2e/cases/fixture/parallel-compose/files/verify-alpha.sh create mode 100644 tests/e2e/cases/fixture/parallel-compose/files/verify-beta.sh create mode 100644 tests/e2e/cases/fixture/parallel-compose/test.yaml diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 7bd09b48..f07bf993 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -12,16 +12,8 @@ jobs: fail-fast: false matrix: include: - - name: assert-a - filter: "assert/absent,assert/contains,assert/count" - - name: assert-b - filter: "assert/match,assert/not-contains,assert/regex" - - name: custom-and-seed - filter: "custom/,seed/" - - name: compose - filter: "fixture/compose" - - name: remote - filter: "fixture/remote" + - name: non-k3d + filter: "assert/,custom/,seed/,fixture/compose,fixture/parallel-compose,fixture/remote" - name: k3d-fail filter: "fixture/k3d-fail" - name: k3d-smoke diff --git a/AGENTS.md b/AGENTS.md index c27e227c..ba307a50 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,7 +74,7 @@ oats --config oats.toml --timeout 1m ``` Key flags: `--config`, `--suite`, `--tags`, `--timeout`, `--interval`, -`--absent-timeout`, `--gcx`, `--gcx-context` +`--absent-timeout`, `--parallel`, `--gcx`, `--gcx-context`, `--version` ## Code Conventions diff --git a/README.md b/README.md index cfe4a163..b1a17d31 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ Key flags: - `--timeout` - `--interval` - `--absent-timeout` +- `--parallel` - `--gcx` - `--gcx-context` - `--version` @@ -119,7 +120,9 @@ expected: ## Notes -- Cases currently run sequentially inside `oats`; there is no parallel case - execution yet. +- Cases inside one suite still run sequentially. +- Suites can run in parallel with `--parallel N` when fixture isolation allows + it. Today that mainly means remote suites and compose suites where OATS owns + the LGTM ports (`template = "lgtm"`). - Case-local `fixture:` blocks cover the common one-case-per-suite shape. - OATS owns local LGTM bootstrapping and gcx bootstrap for fixture-backed runs. diff --git a/internal/cli/integration_test.go b/internal/cli/integration_test.go index 1828a58b..1568c648 100644 --- a/internal/cli/integration_test.go +++ b/internal/cli/integration_test.go @@ -54,7 +54,7 @@ func TestResolveComposeFiles(t *testing.T) { t.Fatalf("expected cleanup for template=lgtm fixture") } defer func() { _ = cleanup() }() - if len(got) != 1 || !strings.HasSuffix(got[0], ".oats.lgtm.compose.yml") { + if len(got) != 1 || !strings.Contains(filepath.Base(got[0]), ".oats.lgtm.") || !strings.HasSuffix(got[0], ".compose.yml") { t.Fatalf("unexpected template=lgtm resolution: %v", got) } } @@ -67,11 +67,11 @@ func TestResolveEndpoint_ComposeDefaults(t *testing.T) { ep, err := resolveEndpoint(discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, Fixture: discovery.FixtureConfig{Type: "compose", Template: "lgtm"}, - }, "", "localhost", 8080, "http://localhost:4318") + }, fixtureRuntime{GCXConfig: "/tmp/gcx.yaml", OTLPHTTP: "http://127.0.0.1:4318"}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } - if ep.GCXContext != "" || ep.AppHost != "localhost" || ep.AppPort != 8080 || ep.OTLPHTTP != "http://localhost:4318" { + if ep.GCXContext != "" || ep.GCXConfig != "/tmp/gcx.yaml" || ep.AppHost != "localhost" || ep.AppPort != 8080 || ep.OTLPHTTP != "http://127.0.0.1:4318" { t.Fatalf("unexpected endpoint: %+v", ep) } } @@ -84,7 +84,7 @@ func TestResolveEndpoint_K3DUsesFixtureAppPort(t *testing.T) { ep, err := resolveEndpoint(discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "cluster"}, Fixture: discovery.FixtureConfig{Type: "k3d", AppPort: 18080}, - }, "", "localhost", 8080, "http://localhost:4318") + }, fixtureRuntime{GCXConfig: "/tmp/gcx.yaml", OTLPHTTP: "http://127.0.0.1:4318"}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -95,7 +95,9 @@ func TestResolveEndpoint_K3DUsesFixtureAppPort(t *testing.T) { func TestStartFixture_ComposeLifecycle(t *testing.T) { oldFactory := newComposeSuite + oldLookup := lookupComposePort defer func() { newComposeSuite = oldFactory }() + defer func() { lookupComposePort = oldLookup }() var gotFiles, gotEnv []string fake := &fakeSuiteFixture{} @@ -104,8 +106,20 @@ func TestStartFixture_ComposeLifecycle(t *testing.T) { gotEnv = append([]string(nil), env...) return fake, nil } + lookupComposePort = func(files []string, env []string, service, containerPort string) (string, error) { + switch containerPort { + case "3000": + return "43000", nil + case "4318": + return "44318", nil + case "4040": + return "44040", nil + default: + return "", fmt.Errorf("unexpected port %s", containerPort) + } + } - fix, err := startFixture(context.Background(), discovery.Plan{ + fix, _, err := startFixture(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, Fixture: discovery.FixtureConfig{ Type: "compose", @@ -123,8 +137,8 @@ func TestStartFixture_ComposeLifecycle(t *testing.T) { if want := []string{"/tmp/work/a.yml", "/tmp/work/b.yml"}; !equalStrings(gotFiles, want) { t.Fatalf("compose files: got %v want %v", gotFiles, want) } - if want := []string{"FOO=bar"}; !equalStrings(gotEnv, want) { - t.Fatalf("compose env: got %v want %v", gotEnv, want) + if len(gotEnv) != 2 || gotEnv[0] != "FOO=bar" || !strings.HasPrefix(gotEnv[1], "COMPOSE_PROJECT_NAME=oats-smoke-") { + t.Fatalf("compose env: got %v", gotEnv) } if err := fix.Close(); err != nil { t.Fatalf("fixture close: %v", err) @@ -142,7 +156,7 @@ func TestStartFixture_ComposeStartFailure(t *testing.T) { return &fakeSuiteFixture{upErr: fmt.Errorf("boom")}, nil } - _, err := startFixture(context.Background(), discovery.Plan{ + _, _, err := startFixture(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, Fixture: discovery.FixtureConfig{Type: "compose", ComposeFile: "compose.yml"}, FixtureSourceDir: "/tmp/work", @@ -169,7 +183,7 @@ func TestStartFixture_K3DLifecycle(t *testing.T) { }, nil) } - fix, err := startFixture(context.Background(), discovery.Plan{ + fix, _, err := startFixture(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, Fixture: discovery.FixtureConfig{ Type: "k3d", @@ -212,7 +226,7 @@ func TestStartFixture_K3DStartFailure(t *testing.T) { }, nil) } - _, err := startFixture(context.Background(), discovery.Plan{ + _, _, err := startFixture(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, Fixture: discovery.FixtureConfig{ Type: "k3d", @@ -244,6 +258,86 @@ func TestResolveComposeFiles_MissingConfig(t *testing.T) { } } +func TestComposeFilePublishesFixedHostPorts(t *testing.T) { + dir := t.TempDir() + fixed := filepath.Join(dir, "fixed.yml") + random := filepath.Join(dir, "random.yml") + writeFile(t, dir, "fixed.yml", `services: + app: + image: alpine + ports: + - "8080:8080" +`) + writeFile(t, dir, "random.yml", `services: + app: + image: alpine + ports: + - "8080" +`) + got, err := composeFilePublishesFixedHostPorts(fixed) + if err != nil { + t.Fatalf("composeFilePublishesFixedHostPorts fixed: %v", err) + } + if !got { + t.Fatalf("expected fixed host port detection for %s", fixed) + } + got, err = composeFilePublishesFixedHostPorts(random) + if err != nil { + t.Fatalf("composeFilePublishesFixedHostPorts random: %v", err) + } + if got { + t.Fatalf("did not expect fixed host port detection for %s", random) + } +} + +func TestPlanSupportsParallel_ComposeTemplateLGTM(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "parallel-safe" +cases = ["cases/*.yaml"] +fixture = "stack" + +[fixture.stack] +type = "compose" +template = "lgtm" +compose_file = "docker-compose.oats.yml" +`) + writeFile(t, dir, "docker-compose.oats.yml", `services: + app: + image: alpine + command: ["sh", "-c", "sleep 1"] +`) + writeFile(t, dir, "cases/a.yaml", `oats-schema-version: 3 +name: a +seed: + type: inline-otlp + logs: + - service: a + body: line +expected: + logs: + - logql: '{service_name="a"}' + contains: line +`) + + cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + safe, reason := planSupportsParallel(plans[0]) + if !safe { + t.Fatalf("expected plan to be parallel-safe, got false: %s", reason) + } +} + func TestCloseFixture_EmitsTeardownEvent(t *testing.T) { rep := &recordingReporter{} fix := &fakeSuiteFixture{} @@ -508,7 +602,7 @@ expected: t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], "", appHost, appPort, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", appHost, appPort, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -584,7 +678,7 @@ expected: t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -663,7 +757,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -751,7 +845,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -830,7 +924,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -908,7 +1002,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -1006,7 +1100,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected k8s-only assertion to be filtered out, got %d traces", got) } - ep, err := resolveEndpoint(plans[0], "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } diff --git a/internal/cli/main.go b/internal/cli/main.go index b4e3a584..62041657 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -28,7 +28,9 @@ import ( "os/exec" "os/signal" "path/filepath" + "strconv" "strings" + "sync" "syscall" "time" @@ -73,8 +75,27 @@ var ( }, plan.Suite.Name, sourceDir) } waitForGrafanaToken = waitForGrafanaTokenImpl + lookupComposePort = dockerComposePort ) +type fixtureRuntime struct { + GrafanaURL string + OTLPHTTP string + PyroscopeURL string + CustomCheckEnv []string + ComposeFiles []string + ComposeProject string + GCXConfig string + ParallelSafe bool + ParallelDisabled string +} + +type suiteResult struct { + pass int + fail int + err error +} + func Main() { os.Exit(Run()) } @@ -101,6 +122,7 @@ func Run() int { appHost := flag.String("app-host", "localhost", "application host for driving case input requests") appPort := flag.Int("app-port", 8080, "application port for driving case input requests") otlpHTTP := flag.String("otlp-http", "http://localhost:4318", "OTLP/HTTP base URL for inline-otlp seed mode") + parallel := flag.Int("parallel", 1, "number of suites to run in parallel when fixture isolation allows it") noCache := flag.Bool("no-cache", false, "disable the skip-when-unchanged cache for this run") cacheDir := flag.String("cache-dir", defaultCacheDir(), "directory for the skip-when-unchanged cache") @@ -155,6 +177,7 @@ func Run() int { rep := newReporter(os.Stdout, *format, verbosityFromInt(verbose)) defer func() { _ = rep.Close() }() + rep = &lockedReporter{inner: rep} rep.Emit(report.Event{ Type: report.EventRunStart, @@ -167,88 +190,24 @@ func Run() int { defer cancel() runStart := time.Now() - var totalPass, totalFail int - - for _, plan := range plans { - fixtureStart := emitFixtureStart(rep, plan) - fix, err := startFixture(ctx, plan) - if err != nil { - fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) - return 2 - } - if err := waitForFixtureReady(plan); err != nil { - if fix != nil { - _ = closeFixture(rep, plan, fix) - } - fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) - return 2 - } - emitFixtureReady(rep, plan, fixtureStart) - ep, err := resolveEndpoint(plan, *gcxContextOverride, *appHost, *appPort, *otlpHTTP) - if err != nil { - if fix != nil { - _ = closeFixture(rep, plan, fix) - } - fmt.Fprintf(os.Stderr, "suite %q: %v\n", plan.Suite.Name, err) - return 2 - } - - rep.Emit(report.Event{ - Type: report.EventSuiteStart, - Suite: plan.Suite.Name, - Fixture: plan.Suite.Fixture, - FixtureType: plan.Fixture.Type, - CaseCount: len(plan.Cases), - }) - - gcxExec := &engine.GCX{Binary: *gcxBin, Context: ep.GCXContext, Config: ep.GCXConfig, Env: ep.GCXEnv} - r := runner.New(gcxExec, rep, ep, runner.Options{ - Timeout: *timeout, - Interval: *interval, - AbsentTimeout: *absentTimeout, - SeedSettleDelay: *seedSettle, - }) - if !*noCache && *cacheDir != "" { - ttl := time.Duration(cfg.Cache.TTLDays) * 24 * time.Hour - store, cacheErr := cache.New(*cacheDir, ttl, nil) - if cacheErr != nil { - fmt.Fprintln(os.Stderr, "cache disabled:", cacheErr) - } else { - fixtureBytes, _ := json.Marshal(plan.Fixture) // stable across calls - r = r.WithCache(store, runner.CacheContext{ - GCXVersion: gcxVersion(*gcxBin), - OatsVersion: Version, - FixtureBytes: fixtureBytes, - }) - } - } - - var suitePass, suiteFail int - for _, c := range plan.Cases { - if ctx.Err() != nil { - break - } - if r.RunCase(ctx, c) { - suitePass++ - } else { - suiteFail++ - } - } - totalPass += suitePass - totalFail += suiteFail - - rep.Emit(report.Event{ - Type: report.EventSuiteEnd, - Suite: plan.Suite.Name, - Pass: suitePass, - Fail: suiteFail, - }) - if fix != nil { - if closeErr := closeFixture(rep, plan, fix); closeErr != nil { - fmt.Fprintf(os.Stderr, "suite %q: fixture shutdown: %v\n", plan.Suite.Name, closeErr) - return 2 - } - } + opts := runOptions{ + gcxBin: *gcxBin, + gcxContextOverride: *gcxContextOverride, + appHost: *appHost, + appPort: *appPort, + otlpHTTP: *otlpHTTP, + timeout: *timeout, + interval: *interval, + absentTimeout: *absentTimeout, + seedSettle: *seedSettle, + noCache: *noCache, + cacheDir: *cacheDir, + cacheTTLDays: cfg.Cache.TTLDays, + } + totalPass, totalFail, runErr := runPlans(ctx, rep, plans, opts, *parallel) + if runErr != nil { + fmt.Fprintln(os.Stderr, runErr) + return 2 } rep.Emit(report.Event{ @@ -288,7 +247,7 @@ func verbosityFromInt(n int) report.Verbosity { // resolveEndpoint maps a fixture config + an explicit override into the // concrete endpoint the runner needs. -func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, appPort int, otlpHTTP string) (runner.Endpoint, error) { +func resolveEndpoint(plan discovery.Plan, rt fixtureRuntime, gcxContextOverride, appHost string, appPort int, otlpHTTP string) (runner.Endpoint, error) { ep := runner.Endpoint{AppHost: appHost, AppPort: appPort, OTLPHTTP: otlpHTTP} switch plan.Fixture.Type { case "remote": @@ -303,20 +262,24 @@ func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, ap "OATS_APP_URL="+fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), ) case "compose": - ep.GCXEnv = append(ep.GCXEnv, localGrafanaEnv(plan)...) - if cfg, err := writeLocalGCXConfig(plan.FixtureSourceDir); err == nil { - ep.GCXConfig = cfg + if rt.GCXConfig != "" { + ep.GCXConfig = rt.GCXConfig } - ep.CustomCheckEnv = append(ep.CustomCheckEnv, composeCheckEnv(plan, ep)...) + if rt.OTLPHTTP != "" { + ep.OTLPHTTP = rt.OTLPHTTP + } + ep.CustomCheckEnv = append(ep.CustomCheckEnv, rt.CustomCheckEnv...) case "k3d": - ep.GCXEnv = append(ep.GCXEnv, localGrafanaEnv(plan)...) - if cfg, err := writeLocalGCXConfig(plan.FixtureSourceDir); err == nil { - ep.GCXConfig = cfg + if rt.GCXConfig != "" { + ep.GCXConfig = rt.GCXConfig + } + if rt.OTLPHTTP != "" { + ep.OTLPHTTP = rt.OTLPHTTP } if plan.Fixture.AppPort > 0 { ep.AppPort = plan.Fixture.AppPort } - ep.CustomCheckEnv = append(ep.CustomCheckEnv, k3dCheckEnv(ep)...) + ep.CustomCheckEnv = append(ep.CustomCheckEnv, rt.CustomCheckEnv...) case "": // No fixture configured — caller (or --gcx-context) must supply // everything. Useful while plumbing the new CLI against an external setup. @@ -335,6 +298,194 @@ func resolveEndpoint(plan discovery.Plan, gcxContextOverride, appHost string, ap return ep, nil } +type runOptions struct { + gcxBin string + gcxContextOverride string + appHost string + appPort int + otlpHTTP string + timeout time.Duration + interval time.Duration + absentTimeout time.Duration + seedSettle time.Duration + noCache bool + cacheDir string + cacheTTLDays int +} + +func runPlans(ctx context.Context, rep report.Reporter, plans []discovery.Plan, opts runOptions, parallel int) (int, int, error) { + if parallel < 1 { + parallel = 1 + } + if parallel == 1 || len(plans) <= 1 { + return runPlansSequential(ctx, rep, plans, opts) + } + + var serialPlans, parallelPlans []discovery.Plan + for _, plan := range plans { + if safe, _ := planSupportsParallel(plan); safe { + parallelPlans = append(parallelPlans, plan) + } else { + serialPlans = append(serialPlans, plan) + } + } + + totalPass, totalFail, err := runPlansParallel(ctx, rep, parallelPlans, opts, parallel) + if err != nil { + return totalPass, totalFail, err + } + + pass, fail, err := runPlansSequential(ctx, rep, serialPlans, opts) + return totalPass + pass, totalFail + fail, err +} + +func runPlansSequential(ctx context.Context, rep report.Reporter, plans []discovery.Plan, opts runOptions) (int, int, error) { + var totalPass, totalFail int + for _, plan := range plans { + res := runPlan(ctx, rep, plan, opts) + totalPass += res.pass + totalFail += res.fail + if res.err != nil { + return totalPass, totalFail, res.err + } + } + return totalPass, totalFail, nil +} + +func runPlansParallel(parent context.Context, rep report.Reporter, plans []discovery.Plan, opts runOptions, parallel int) (int, int, error) { + if len(plans) == 0 { + return 0, 0, nil + } + ctx, cancel := context.WithCancel(parent) + defer cancel() + + workCh := make(chan discovery.Plan) + resultCh := make(chan suiteResult, len(plans)) + workers := parallel + if workers > len(plans) { + workers = len(plans) + } + + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for plan := range workCh { + res := runPlan(ctx, rep, plan, opts) + if res.err != nil { + cancel() + } + resultCh <- res + } + }() + } + + go func() { + defer close(workCh) + for _, plan := range plans { + select { + case <-ctx.Done(): + return + case workCh <- plan: + } + } + }() + + go func() { + wg.Wait() + close(resultCh) + }() + + var totalPass, totalFail int + var firstErr error + for res := range resultCh { + totalPass += res.pass + totalFail += res.fail + if res.err != nil && firstErr == nil { + firstErr = res.err + } + } + return totalPass, totalFail, firstErr +} + +func runPlan(ctx context.Context, rep report.Reporter, plan discovery.Plan, opts runOptions) suiteResult { + fixtureStart := emitFixtureStart(rep, plan) + fix, rt, err := startFixture(ctx, plan) + if err != nil { + return suiteResult{err: fmt.Errorf("suite %q: %w", plan.Suite.Name, err)} + } + if err := waitForFixtureReady(plan, rt); err != nil { + if fix != nil { + _ = closeFixture(rep, plan, fix) + } + return suiteResult{err: fmt.Errorf("suite %q: %w", plan.Suite.Name, err)} + } + emitFixtureReady(rep, plan, fixtureStart) + ep, err := resolveEndpoint(plan, rt, opts.gcxContextOverride, opts.appHost, opts.appPort, opts.otlpHTTP) + if err != nil { + if fix != nil { + _ = closeFixture(rep, plan, fix) + } + return suiteResult{err: fmt.Errorf("suite %q: %w", plan.Suite.Name, err)} + } + + rep.Emit(report.Event{ + Type: report.EventSuiteStart, + Suite: plan.Suite.Name, + Fixture: plan.Suite.Fixture, + FixtureType: plan.Fixture.Type, + CaseCount: len(plan.Cases), + }) + + gcxExec := &engine.GCX{Binary: opts.gcxBin, Context: ep.GCXContext, Config: ep.GCXConfig, Env: ep.GCXEnv} + r := runner.New(gcxExec, rep, ep, runner.Options{ + Timeout: opts.timeout, + Interval: opts.interval, + AbsentTimeout: opts.absentTimeout, + SeedSettleDelay: opts.seedSettle, + }) + if !opts.noCache && opts.cacheDir != "" { + ttl := time.Duration(opts.cacheTTLDays) * 24 * time.Hour + store, cacheErr := cache.New(opts.cacheDir, ttl, nil) + if cacheErr != nil { + fmt.Fprintln(os.Stderr, "cache disabled:", cacheErr) + } else { + fixtureBytes, _ := json.Marshal(plan.Fixture) + r = r.WithCache(store, runner.CacheContext{ + GCXVersion: gcxVersion(opts.gcxBin), + OatsVersion: Version, + FixtureBytes: fixtureBytes, + }) + } + } + + var suitePass, suiteFail int + for _, c := range plan.Cases { + if ctx.Err() != nil { + break + } + if r.RunCase(ctx, c) { + suitePass++ + } else { + suiteFail++ + } + } + + rep.Emit(report.Event{ + Type: report.EventSuiteEnd, + Suite: plan.Suite.Name, + Pass: suitePass, + Fail: suiteFail, + }) + if fix != nil { + if closeErr := closeFixture(rep, plan, fix); closeErr != nil { + return suiteResult{pass: suitePass, fail: suiteFail, err: fmt.Errorf("suite %q: fixture shutdown: %w", plan.Suite.Name, closeErr)} + } + } + return suiteResult{pass: suitePass, fail: suiteFail} +} + type suiteFixture interface { Close() error } @@ -344,37 +495,87 @@ type startableSuiteFixture interface { Up() error } -func startFixture(ctx context.Context, plan discovery.Plan) (suiteFixture, error) { +func startFixture(ctx context.Context, plan discovery.Plan) (suiteFixture, fixtureRuntime, error) { switch plan.Fixture.Type { case "", "remote": - return nil, nil + return nil, fixtureRuntime{ParallelSafe: true}, nil case "compose": composeFiles, cleanup, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) if err != nil { - return nil, err + return nil, fixtureRuntime{}, err } - suite, err := newComposeSuite(composeFiles, plan.Fixture.Env) + project := composeProjectName(plan) + suiteEnv := append([]string(nil), plan.Fixture.Env...) + suiteEnv = append(suiteEnv, "COMPOSE_PROJECT_NAME="+project) + suite, err := newComposeSuite(composeFiles, suiteEnv) if err != nil { if cleanup != nil { _ = cleanup() } - return nil, err + return nil, fixtureRuntime{}, err } if err := startSuiteFixture(suite); err != nil { if cleanup != nil { _ = cleanup() } - return nil, err + return nil, fixtureRuntime{}, err + } + grafanaPort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "3000") + if err != nil { + _ = suite.Close() + if cleanup != nil { + _ = cleanup() + } + return nil, fixtureRuntime{}, err } - return composeFixture{suite: suite, cleanup: cleanup}, nil + otlpPort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "4318") + if err != nil { + _ = suite.Close() + if cleanup != nil { + _ = cleanup() + } + return nil, fixtureRuntime{}, err + } + pyroscopePort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "4040") + if err != nil { + _ = suite.Close() + if cleanup != nil { + _ = cleanup() + } + return nil, fixtureRuntime{}, err + } + rt := fixtureRuntime{ + GrafanaURL: "http://127.0.0.1:" + grafanaPort, + OTLPHTTP: "http://127.0.0.1:" + otlpPort, + PyroscopeURL: "http://127.0.0.1:" + pyroscopePort, + ComposeFiles: composeFiles, + ComposeProject: project, + } + if cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL); cfgErr == nil { + rt.GCXConfig = cfg + } + rt.CustomCheckEnv = composeCheckEnv(plan, rt) + rt.ParallelSafe, rt.ParallelDisabled = planSupportsParallel(plan) + return composeFixture{suite: suite, cleanup: cleanup}, rt, nil case "k3d": ep := newKubernetesEndpoint(plan) if err := ep.Start(ctx); err != nil { - return nil, err + return nil, fixtureRuntime{}, err + } + rt := fixtureRuntime{ + GrafanaURL: "http://localhost:3000", + OTLPHTTP: "http://localhost:4318", + PyroscopeURL: "http://localhost:4040", + CustomCheckEnv: k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: plan.Fixture.AppPort}), + ParallelSafe: false, + ParallelDisabled: "k3d fixtures currently use shared localhost port-forwards", + } + if cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL); cfgErr == nil { + rt.GCXConfig = cfg } - return endpointFixture{ep: ep}, nil + return endpointFixture{ep: ep}, rt, nil default: - return nil, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) + return nil, fixtureRuntime{}, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) } } @@ -479,24 +680,24 @@ func (c composeFixture) Close() error { } func writeBuiltinLGTMCompose(sourceDir string) (string, error) { - path := filepath.Join(sourceDir, ".oats.lgtm.compose.yml") - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + if err := os.MkdirAll(sourceDir, 0o755); err != nil { return "", err } - f, err := os.Create(path) + f, err := os.CreateTemp(sourceDir, ".oats.lgtm.*.compose.yml") if err != nil { return "", err } + path := f.Name() const body = `services: lgtm: image: ${LGTM_IMAGE:-docker.io/grafana/otel-lgtm:latest} ports: - - "3000:3000" - - "4317:4317" - - "4318:4318" - - "3200:3200" - - "4040:4040" - - "9090:9090" + - "127.0.0.1::3000" + - "127.0.0.1::4317" + - "127.0.0.1::4318" + - "127.0.0.1::3200" + - "127.0.0.1::4040" + - "127.0.0.1::9090" ` if _, err := f.WriteString(body); err != nil { _ = f.Close() @@ -510,19 +711,14 @@ func writeBuiltinLGTMCompose(sourceDir string) (string, error) { return path, nil } -func grafanaURL() string { return "http://localhost:3000" } -func pyroscopeURL() string { return "http://localhost:4040" } +func grafanaURL() string { return "http://localhost:3000" } -func localGrafanaEnv(plan discovery.Plan) []string { - return nil -} - -func writeLocalGCXConfig(_ string) (string, error) { - cfg := `current-context: local +func writeLocalGCXConfig(grafanaURL string) (string, error) { + cfg := fmt.Sprintf(`current-context: local contexts: local: grafana: - server: http://localhost:3000 + server: %s user: admin password: admin org-id: 1 @@ -532,7 +728,7 @@ contexts: loki: loki tempo: tempo pyroscope: pyroscope -` +`, grafanaURL) f, err := os.CreateTemp("", "oats-gcx-*.yaml") if err != nil { return "", err @@ -554,18 +750,19 @@ contexts: return path, nil } -func composeCheckEnv(plan discovery.Plan, ep runner.Endpoint) []string { - files, _, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) - if err != nil { +func composeCheckEnv(plan discovery.Plan, rt fixtureRuntime) []string { + files := rt.ComposeFiles + if len(files) == 0 { return []string{"OATS_FIXTURE_TYPE=compose"} } return []string{ "OATS_FIXTURE_TYPE=compose", + "COMPOSE_PROJECT_NAME=" + rt.ComposeProject, "COMPOSE_FILE=" + strings.Join(files, string(os.PathListSeparator)), "OATS_COMPOSE_FILE_ARGS=" + composeFileArgs(files), - "OATS_APP_URL=" + fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), - "OATS_GRAFANA_URL=" + grafanaURL(), - "OATS_PYROSCOPE_URL=" + pyroscopeURL(), + "OATS_GRAFANA_URL=" + rt.GrafanaURL, + "OATS_OTLP_HTTP=" + rt.OTLPHTTP, + "OATS_PYROSCOPE_URL=" + rt.PyroscopeURL, } } @@ -585,8 +782,9 @@ func k3dCheckEnv(ep runner.Endpoint) []string { return []string{ "OATS_FIXTURE_TYPE=k3d", "OATS_APP_URL=" + fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), - "OATS_GRAFANA_URL=" + grafanaURL(), - "OATS_PYROSCOPE_URL=" + pyroscopeURL(), + "OATS_GRAFANA_URL=http://localhost:3000", + "OATS_OTLP_HTTP=http://localhost:4318", + "OATS_PYROSCOPE_URL=http://localhost:4040", } } @@ -611,19 +809,147 @@ func waitForGrafanaTokenImpl(plan discovery.Plan) (string, error) { return "", fmt.Errorf("timed out waiting for Grafana service-account token") } -func waitForFixtureReady(plan discovery.Plan) error { +func waitForFixtureReady(plan discovery.Plan, rt fixtureRuntime) error { switch plan.Fixture.Type { case "compose", "k3d": - if err := waitForHTTP(grafanaURL()+"/api/health", 2*time.Minute); err != nil { + if err := waitForHTTP(rt.GrafanaURL+"/api/health", 2*time.Minute); err != nil { return fmt.Errorf("wait for grafana: %w", err) } - if err := waitForHTTP("http://localhost:4318", 2*time.Minute); err != nil { + if err := waitForHTTP(rt.OTLPHTTP, 2*time.Minute); err != nil { return fmt.Errorf("wait for otlp-http: %w", err) } } return nil } +func planSupportsParallel(plan discovery.Plan) (bool, string) { + switch plan.Fixture.Type { + case "", "remote": + return true, "" + case "compose": + if plan.Fixture.Template != "lgtm" { + return false, "compose fixtures are only parallel-safe when OATS owns the LGTM ports via template=lgtm" + } + for _, c := range plan.Cases { + if c.Seed.Type == "app" { + return false, "compose suites with app seeds still rely on shared fixed app ports" + } + } + for _, file := range extraComposeFiles(plan) { + if fixed, err := composeFilePublishesFixedHostPorts(file); err != nil { + return false, fmt.Sprintf("compose port inspection failed for %s: %v", file, err) + } else if fixed { + return false, fmt.Sprintf("compose file %s publishes fixed host ports", filepath.Base(file)) + } + } + return true, "" + case "k3d": + return false, "k3d fixtures currently use shared localhost port-forwards" + default: + return false, "fixture type is not parallel-safe" + } +} + +func composeProjectName(plan discovery.Plan) string { + name := strings.ToLower(plan.Suite.Name) + if name == "" { + name = "oats" + } + var b strings.Builder + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + default: + b.WriteByte('-') + } + } + slug := strings.Trim(b.String(), "-") + if slug == "" { + slug = "oats" + } + if len(slug) > 32 { + slug = slug[:32] + } + return fmt.Sprintf("oats-%s-%d", slug, time.Now().UnixNano()) +} + +func extraComposeFiles(plan discovery.Plan) []string { + switch { + case plan.Fixture.ComposeFile != "": + return []string{filepath.Join(plan.FixtureSourceDir, plan.Fixture.ComposeFile)} + case len(plan.Fixture.ComposeFiles) > 0: + files := make([]string, 0, len(plan.Fixture.ComposeFiles)) + for _, file := range plan.Fixture.ComposeFiles { + files = append(files, filepath.Join(plan.FixtureSourceDir, file)) + } + return files + default: + return nil + } +} + +func composeFilePublishesFixedHostPorts(path string) (bool, error) { + data, err := os.ReadFile(path) + if err != nil { + return false, err + } + lines := strings.Split(string(data), "\n") + inPorts := false + portsIndent := 0 + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + indent := len(line) - len(strings.TrimLeft(line, " ")) + if inPorts && indent <= portsIndent { + inPorts = false + } + if strings.HasPrefix(trimmed, "ports:") { + inPorts = true + portsIndent = indent + continue + } + if !inPorts { + continue + } + if strings.Contains(trimmed, "published:") { + value := strings.TrimSpace(strings.TrimPrefix(trimmed, "published:")) + if value != "" && value != "0" { + return true, nil + } + continue + } + if !strings.HasPrefix(trimmed, "-") { + continue + } + value := strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "-")), `"'`) + if value == "" { + continue + } + if fixedShortPortMapping(value) { + return true, nil + } + } + return false, nil +} + +func fixedShortPortMapping(value string) bool { + if !strings.Contains(value, ":") { + return false + } + parts := strings.Split(value, ":") + if len(parts) < 2 { + return false + } + hostPart := strings.Trim(parts[len(parts)-2], "[]") + if _, err := strconv.Atoi(hostPart); err == nil && hostPart != "0" { + return true + } + return false +} + func waitForHTTP(url string, timeout time.Duration) error { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { @@ -658,6 +984,44 @@ func readComposeGrafanaToken(plan discovery.Plan) (string, error) { return string(out), nil } +func dockerComposePort(files []string, env []string, service, containerPort string) (string, error) { + args := []string{"compose"} + for _, f := range files { + args = append(args, "-f", f) + } + args = append(args, "port", service, containerPort) + cmd := exec.Command("docker", args...) + cmd.Env = append(cmd.Environ(), env...) + out, err := cmd.Output() + if err != nil { + return "", err + } + host, port, err := splitDockerHostPort(strings.TrimSpace(string(out))) + if err != nil { + return "", err + } + if host == "" || port == "" { + return "", fmt.Errorf("invalid docker compose port output %q", strings.TrimSpace(string(out))) + } + return port, nil +} + +func splitDockerHostPort(addr string) (string, string, error) { + addr = strings.TrimSpace(addr) + if strings.HasPrefix(addr, "[") { + end := strings.Index(addr, "]") + if end < 0 || end+2 > len(addr) || addr[end+1] != ':' { + return "", "", fmt.Errorf("invalid address %q", addr) + } + return addr[1:end], addr[end+2:], nil + } + idx := strings.LastIndex(addr, ":") + if idx < 0 { + return "", "", fmt.Errorf("invalid address %q", addr) + } + return addr[:idx], addr[idx+1:], nil +} + func readK3DGrafanaToken() (string, error) { cmd := exec.Command("kubectl", "exec", "deploy/lgtm", "--", "sh", "-c", "cat /tmp/grafana-sa-token 2>/dev/null || true") out, err := cmd.Output() @@ -718,3 +1082,20 @@ func signalAwareContext() (context.Context, context.CancelFunc) { }() return ctx, cancel } + +type lockedReporter struct { + mu sync.Mutex + inner report.Reporter +} + +func (r *lockedReporter) Emit(e report.Event) { + r.mu.Lock() + defer r.mu.Unlock() + r.inner.Emit(e) +} + +func (r *lockedReporter) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + return r.inner.Close() +} diff --git a/testhelpers/compose/compose.go b/testhelpers/compose/compose.go index 749c5487..382e6cfe 100644 --- a/testhelpers/compose/compose.go +++ b/testhelpers/compose/compose.go @@ -23,6 +23,8 @@ type Compose struct { Env []string } +var dockerPruneMu sync.Mutex + func defaultEnv() []string { return os.Environ() } @@ -53,7 +55,9 @@ func SuiteFiles(composeFiles []string, env []string) (*Compose, error) { func (c *Compose) Up() error { // networks accumulate over time and can cause issues with the tests + dockerPruneMu.Lock() err := c.runDocker(newCommand("network", "prune", "-f", "--filter", "until=5m").withCompose(false)) + dockerPruneMu.Unlock() if err != nil { return fmt.Errorf("failed to prune docker networks: %w", err) } @@ -101,7 +105,10 @@ func (c *Compose) runDocker(cc command) error { slog.Info("Running", "command", cmd.String(), "compose_files", c.Paths) stdout, _ := cmd.StdoutPipe() cmd.Stderr = cmd.Stdout + wg := sync.WaitGroup{} + wg.Add(1) go func() { + defer wg.Done() reader := bufio.NewReader(stdout) line, err := reader.ReadString('\n') for err == nil { @@ -114,7 +121,11 @@ func (c *Compose) runDocker(cc command) error { if err != nil { return fmt.Errorf("failed to start docker command: %w", err) } - go func() { _ = cmd.Wait() }() + err = cmd.Wait() + wg.Wait() + if err != nil { + return fmt.Errorf("failed to run docker command: %w", err) + } } else { slog.Info("Running", "command", cmd.String(), "compose_files", c.Paths) cmd.Stdout = os.Stdout diff --git a/tests/e2e/cases/fixture/parallel-compose/files/alpha.yaml b/tests/e2e/cases/fixture/parallel-compose/files/alpha.yaml new file mode 100644 index 00000000..92f36bc4 --- /dev/null +++ b/tests/e2e/cases/fixture/parallel-compose/files/alpha.yaml @@ -0,0 +1,15 @@ +oats-schema-version: 3 +name: alpha +seed: + type: inline-otlp + logs: + - service: alpha + body: alpha-line +expected: + logs: + - logql: '{service_name="alpha"}' + contains: alpha-line + compose-logs: + - app started ok + custom-checks: + - script: ./verify-alpha.sh diff --git a/tests/e2e/cases/fixture/parallel-compose/files/beta.yaml b/tests/e2e/cases/fixture/parallel-compose/files/beta.yaml new file mode 100644 index 00000000..1f5dd8ab --- /dev/null +++ b/tests/e2e/cases/fixture/parallel-compose/files/beta.yaml @@ -0,0 +1,15 @@ +oats-schema-version: 3 +name: beta +seed: + type: inline-otlp + logs: + - service: beta + body: beta-line +expected: + logs: + - logql: '{service_name="beta"}' + contains: beta-line + compose-logs: + - app started ok + custom-checks: + - script: ./verify-beta.sh diff --git a/tests/e2e/cases/fixture/parallel-compose/files/docker-compose.oats.yml b/tests/e2e/cases/fixture/parallel-compose/files/docker-compose.oats.yml new file mode 100644 index 00000000..878c70fb --- /dev/null +++ b/tests/e2e/cases/fixture/parallel-compose/files/docker-compose.oats.yml @@ -0,0 +1,4 @@ +services: + app: + image: alpine:3.22 + command: ["sh", "-c", "echo app started ok && sleep 3"] diff --git a/tests/e2e/cases/fixture/parallel-compose/files/oats.toml b/tests/e2e/cases/fixture/parallel-compose/files/oats.toml new file mode 100644 index 00000000..c52ee17d --- /dev/null +++ b/tests/e2e/cases/fixture/parallel-compose/files/oats.toml @@ -0,0 +1,17 @@ +[meta] +version = 2 + +[[suite]] +name = "alpha" +cases = ["alpha.yaml"] +fixture = "shared" + +[[suite]] +name = "beta" +cases = ["beta.yaml"] +fixture = "shared" + +[fixture.shared] +type = "compose" +template = "lgtm" +compose_file = "docker-compose.oats.yml" diff --git a/tests/e2e/cases/fixture/parallel-compose/files/verify-alpha.sh b/tests/e2e/cases/fixture/parallel-compose/files/verify-alpha.sh new file mode 100644 index 00000000..65a59f76 --- /dev/null +++ b/tests/e2e/cases/fixture/parallel-compose/files/verify-alpha.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +dir="$(cd "$(dirname "$0")" && pwd)" +proof="$dir/.parallel-proof" +mkdir -p "$proof" +touch "$proof/alpha" +for _ in $(seq 1 50); do + if [ -f "$proof/beta" ]; then + exit 0 + fi + sleep 0.2 +done +echo "beta verifier never overlapped with alpha" >&2 +exit 1 diff --git a/tests/e2e/cases/fixture/parallel-compose/files/verify-beta.sh b/tests/e2e/cases/fixture/parallel-compose/files/verify-beta.sh new file mode 100644 index 00000000..982c8799 --- /dev/null +++ b/tests/e2e/cases/fixture/parallel-compose/files/verify-beta.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +dir="$(cd "$(dirname "$0")" && pwd)" +proof="$dir/.parallel-proof" +mkdir -p "$proof" +touch "$proof/beta" +for _ in $(seq 1 50); do + if [ -f "$proof/alpha" ]; then + exit 0 + fi + sleep 0.2 +done +echo "alpha verifier never overlapped with beta" >&2 +exit 1 diff --git a/tests/e2e/cases/fixture/parallel-compose/test.yaml b/tests/e2e/cases/fixture/parallel-compose/test.yaml new file mode 100644 index 00000000..2d3618fd --- /dev/null +++ b/tests/e2e/cases/fixture/parallel-compose/test.yaml @@ -0,0 +1,7 @@ +name: parallel-compose +tags: [fixture, compose, parallel] +run: + args: ["--parallel", "2"] +expect: + stdout_contains: + - PASS 2/2 diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 6d85c318..e6d79e15 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -583,14 +583,17 @@ func copyFiles(t *testing.T, srcDir, dstDir string, ph placeholders) { func markExecutables(t *testing.T, filesDir string) { t.Helper() - for _, rel := range []string{"verify.sh", "setup.sh"} { - path := filepath.Join(filesDir, rel) - if _, err := os.Stat(path); err == nil { + _ = filepath.WalkDir(filesDir, func(path string, d os.DirEntry, err error) error { + if err != nil || d == nil || d.IsDir() { + return err + } + if strings.HasSuffix(d.Name(), ".sh") { if chmodErr := os.Chmod(path, 0o755); chmodErr != nil { t.Fatalf("chmod %s: %v", path, chmodErr) } } - } + return nil + }) binDir := filepath.Join(filesDir, "bin") _ = filepath.WalkDir(binDir, func(path string, d os.DirEntry, err error) error { if err != nil || d == nil || d.IsDir() { From 8700c68c935132f6b0602373e4309d5cb331b706 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 15:27:57 +0200 Subject: [PATCH 079/124] ci: use catch-all non-k3d e2e shard Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index f07bf993..b4af42a2 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -13,7 +13,7 @@ jobs: matrix: include: - name: non-k3d - filter: "assert/,custom/,seed/,fixture/compose,fixture/parallel-compose,fixture/remote" + filter: "" - name: k3d-fail filter: "fixture/k3d-fail" - name: k3d-smoke From ec9694c8407f5ad9de9e5d02e62f194fb9ebf03b Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 15:31:20 +0200 Subject: [PATCH 080/124] test: support excluding e2e case filters Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 2 +- tests/e2e/e2e_test.go | 47 ++++++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index b4af42a2..ae4d09b1 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -13,7 +13,7 @@ jobs: matrix: include: - name: non-k3d - filter: "" + filter: "-fixture/k3d-fail,-fixture/k3d-smoke" - name: k3d-fail filter: "fixture/k3d-fail" - name: k3d-smoke diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index e6d79e15..81cfe400 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -408,12 +408,55 @@ func splitFilter(raw string) []string { } func matchesAnyFilter(path string, filters []string) bool { + if len(filters) == 0 { + return true + } + matchedInclude := false + haveInclude := false for _, f := range filters { + if strings.HasPrefix(f, "-") { + if strings.Contains(path, strings.TrimPrefix(f, "-")) { + return false + } + continue + } + haveInclude = true if strings.Contains(path, f) { - return true + matchedInclude = true } } - return false + if haveInclude { + return matchedInclude + } + return true +} + +func TestMatchesAnyFilter(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + filters []string + want bool + }{ + {name: "no filters matches all", path: "fixture/compose-logs", want: true}, + {name: "positive include match", path: "fixture/compose-logs", filters: []string{"fixture/compose"}, want: true}, + {name: "positive include miss", path: "fixture/k3d-smoke", filters: []string{"fixture/compose"}, want: false}, + {name: "exclude only keeps other cases", path: "fixture/compose-logs", filters: []string{"-fixture/k3d"}, want: true}, + {name: "exclude only drops match", path: "fixture/k3d-smoke", filters: []string{"-fixture/k3d"}, want: false}, + {name: "include plus exclude prefers exclude", path: "fixture/k3d-smoke", filters: []string{"fixture/", "-fixture/k3d"}, want: false}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := matchesAnyFilter(tc.path, tc.filters); got != tc.want { + t.Fatalf("matchesAnyFilter(%q, %v) = %v, want %v", tc.path, tc.filters, got, tc.want) + } + }) + } } func runCase(t *testing.T, root, dir string) { From 37208eba9be6bac103bc608a8d2034d43228e7b1 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 15:41:31 +0200 Subject: [PATCH 081/124] fix: address remaining review feedback Signed-off-by: Gregor Zeitlinger --- UPGRADING.md | 8 +++--- discovery/discovery.go | 8 +++--- report/ndjson.go | 2 +- report/report_test.go | 9 ++++++ wait/wait.go | 64 ++++++++++++++++++++++++++++++++++++++---- wait/wait_test.go | 8 ++++++ 6 files changed, 84 insertions(+), 15 deletions(-) diff --git a/UPGRADING.md b/UPGRADING.md index 00d8f395..7edca93e 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -32,14 +32,14 @@ All test files must now include an `oats-schema-version` tag. Full release notes: -### Add `oats-schema-version: 3` to all test files +### Add `oats-schema-version: 2` to all test files All OATS test files must now include the `oats-schema-version` field at the -top level. The current version is `3`. +top level. In `v0.6.0` the required version was `2`. ```yaml # ✅ Required in all test files -oats-schema-version: 3 +oats-schema-version: 2 docker-compose: files: @@ -70,7 +70,7 @@ points must now use the `oats-template: true` flag: ```yaml # Template file (e.g., oats-base.yaml) -oats-schema-version: 3 +oats-schema-version: 2 oats-template: true docker-compose: diff --git a/discovery/discovery.go b/discovery/discovery.go index c964170c..778e6b38 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -2,10 +2,10 @@ // concrete run plan. // // In OATS v1, the runner walked the file system for any yaml carrying -// "oats-schema-version" and ran whatever it found. the current format declares the plan up -// front: oats.toml lists suites, each suite lists cases (path globs) and the -// fixture they share. "oats list" prints the plan before "oats run" executes -// it. +// "oats-schema-version" and ran whatever it found. The current format +// declares the plan up front: oats.toml lists suites, each suite lists cases +// (path globs) and the fixture they share. "oats list" prints the plan +// before "oats run" executes it. package discovery import ( diff --git a/report/ndjson.go b/report/ndjson.go index 7e53f757..a8f2efc0 100644 --- a/report/ndjson.go +++ b/report/ndjson.go @@ -40,7 +40,7 @@ func (r *NDJSONReporter) shouldEmit(e Event) bool { case EventCasePass: return r.v >= VerbosePasses case EventGCXExec: - return r.v >= VerboseAll + return r.v >= VerboseCmd case EventFixtureStart, EventFixtureReady, EventFixtureTeardown: return r.v >= VerboseAll } diff --git a/report/report_test.go b/report/report_test.go index fdaed561..b457b181 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -166,6 +166,15 @@ func TestNDJSONReporter_EmitsFixtureLifecycleAtVerboseAll(t *testing.T) { } } +func TestNDJSONReporter_EmitsGCXExecAtVerboseCmd(t *testing.T) { + var buf bytes.Buffer + r := NewNDJSONReporter(&buf, VerboseCmd) + r.Emit(Event{Type: EventGCXExec, Cmd: "gcx logs --query x"}) + if !strings.Contains(buf.String(), `"gcx.exec"`) { + t.Fatalf("expected gcx.exec in NDJSON output:\n%s", buf.String()) + } +} + func TestSplitSource(t *testing.T) { cases := []struct { in string diff --git a/wait/wait.go b/wait/wait.go index 267396e4..6e988123 100644 --- a/wait/wait.go +++ b/wait/wait.go @@ -54,6 +54,8 @@ func Until[F any](ctx context.Context, opts Options, asserter Asserter[F]) Resul opts = withDefaults(opts) start := time.Now() deadline := start.Add(opts.Timeout) + var timer *time.Timer + defer stopTimer(timer) var last []F iter := 0 @@ -69,9 +71,7 @@ func Until[F any](ctx context.Context, opts Options, asserter Asserter[F]) Resul if ctx.Err() != nil { return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: last} } - select { - case <-time.After(opts.Interval): - case <-ctx.Done(): + if !waitForNextPoll(ctx, &timer, sleepInterval(opts.Interval, deadline)) { return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: last} } } @@ -85,6 +85,8 @@ func While[F any](ctx context.Context, opts Options, asserter Asserter[F]) Resul opts = withDefaults(opts) start := time.Now() deadline := start.Add(opts.Timeout) + var timer *time.Timer + defer stopTimer(timer) iter := 0 for { @@ -99,9 +101,7 @@ func While[F any](ctx context.Context, opts Options, asserter Asserter[F]) Resul if ctx.Err() != nil { return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} } - select { - case <-time.After(opts.Interval): - case <-ctx.Done(): + if !waitForNextPoll(ctx, &timer, sleepInterval(opts.Interval, deadline)) { return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} } } @@ -116,3 +116,55 @@ func withDefaults(o Options) Options { } return o } + +func sleepInterval(interval time.Duration, deadline time.Time) time.Duration { + remaining := time.Until(deadline) + if remaining <= 0 { + return 0 + } + if remaining < interval { + return remaining + } + return interval +} + +func waitForNextPoll(ctx context.Context, timer **time.Timer, d time.Duration) bool { + if d <= 0 { + return true + } + if *timer == nil { + *timer = time.NewTimer(d) + } else { + if !(*timer).Stop() { + select { + case <-(*timer).C: + default: + } + } + (*timer).Reset(d) + } + select { + case <-(*timer).C: + return true + case <-ctx.Done(): + if !(*timer).Stop() { + select { + case <-(*timer).C: + default: + } + } + return false + } +} + +func stopTimer(timer *time.Timer) { + if timer == nil { + return + } + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } +} diff --git a/wait/wait_test.go b/wait/wait_test.go index 1703a8cc..b454bada 100644 --- a/wait/wait_test.go +++ b/wait/wait_test.go @@ -36,6 +36,7 @@ func TestUntil_SucceedsAfterSeveralTries(t *testing.T) { } func TestUntil_FailsAtDeadline(t *testing.T) { + start := time.Now() r := Until[string](context.Background(), Options{Timeout: 30 * time.Millisecond, Interval: 5 * time.Millisecond}, func() []string { return []string{"never passes"} }) @@ -45,6 +46,9 @@ func TestUntil_FailsAtDeadline(t *testing.T) { if len(r.LastFailures) == 0 { t.Errorf("LastFailures should carry the last asserter output") } + if elapsed := time.Since(start); elapsed > 60*time.Millisecond { + t.Fatalf("Until overshot timeout too far: %s", elapsed) + } } func TestUntil_RunsAtLeastOnceEvenWithTightDeadline(t *testing.T) { @@ -64,6 +68,7 @@ func TestUntil_RunsAtLeastOnceEvenWithTightDeadline(t *testing.T) { } func TestWhile_HoldsForEntireWindow(t *testing.T) { + start := time.Now() r := While[string](context.Background(), Options{Timeout: 30 * time.Millisecond, Interval: 5 * time.Millisecond}, func() []string { return nil // never fails }) @@ -73,6 +78,9 @@ func TestWhile_HoldsForEntireWindow(t *testing.T) { if r.Iterations < 2 { t.Errorf("expected multiple polls in 30ms with 5ms interval, got %d", r.Iterations) } + if elapsed := time.Since(start); elapsed > 60*time.Millisecond { + t.Fatalf("While overshot timeout too far: %s", elapsed) + } } func TestWhile_FailsOnFirstFailure(t *testing.T) { From c3a02d1233c0141e4d3fa3b586380aa946a69c97 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 15:46:23 +0200 Subject: [PATCH 082/124] ci: simplify e2e shard names Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index ae4d09b1..76fc352d 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -11,13 +11,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - name: non-k3d - filter: "-fixture/k3d-fail,-fixture/k3d-smoke" - - name: k3d-fail - filter: "fixture/k3d-fail" - - name: k3d-smoke - filter: "fixture/k3d-smoke" + shard: [non-k3d, k3d-fail, k3d-smoke] steps: - name: Check out uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -27,7 +21,21 @@ jobs: with: version: v2026.6.14 sha256: 96ae1ef7b00a6ebbbec23ba1016d6e722f5e904966272f621d15326429e90d53 - - name: Run ${{ matrix.name }} e2e tests - env: - OATS_E2E_FILTER: ${{ matrix.filter }} - run: mise run e2e-test + - name: Run ${{ matrix.shard }} e2e tests + run: | + case "${{ matrix.shard }}" in + non-k3d) + export OATS_E2E_FILTER="-fixture/k3d-fail,-fixture/k3d-smoke" + ;; + k3d-fail) + export OATS_E2E_FILTER="fixture/k3d-fail" + ;; + k3d-smoke) + export OATS_E2E_FILTER="fixture/k3d-smoke" + ;; + *) + echo "unknown e2e shard: ${{ matrix.shard }}" >&2 + exit 1 + ;; + esac + mise run e2e-test From 1db6c01dd4bacb20271ef6bb48d9c595984802ea Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 15:52:31 +0200 Subject: [PATCH 083/124] fix: polish remaining review nits Signed-off-by: Gregor Zeitlinger --- report/report_test.go | 23 +++++++++++++++++++++++ report/text.go | 24 ++++++++++++++++++++++++ testhelpers/compose/compose.go | 17 +++++++++++++---- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/report/report_test.go b/report/report_test.go index b457b181..517594c8 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -87,6 +87,29 @@ func TestTextReporter_GHAAnnotationsWhenEnabled(t *testing.T) { } } +func TestTextReporter_GHAAnnotationsWithoutSource(t *testing.T) { + t.Setenv("GITHUB_ACTIONS", "true") + + var buf bytes.Buffer + r := NewTextReporter(&buf, VerboseDefault) + r.Emit(Event{Type: EventRunStart}) + r.Emit(Event{ + Type: EventAssertFail, + Case: "x", + Msg: "oops", + }) + r.Emit(Event{Type: EventCaseFail, Case: "x"}) + r.Emit(Event{Type: EventRunEnd}) + + out := buf.String() + if !strings.Contains(out, "::error::oops") { + t.Fatalf("expected source-free gha annotation:\n%s", out) + } + if strings.Contains(out, "::error file=") { + t.Fatalf("did not expect empty file annotation:\n%s", out) + } +} + func TestTextReporter_VerbosePassPrintsPasses(t *testing.T) { var buf bytes.Buffer r := NewTextReporter(&buf, VerbosePasses) diff --git a/report/text.go b/report/text.go index 10057808..6dd1e612 100644 --- a/report/text.go +++ b/report/text.go @@ -48,6 +48,7 @@ func NewTextReporter(w io.Writer, v Verbosity) *TextReporter { func (r *TextReporter) Emit(e Event) { switch e.Type { case EventRunStart: + r.resetRunState() r.runStart = nonZeroOrNow(e.Ts) case EventRunEnd: r.flushRunEnd(e) @@ -78,6 +79,14 @@ func (r *TextReporter) Emit(e Event) { func (r *TextReporter) Close() error { return nil } +func (r *TextReporter) resetRunState() { + r.pass = 0 + r.fail = 0 + r.skip = 0 + r.failBlocks = nil + r.knownErrAt = make(map[string]struct{}) +} + func (r *TextReporter) recordFailure(e Event) { var b strings.Builder src := e.Source @@ -102,6 +111,21 @@ func (r *TextReporter) recordFailure(e Event) { } func (r *TextReporter) emitGHAAnnotation(e Event) { + if e.Source == "" { + const key = "(unknown source):0" + if _, dup := r.knownErrAt[key]; dup { + return + } + r.knownErrAt[key] = struct{}{} + + msg := e.Msg + if msg == "" { + msg = "OATS assertion failed" + } + r.write("::error::%s\n", ghaEscape(msg)) + return + } + file, line := splitSource(e.Source) // Suppress duplicate annotations for the same source position so a case // with N substring failures does not flood the PR diff. diff --git a/testhelpers/compose/compose.go b/testhelpers/compose/compose.go index 382e6cfe..23d5d60b 100644 --- a/testhelpers/compose/compose.go +++ b/testhelpers/compose/compose.go @@ -90,20 +90,29 @@ func (c *Compose) runDocker(cc command) error { cmd := exec.Command(c.Command, cmdArgs...) cmd.Env = c.Env if cc.logConsumer != nil { - stdout, _ := cmd.StdoutPipe() + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("failed to open docker stdout pipe: %w", err) + } cmd.Stderr = cmd.Stdout wg := sync.WaitGroup{} wg.Add(1) go cc.logConsumer(stdout, &wg) - err := cmd.Start() + err = cmd.Start() if err != nil { return fmt.Errorf("failed to start docker command: %w", err) } wg.Wait() + if err := cmd.Wait(); err != nil { + return fmt.Errorf("failed to run docker command: %w", err) + } } else if cc.background { slog.Info("Running", "command", cmd.String(), "compose_files", c.Paths) - stdout, _ := cmd.StdoutPipe() + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("failed to open docker stdout pipe: %w", err) + } cmd.Stderr = cmd.Stdout wg := sync.WaitGroup{} wg.Add(1) @@ -117,7 +126,7 @@ func (c *Compose) runDocker(cc command) error { } }() - err := cmd.Start() + err = cmd.Start() if err != nil { return fmt.Errorf("failed to start docker command: %w", err) } From cbbc6646cb6dcfb875f9284de8b1251f80ac5f6d Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 15:56:58 +0200 Subject: [PATCH 084/124] fix: use dynamic local ports for k3d fixtures Signed-off-by: Gregor Zeitlinger --- internal/cli/integration_test.go | 36 +++++++- internal/cli/main.go | 82 +++++++++++++++---- testhelpers/kubernetes/kubernetes.go | 4 +- testhelpers/kubernetes/kubernetes_test.go | 8 +- .../remote/remote_observability_endpoint.go | 2 + 5 files changed, 109 insertions(+), 23 deletions(-) diff --git a/internal/cli/integration_test.go b/internal/cli/integration_test.go index 1568c648..0ace6898 100644 --- a/internal/cli/integration_test.go +++ b/internal/cli/integration_test.go @@ -172,8 +172,10 @@ func TestStartFixture_K3DLifecycle(t *testing.T) { var capturedPlan discovery.Plan var starts, stops int - newKubernetesEndpoint = func(plan discovery.Plan) *remote.Endpoint { + var capturedPorts remote.PortsConfig + newKubernetesEndpoint = func(plan discovery.Plan, ports remote.PortsConfig) *remote.Endpoint { capturedPlan = plan + capturedPorts = ports return remote.NewEndpoint("localhost", remote.PortsConfig{}, func(ctx context.Context) error { starts++ return nil @@ -206,6 +208,9 @@ func TestStartFixture_K3DLifecycle(t *testing.T) { if capturedPlan.FixtureSourceDir != "/tmp/work" || capturedPlan.Suite.Name != "cluster-smoke" || capturedPlan.Fixture.AppPort != 18080 { t.Fatalf("unexpected endpoint factory args: plan=%+v", capturedPlan) } + if capturedPorts.GrafanaHTTPPort == 0 || capturedPorts.OTLPHTTPPort == 0 || capturedPorts.LokiHttpPort == 0 || capturedPorts.PrometheusHTTPPort == 0 || capturedPorts.TempoHTTPPort == 0 || capturedPorts.PyroscopeHttpPort == 0 { + t.Fatalf("expected allocated k3d ports, got %+v", capturedPorts) + } if err := fix.Close(); err != nil { t.Fatalf("fixture close: %v", err) } @@ -218,7 +223,7 @@ func TestStartFixture_K3DStartFailure(t *testing.T) { oldFactory := newKubernetesEndpoint defer func() { newKubernetesEndpoint = oldFactory }() - newKubernetesEndpoint = func(plan discovery.Plan) *remote.Endpoint { + newKubernetesEndpoint = func(plan discovery.Plan, ports remote.PortsConfig) *remote.Endpoint { return remote.NewEndpoint("localhost", remote.PortsConfig{}, func(ctx context.Context) error { return fmt.Errorf("cluster boom") }, func(ctx context.Context) error { @@ -244,6 +249,24 @@ func TestStartFixture_K3DStartFailure(t *testing.T) { } } +func TestK3DCheckEnv_UsesConfiguredPorts(t *testing.T) { + got := k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: 18080}, remote.PortsConfig{ + GrafanaHTTPPort: 13000, + OTLPHTTPPort: 14318, + PyroscopeHttpPort: 14040, + }) + for _, want := range []string{ + "OATS_APP_URL=http://localhost:18080", + "OATS_GRAFANA_URL=http://localhost:13000", + "OATS_OTLP_HTTP=http://localhost:14318", + "OATS_PYROSCOPE_URL=http://localhost:14040", + } { + if !containsString(got, want) { + t.Fatalf("k3dCheckEnv missing %q in %v", want, got) + } + } +} + func TestResolveComposeFiles_UnsupportedTemplate(t *testing.T) { _, _, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "weird"}) if err == nil || !strings.Contains(err.Error(), `unsupported compose fixture template "weird"`) { @@ -1192,6 +1215,15 @@ func equalStrings(got, want []string) bool { return true } +func containsString(items []string, want string) bool { + for _, item := range items { + if item == want { + return true + } + } + return false +} + type recordingReporter struct { events []report.Event } diff --git a/internal/cli/main.go b/internal/cli/main.go index 62041657..9451939d 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -23,6 +23,7 @@ import ( "encoding/json" "flag" "fmt" + "net" "net/http" "os" "os/exec" @@ -53,7 +54,7 @@ var ( newComposeSuite = func(files []string, env []string) (suiteFixture, error) { return compose.SuiteFiles(files, env) } - newKubernetesEndpoint = func(plan discovery.Plan) *remote.Endpoint { + newKubernetesEndpoint = func(plan discovery.Plan, ports remote.PortsConfig) *remote.Endpoint { sourceDir := plan.FixtureSourceDir if sourceDir == "" { sourceDir = "." @@ -67,12 +68,7 @@ var ( AppDockerPort: plan.Fixture.AppPort, ImportImages: plan.Fixture.ImportImages, } - return kubernetes.NewEndpoint("localhost", model, remote.PortsConfig{ - PrometheusHTTPPort: 9090, - LokiHttpPort: 3100, - TempoHTTPPort: 3200, - PyroscopeHttpPort: 4040, - }, plan.Suite.Name, sourceDir) + return kubernetes.NewEndpoint("localhost", model, ports, plan.Suite.Name, sourceDir) } waitForGrafanaToken = waitForGrafanaTokenImpl lookupComposePort = dockerComposePort @@ -558,17 +554,21 @@ func startFixture(ctx context.Context, plan discovery.Plan) (suiteFixture, fixtu rt.ParallelSafe, rt.ParallelDisabled = planSupportsParallel(plan) return composeFixture{suite: suite, cleanup: cleanup}, rt, nil case "k3d": - ep := newKubernetesEndpoint(plan) + ports, err := allocateK3DPorts() + if err != nil { + return nil, fixtureRuntime{}, err + } + ep := newKubernetesEndpoint(plan, ports) if err := ep.Start(ctx); err != nil { return nil, fixtureRuntime{}, err } rt := fixtureRuntime{ - GrafanaURL: "http://localhost:3000", - OTLPHTTP: "http://localhost:4318", - PyroscopeURL: "http://localhost:4040", - CustomCheckEnv: k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: plan.Fixture.AppPort}), + GrafanaURL: fmt.Sprintf("http://localhost:%d", ports.GrafanaHTTPPort), + OTLPHTTP: fmt.Sprintf("http://localhost:%d", ports.OTLPHTTPPort), + PyroscopeURL: fmt.Sprintf("http://localhost:%d", ports.PyroscopeHttpPort), + CustomCheckEnv: k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: plan.Fixture.AppPort}, ports), ParallelSafe: false, - ParallelDisabled: "k3d fixtures currently use shared localhost port-forwards", + ParallelDisabled: "k3d fixtures currently use shared clusters and kubectl port-forwards", } if cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL); cfgErr == nil { rt.GCXConfig = cfg @@ -778,13 +778,13 @@ func shellQuote(s string) string { return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" } -func k3dCheckEnv(ep runner.Endpoint) []string { +func k3dCheckEnv(ep runner.Endpoint, ports remote.PortsConfig) []string { return []string{ "OATS_FIXTURE_TYPE=k3d", "OATS_APP_URL=" + fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), - "OATS_GRAFANA_URL=http://localhost:3000", - "OATS_OTLP_HTTP=http://localhost:4318", - "OATS_PYROSCOPE_URL=http://localhost:4040", + "OATS_GRAFANA_URL=" + fmt.Sprintf("http://localhost:%d", ports.GrafanaHTTPPort), + "OATS_OTLP_HTTP=" + fmt.Sprintf("http://localhost:%d", ports.OTLPHTTPPort), + "OATS_PYROSCOPE_URL=" + fmt.Sprintf("http://localhost:%d", ports.PyroscopeHttpPort), } } @@ -850,6 +850,54 @@ func planSupportsParallel(plan discovery.Plan) (bool, string) { } } +func allocateK3DPorts() (remote.PortsConfig, error) { + grafanaPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + otlpHTTPPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + lokiPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + promPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + tempoPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + pyroscopePort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + return remote.PortsConfig{ + GrafanaHTTPPort: grafanaPort, + OTLPHTTPPort: otlpHTTPPort, + LokiHttpPort: lokiPort, + PrometheusHTTPPort: promPort, + TempoHTTPPort: tempoPort, + PyroscopeHttpPort: pyroscopePort, + }, nil +} + +func findFreePort() (int, error) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer func() { _ = ln.Close() }() + addr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + return 0, fmt.Errorf("unexpected listener address %T", ln.Addr()) + } + return addr.Port, nil +} + func composeProjectName(plan discovery.Plan) string { name := strings.ToLower(plan.Suite.Name) if name == "" { diff --git a/testhelpers/kubernetes/kubernetes.go b/testhelpers/kubernetes/kubernetes.go index 2b57d727..09695183 100644 --- a/testhelpers/kubernetes/kubernetes.go +++ b/testhelpers/kubernetes/kubernetes.go @@ -123,11 +123,11 @@ func start(model *Kubernetes, ports remote.PortsConfig, testName string, run fun if err != nil { return err } - err = portForward(3000, 3000) + err = portForward(ports.GrafanaHTTPPort, 3000) if err != nil { return err } - err = portForward(4318, 4318) + err = portForward(ports.OTLPHTTPPort, 4318) if err != nil { return err } diff --git a/testhelpers/kubernetes/kubernetes_test.go b/testhelpers/kubernetes/kubernetes_test.go index a6a8f9c7..eddeef37 100644 --- a/testhelpers/kubernetes/kubernetes_test.go +++ b/testhelpers/kubernetes/kubernetes_test.go @@ -40,6 +40,8 @@ func TestStart_DefaultDockerContextAndCommandSequence(t *testing.T) { ImportImages: []string{"busybox:latest"}, } ports := remote.PortsConfig{ + GrafanaHTTPPort: 13000, + OTLPHTTPPort: 14318, PrometheusHTTPPort: 19090, LokiHttpPort: 13100, TempoHTTPPort: 13200, @@ -74,8 +76,8 @@ func TestStart_DefaultDockerContextAndCommandSequence(t *testing.T) { "fg: kubectl wait --timeout=5m --for=condition=available deployment/lgtm", "bg: kubectl port-forward service/dice 18080:8080", "bg: kubectl port-forward service/lgtm 13100:3100", - "bg: kubectl port-forward service/lgtm 3000:3000", - "bg: kubectl port-forward service/lgtm 4318:4318", + "bg: kubectl port-forward service/lgtm 13000:3000", + "bg: kubectl port-forward service/lgtm 14318:4318", "bg: kubectl port-forward service/lgtm 19090:9090", "bg: kubectl port-forward service/lgtm 13200:3200", "bg: kubectl port-forward service/lgtm 14040:4040", @@ -97,6 +99,8 @@ func TestStartWaitsForLgtmDeploymentAvailability(t *testing.T) { AppDockerPort: 8080, } ports := remote.PortsConfig{ + GrafanaHTTPPort: 3000, + OTLPHTTPPort: 4318, LokiHttpPort: 3100, PrometheusHTTPPort: 9090, TempoHTTPPort: 3200, diff --git a/testhelpers/remote/remote_observability_endpoint.go b/testhelpers/remote/remote_observability_endpoint.go index e2c489e9..c278b978 100644 --- a/testhelpers/remote/remote_observability_endpoint.go +++ b/testhelpers/remote/remote_observability_endpoint.go @@ -21,6 +21,8 @@ import ( type PortsConfig struct { TracesGRPCPort int TracesHTTPPort int + GrafanaHTTPPort int + OTLPHTTPPort int TempoHTTPPort int MimirHTTPPort int PrometheusHTTPPort int From efacd28f959225e8540573944b7c17589add0d6c Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 15:57:24 +0200 Subject: [PATCH 085/124] docs: finish review follow-up cleanup Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 1 + UPGRADING.md | 2 +- migrate/migrate.go | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 76fc352d..fc0e4450 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -22,6 +22,7 @@ jobs: version: v2026.6.14 sha256: 96ae1ef7b00a6ebbbec23ba1016d6e722f5e904966272f621d15326429e90d53 - name: Run ${{ matrix.shard }} e2e tests + # OATS_E2E_FILTER is not part of the matrix to avoid that it shows up in the gh action (too long) run: | case "${{ matrix.shard }}" in non-k3d) diff --git a/UPGRADING.md b/UPGRADING.md index 7edca93e..50207c1b 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -97,7 +97,7 @@ and want to avoid parsing all of them. ### Migration Steps -1. Add `oats-schema-version: 3` to all your test files +1. Add `oats-schema-version: 2` to all your test files 2. Add `oats-template: true` to any template files (files that are included but not entry points) 3. (Optional) Consider passing specific file paths instead of directories for diff --git a/migrate/migrate.go b/migrate/migrate.go index ffc060b0..adcb4f90 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -55,6 +55,9 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Cas c.Name = fmt.Sprintf("%s [%s]", name, selectedMatrix.Name) warnings = append(warnings, fmt.Sprintf("flattened single matrix entry %q into the migrated case", selectedMatrix.Name)) if selectedMatrix.DockerCompose != nil { + if len(selectedMatrix.DockerCompose.Files) == 0 { + return nil, warnings, fmt.Errorf("matrix docker-compose present but no files declared") + } c.Seed.Type = "app" c.Seed.Compose = selectedMatrix.DockerCompose.Files[0] warnings = append(warnings, "single matrix docker-compose fixture selected; paste the suggested [fixture] block below into oats.toml") From 40d5a1bcf86a3feb14b9e3104343e99a5c69c8f2 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 16:37:08 +0200 Subject: [PATCH 086/124] fix: address final review comments Signed-off-by: Gregor Zeitlinger --- migrate/migrate.go | 16 ++++++- migrate/migrate_test.go | 25 ++++++++-- testhelpers/kubernetes/kubernetes.go | 4 +- testhelpers/kubernetes/kubernetes_test.go | 48 +++++++++++++++++++ .../remote/remote_observability_endpoint.go | 14 ++++++ wait/wait.go | 5 +- 6 files changed, 102 insertions(+), 10 deletions(-) diff --git a/migrate/migrate.go b/migrate/migrate.go index adcb4f90..26d66f6d 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -4,6 +4,7 @@ import ( "fmt" "path/filepath" "regexp" + "sort" "strings" "github.com/grafana/oats/casefile" @@ -197,7 +198,8 @@ func convertSignal(label string, s model.ExpectedSignal) (casefile.AssertionComm if s.NameEquals != "" { entry.Name = strPtr(s.NameEquals) } - for k, v := range s.Attributes { + for _, k := range sortedMapKeys(s.Attributes) { + v := s.Attributes[k] entry.Attributes = append(entry.Attributes, casefile.AttributeMatcher{Key: k, Value: strPtr(v)}) } out.Match = append(out.Match, entry) @@ -207,7 +209,8 @@ func convertSignal(label string, s model.ExpectedSignal) (casefile.AssertionComm if s.NameRegexp != "" { entry.Name = strPtr(s.NameRegexp) } - for k, v := range s.AttributeRegexp { + for _, k := range sortedMapKeys(s.AttributeRegexp) { + v := s.AttributeRegexp[k] if v == ".*" { entry.Attributes = append(entry.Attributes, casefile.AttributeMatcher{Key: k}) } else { @@ -224,6 +227,15 @@ func convertSignal(label string, s model.ExpectedSignal) (casefile.AssertionComm return out, warnings } +func sortedMapKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + func withoutMatch(a casefile.AssertionCommon) casefile.AssertionCommon { a.Match = nil return a diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index e141fea1..9a081837 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -22,8 +22,8 @@ func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { TraceQL: `{ name = "GET /stock" }`, Signal: model.ExpectedSignal{ NameEquals: "GET /stock", - Attributes: map[string]string{"db.system": "h2"}, - AttributeRegexp: map[string]string{"trace_id": ".*"}, + Attributes: map[string]string{"service.name": "shop", "db.system": "h2"}, + AttributeRegexp: map[string]string{"trace_id": ".*", "span.kind": "server"}, }, }}, Logs: []model.ExpectedLogs{{ @@ -62,7 +62,13 @@ func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { if len(c.Expected.Traces) != 1 || len(c.Expected.Traces[0].MatchSpans) != 2 { fatalf(t, "expected trace strict+regexp split, got %+v", c.Expected.Traces) } - if got := c.Expected.Traces[0].MatchSpans[1].Attributes[0]; got.Key != "trace_id" || got.Value != nil { + if got := c.Expected.Traces[0].MatchSpans[0].Attributes; len(got) != 2 || got[0].Key != "db.system" || got[1].Key != "service.name" { + fatalf(t, "expected strict attributes sorted, got %+v", got) + } + if got := c.Expected.Traces[0].MatchSpans[1].Attributes[0]; got.Key != "span.kind" || got.Value == nil || *got.Value != "server" { + fatalf(t, "expected regexp attributes sorted, got %+v", c.Expected.Traces[0].MatchSpans[1].Attributes) + } + if got := c.Expected.Traces[0].MatchSpans[1].Attributes[1]; got.Key != "trace_id" || got.Value != nil { fatalf(t, "expected trace_id .* to map to presence, got %+v", got) } if c.Expected.Logs[0].Count != ">= 1" { @@ -80,6 +86,19 @@ func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { } } +func TestConvertDefinition_SingleMatrixComposeRequiresFile(t *testing.T) { + def := model.TestCaseDefinition{ + Matrix: []model.Matrix{{ + Name: "docker", + DockerCompose: &model.DockerCompose{}, + }}, + } + _, _, err := ConvertDefinition(def, "matrix no files") + if err == nil || !strings.Contains(err.Error(), "matrix docker-compose present but no files declared") { + fatalf(t, "expected helpful matrix file error, got %v", err) + } +} + func TestConvertFile_RendersYAML(t *testing.T) { sample := filepath.Join("..", "yaml", "testdata", "valid-tests", "oats.yaml") out, warnings, err := ConvertFile(sample) diff --git a/testhelpers/kubernetes/kubernetes.go b/testhelpers/kubernetes/kubernetes.go index 09695183..be2554e0 100644 --- a/testhelpers/kubernetes/kubernetes.go +++ b/testhelpers/kubernetes/kubernetes.go @@ -123,11 +123,11 @@ func start(model *Kubernetes, ports remote.PortsConfig, testName string, run fun if err != nil { return err } - err = portForward(ports.GrafanaHTTPPort, 3000) + err = portForward(ports.EffectiveGrafanaHTTPPort(), 3000) if err != nil { return err } - err = portForward(ports.OTLPHTTPPort, 4318) + err = portForward(ports.EffectiveOTLPHTTPPort(), 4318) if err != nil { return err } diff --git a/testhelpers/kubernetes/kubernetes_test.go b/testhelpers/kubernetes/kubernetes_test.go index eddeef37..f1e158da 100644 --- a/testhelpers/kubernetes/kubernetes_test.go +++ b/testhelpers/kubernetes/kubernetes_test.go @@ -124,3 +124,51 @@ func TestStartWaitsForLgtmDeploymentAvailability(t *testing.T) { "deployment/lgtm", }) } + +func TestStart_FallsBackToLegacyGrafanaAndOTLPPorts(t *testing.T) { + t.Parallel() + + model := &Kubernetes{ + Dir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerContext: ".", + AppDockerTag: "dice:test", + AppDockerPort: 18080, + } + ports := remote.PortsConfig{ + LokiHttpPort: 3100, + PrometheusHTTPPort: 9090, + TempoHTTPPort: 3200, + PyroscopeHttpPort: 4040, + } + + var calls []string + run := func(cmd *exec.Cmd, background bool) error { + mode := "fg" + if background { + mode = "bg" + } + calls = append(calls, mode+": "+strings.Join(cmd.Args, " ")) + return nil + } + + if err := start(model, ports, "legacy-ports", run); err != nil { + t.Fatalf("start: %v", err) + } + if !contains(calls, "bg: kubectl port-forward service/lgtm 3000:3000") { + t.Fatalf("expected Grafana legacy fallback port-forward, got %#v", calls) + } + if !contains(calls, "bg: kubectl port-forward service/lgtm 4318:4318") { + t.Fatalf("expected OTLP legacy fallback port-forward, got %#v", calls) + } +} + +func contains(items []string, want string) bool { + for _, item := range items { + if item == want { + return true + } + } + return false +} diff --git a/testhelpers/remote/remote_observability_endpoint.go b/testhelpers/remote/remote_observability_endpoint.go index c278b978..7bd023dc 100644 --- a/testhelpers/remote/remote_observability_endpoint.go +++ b/testhelpers/remote/remote_observability_endpoint.go @@ -30,6 +30,20 @@ type PortsConfig struct { PyroscopeHttpPort int } +func (p PortsConfig) EffectiveGrafanaHTTPPort() int { + if p.GrafanaHTTPPort != 0 { + return p.GrafanaHTTPPort + } + return 3000 +} + +func (p PortsConfig) EffectiveOTLPHTTPPort() int { + if p.OTLPHTTPPort != 0 { + return p.OTLPHTTPPort + } + return 4318 +} + type Endpoint struct { host string ports PortsConfig diff --git a/wait/wait.go b/wait/wait.go index 6e988123..7407f228 100644 --- a/wait/wait.go +++ b/wait/wait.go @@ -37,9 +37,8 @@ type Options struct { } // Result is what Until and While return. Iterations counts how many poll -// attempts ran; LastFailures is the most recent failure set observed (empty -// on Until success, possibly populated on While success when the asserter -// reports transient failures the caller chose to ignore). +// attempts ran; LastFailures is the most recent failure set observed (nil on +// success, populated on failure). type Result[F any] struct { OK bool Iterations int From 4883ded0557233c7f69a48dafc3b86d0c14e4f13 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 16:44:35 +0200 Subject: [PATCH 087/124] ci: split core and fixture e2e shards Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index fc0e4450..436be18c 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [non-k3d, k3d-fail, k3d-smoke] + shard: [core, fixtures, k3d-fail, k3d-smoke] steps: - name: Check out uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -25,8 +25,11 @@ jobs: # OATS_E2E_FILTER is not part of the matrix to avoid that it shows up in the gh action (too long) run: | case "${{ matrix.shard }}" in - non-k3d) - export OATS_E2E_FILTER="-fixture/k3d-fail,-fixture/k3d-smoke" + core) + export OATS_E2E_FILTER="assert/,custom/,seed/" + ;; + fixtures) + export OATS_E2E_FILTER="fixture/compose,fixture/parallel-compose,fixture/remote" ;; k3d-fail) export OATS_E2E_FILTER="fixture/k3d-fail" From 21f12b09f05a7c3ac89acdbe54375ab12fbd71f3 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Fri, 3 Jul 2026 16:47:32 +0200 Subject: [PATCH 088/124] ci: use catch-all e2e shard filters Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 436be18c..e902a5a6 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -26,10 +26,10 @@ jobs: run: | case "${{ matrix.shard }}" in core) - export OATS_E2E_FILTER="assert/,custom/,seed/" + export OATS_E2E_FILTER="-fixture/" ;; fixtures) - export OATS_E2E_FILTER="fixture/compose,fixture/parallel-compose,fixture/remote" + export OATS_E2E_FILTER="fixture/,-fixture/k3d-fail,-fixture/k3d-smoke" ;; k3d-fail) export OATS_E2E_FILTER="fixture/k3d-fail" From 2427100dc1076a25387702827ac3dd91ce7b799a Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 08:24:44 +0000 Subject: [PATCH 089/124] ci: split compose and remote e2e shards Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index e902a5a6..a1f9ffec 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [core, fixtures, k3d-fail, k3d-smoke] + shard: [core, compose, remote, k3d-fail, k3d-smoke] steps: - name: Check out uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -28,8 +28,11 @@ jobs: core) export OATS_E2E_FILTER="-fixture/" ;; - fixtures) - export OATS_E2E_FILTER="fixture/,-fixture/k3d-fail,-fixture/k3d-smoke" + compose) + export OATS_E2E_FILTER="fixture/compose" + ;; + remote) + export OATS_E2E_FILTER="fixture/remote" ;; k3d-fail) export OATS_E2E_FILTER="fixture/k3d-fail" From 171daed22adf5a013d90178e7ab9f594894713ce Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 08:47:12 +0000 Subject: [PATCH 090/124] ci: log e2e setup timings Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 4 ++++ tests/e2e/e2e_test.go | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 39a444f9..780b1dc6 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -24,6 +24,7 @@ jobs: - name: Run ${{ matrix.shard }} e2e tests # OATS_E2E_FILTER is not part of the matrix to avoid that it shows up in the gh action (too long) run: | + start_ts="$(date +%s)" case "${{ matrix.shard }}" in core) export OATS_E2E_FILTER="-fixture/" @@ -45,4 +46,7 @@ jobs: exit 1 ;; esac + echo "e2e timing: shard=${{ matrix.shard }} filter=${OATS_E2E_FILTER}" mise run e2e-test + end_ts="$(date +%s)" + echo "e2e timing: shard=${{ matrix.shard }} total=$((end_ts-start_ts))s" diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 81cfe400..7fb0626d 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -79,11 +79,13 @@ func TestMain(m *testing.M) { } shared.RepoRoot = root if runtime.GOOS != "windows" { + start := time.Now() if err := prepareLocalTools(&shared); err != nil { fmt.Fprintf(os.Stderr, "e2e setup failed: %v\n", err) _ = teardownSharedEnv(&shared) os.Exit(1) } + fmt.Fprintf(os.Stderr, "e2e timing: prepareLocalTools finished in %s\n", time.Since(start).Round(time.Millisecond)) } code := m.Run() if runtime.GOOS != "windows" { @@ -162,6 +164,8 @@ func TestCases(t *testing.T) { } func setupSharedEnv(env *sharedEnv) error { + start := time.Now() + fmt.Fprintln(os.Stderr, "e2e timing: setting up shared remote LGTM env") if _, err := exec.LookPath("docker"); err != nil { return err } @@ -183,9 +187,11 @@ func setupSharedEnv(env *sharedEnv) error { up := exec.Command("docker", "compose", "-p", env.Project, "-f", env.ComposeFile, "up", "-d") up.Stdout = os.Stdout up.Stderr = os.Stderr + upStart := time.Now() if err := up.Run(); err != nil { return fmt.Errorf("start shared lgtm: %w", err) } + fmt.Fprintf(os.Stderr, "e2e timing: docker compose up for shared LGTM finished in %s\n", time.Since(upStart).Round(time.Millisecond)) grafanaPort, err := dockerComposePort(env.Project, env.ComposeFile, "lgtm", "3000") if err != nil { return err @@ -199,12 +205,15 @@ func setupSharedEnv(env *sharedEnv) error { if err := writeGCXConfig(env.GCXConfig, env.GrafanaURL); err != nil { return err } + waitStart := time.Now() if err := waitForHTTP(env.GrafanaURL+"/api/health", 2*time.Minute); err != nil { return err } if err := waitForHTTP(env.RemoteOTLPHTTP, 2*time.Minute); err != nil { return err } + fmt.Fprintf(os.Stderr, "e2e timing: shared LGTM health checks finished in %s\n", time.Since(waitStart).Round(time.Millisecond)) + fmt.Fprintf(os.Stderr, "e2e timing: shared remote LGTM env ready in %s\n", time.Since(start).Round(time.Millisecond)) return nil } @@ -218,6 +227,8 @@ func prepareLocalTools(env *sharedEnv) error { } env.TempDir = tmp binDir := filepath.Join(tmp, "bin") + start := time.Now() + fmt.Fprintf(os.Stderr, "e2e timing: building local tools into %s\n", binDir) build := exec.Command("bash", "-lc", fmt.Sprintf("./scripts/build-local-tools.sh %q", binDir)) build.Dir = env.RepoRoot build.Stdout = os.Stdout @@ -225,6 +236,7 @@ func prepareLocalTools(env *sharedEnv) error { if err := build.Run(); err != nil { return fmt.Errorf("build local tools: %w", err) } + fmt.Fprintf(os.Stderr, "e2e timing: build-local-tools finished in %s\n", time.Since(start).Round(time.Millisecond)) env.OATS = filepath.Join(binDir, "oats") env.GCX = filepath.Join(binDir, "gcx") env.GCXConfig = filepath.Join(tmp, "gcx.yaml") From 19bd0e058c8b8a1599d16661bed45b3afd58e6d4 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 08:56:05 +0000 Subject: [PATCH 091/124] ci: run e2e tests in verbose mode Signed-off-by: Gregor Zeitlinger --- mise.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mise.toml b/mise.toml index 2fad9b4b..0d4f62bf 100644 --- a/mise.toml +++ b/mise.toml @@ -38,7 +38,7 @@ run = 'GOFLAGS=-buildvcs=false go test $(GOFLAGS=-buildvcs=false go list ./... | [tasks.e2e-test] description = "Run e2e case suite" -run = "go test -buildvcs=false ./tests/e2e -run TestCases" +run = "go test -v -buildvcs=false ./tests/e2e -run TestCases" [tasks.build] description = "Build the project" From 810ff3cf4d7e63f62c898f7244bc977ababdf7af Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 10:36:24 +0000 Subject: [PATCH 092/124] ci: reuse prebuilt e2e tools across shards Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 30 ++++++++++++++++++++++++++++-- tests/e2e/e2e_test.go | 13 +++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 780b1dc6..694114f7 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -1,4 +1,3 @@ ---- name: E2E tests on: [pull_request] @@ -6,7 +5,28 @@ on: [pull_request] permissions: {} jobs: + build-e2e-tools: + runs-on: ubuntu-24.04 + steps: + - name: Check out + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + with: + version: v2026.7.0 + sha256: 0744cb3c303baf0d308ff7b112ed41f22abb6029cb5644fd3a8ce74b29f16a68 + - name: Build e2e tools + run: ./scripts/build-local-tools.sh .e2e-tools/bin + - name: Upload e2e tools + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.6.2 + with: + name: e2e-tools + path: .e2e-tools/bin + if-no-files-found: error + test-e2e: + needs: build-e2e-tools runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -21,6 +41,11 @@ jobs: with: version: v2026.7.0 sha256: 0744cb3c303baf0d308ff7b112ed41f22abb6029cb5644fd3a8ce74b29f16a68 + - name: Download e2e tools + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: e2e-tools + path: .e2e-tools/bin - name: Run ${{ matrix.shard }} e2e tests # OATS_E2E_FILTER is not part of the matrix to avoid that it shows up in the gh action (too long) run: | @@ -46,7 +71,8 @@ jobs: exit 1 ;; esac - echo "e2e timing: shard=${{ matrix.shard }} filter=${OATS_E2E_FILTER}" + export OATS_E2E_BIN_DIR="$PWD/.e2e-tools/bin" + echo "e2e timing: shard=${{ matrix.shard }} filter=${OATS_E2E_FILTER} bin_dir=${OATS_E2E_BIN_DIR}" mise run e2e-test end_ts="$(date +%s)" echo "e2e timing: shard=${{ matrix.shard }} total=$((end_ts-start_ts))s" diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 7fb0626d..1d172028 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -227,6 +227,19 @@ func prepareLocalTools(env *sharedEnv) error { } env.TempDir = tmp binDir := filepath.Join(tmp, "bin") + if prebuiltDir := os.Getenv("OATS_E2E_BIN_DIR"); prebuiltDir != "" { + oats := filepath.Join(prebuiltDir, "oats") + gcx := filepath.Join(prebuiltDir, "gcx") + if _, err := os.Stat(oats); err == nil { + if _, err := os.Stat(gcx); err == nil { + fmt.Fprintf(os.Stderr, "e2e timing: reusing prebuilt tools from %s\n", prebuiltDir) + env.OATS = oats + env.GCX = gcx + env.GCXConfig = filepath.Join(tmp, "gcx.yaml") + return nil + } + } + } start := time.Now() fmt.Fprintf(os.Stderr, "e2e timing: building local tools into %s\n", binDir) build := exec.Command("bash", "-lc", fmt.Sprintf("./scripts/build-local-tools.sh %q", binDir)) From 031104950c468f391abbde8fabf51a0296ecb76b Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 10:46:09 +0000 Subject: [PATCH 093/124] ci: use absolute path for prebuilt e2e tools Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 694114f7..e7e5d705 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -17,7 +17,7 @@ jobs: version: v2026.7.0 sha256: 0744cb3c303baf0d308ff7b112ed41f22abb6029cb5644fd3a8ce74b29f16a68 - name: Build e2e tools - run: ./scripts/build-local-tools.sh .e2e-tools/bin + run: ./scripts/build-local-tools.sh "$PWD/.e2e-tools/bin" - name: Upload e2e tools uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.6.2 with: From 38a4df0e9ddc84efa2a4879079de975b11d6eb01 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 10:53:17 +0000 Subject: [PATCH 094/124] ci: restore e2e tool execute bits Signed-off-by: Gregor Zeitlinger --- .github/workflows/e2e_test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index e7e5d705..ad6f1b31 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -46,6 +46,8 @@ jobs: with: name: e2e-tools path: .e2e-tools/bin + - name: Restore e2e tool permissions + run: chmod +x .e2e-tools/bin/oats .e2e-tools/bin/gcx - name: Run ${{ matrix.shard }} e2e tests # OATS_E2E_FILTER is not part of the matrix to avoid that it shows up in the gh action (too long) run: | From c3021bdb3753670be506481b7c8a8ed50f0ea8a3 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 12:00:12 +0000 Subject: [PATCH 095/124] fix: clean up local gcx config temp file and address review nits - internal/cli: remove per-run oats-gcx-*.yaml temp config on fixture teardown; fail loudly instead of swallowing the write error - compose: start docker before the log-consumer goroutine so a failed Start can't leak a blocked reader - wait: capture timer by reference in cleanup defer; fix LastFailures doc - engine: correct Execute doc (--config/--context prepend order) - tests: relax wait overshoot bounds for CI, explicit SeedSettleDelay units, fix trimmed-vs-untrimmed engine assertions, rename cache test - docs: fix `oats -v` example (verbosity is -v=N) Signed-off-by: Gregor Zeitlinger --- README.md | 2 +- engine/engine.go | 7 ++-- engine/engine_test.go | 8 ++--- internal/cli/main.go | 58 +++++++++++++++++++++++++++----- runner/cache_integration_test.go | 13 ++++--- testhelpers/compose/compose.go | 11 +++--- wait/wait.go | 9 ++--- wait/wait_test.go | 8 +++-- 8 files changed, 84 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index b1a17d31..0dab1a13 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ oats --config oats.toml --tags traces,logs oats --config oats.toml --gcx-context my-lgtm oats --config oats.toml --no-cache oats --format ndjson -oats -v +oats -v=1 oats -v=2 oats -v=3 ``` diff --git a/engine/engine.go b/engine/engine.go index 92c6bb69..ac587ddb 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -63,9 +63,10 @@ type GCX struct { Timeout time.Duration } -// Execute runs gcx with args. The --context flag is prepended automatically -// when GCX.Context is set, so callers pass the verb and its flags (e.g. -// "traces", "search", "--since=10m", "{ ... }"). +// Execute runs gcx with args. When set, GCX.Config and GCX.Context are +// prepended automatically (as "--config --context "), so callers +// pass just the verb and its flags (e.g. "traces", "search", "--since=10m", +// "{ ... }"). func (g *GCX) Execute(ctx context.Context, args ...string) (*Result, error) { if g.Binary == "" { return nil, fmt.Errorf("engine: gcx binary path is empty") diff --git a/engine/engine_test.go b/engine/engine_test.go index 2db75f29..62623e2b 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -16,8 +16,8 @@ func TestGCX_ExecuteCapturesStdout(t *testing.T) { if r.ExitCode != 0 { t.Errorf("ExitCode: got %d, want 0", r.ExitCode) } - if strings.TrimSpace(r.Stdout) != "hello" { - t.Errorf("Stdout: got %q, want %q", r.Stdout, "hello\n") + if got := strings.TrimSpace(r.Stdout); got != "hello" { + t.Errorf("Stdout: got %q, want %q", got, "hello") } } @@ -27,8 +27,8 @@ func TestGCX_ExecuteCapturesStderr(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if strings.TrimSpace(r.Stderr) != "oops" { - t.Errorf("Stderr: got %q, want %q", r.Stderr, "oops\n") + if got := strings.TrimSpace(r.Stderr); got != "oops" { + t.Errorf("Stderr: got %q, want %q", got, "oops") } } diff --git a/internal/cli/main.go b/internal/cli/main.go index 9451939d..e2982335 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -13,7 +13,7 @@ // --gcx Path to gcx binary (default "gcx" on PATH) // --list Print the run plan and exit (no execution) // --format Output format: "text" (default) or "ndjson" -// -v / -v=2 / -v=3 Progressive verbosity (passes / commands / lifecycle) +// -v=1 / -v=2 / -v=3 Progressive verbosity (passes / commands / lifecycle) // --suite Comma-separated suite names to include // --tags Comma-separated tag any-match filter package cli @@ -547,9 +547,15 @@ func startFixture(ctx context.Context, plan discovery.Plan) (suiteFixture, fixtu ComposeFiles: composeFiles, ComposeProject: project, } - if cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL); cfgErr == nil { - rt.GCXConfig = cfg + cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL) + if cfgErr != nil { + if cleanup != nil { + _ = cleanup() + } + return nil, fixtureRuntime{}, fmt.Errorf("write local gcx config: %w", cfgErr) } + rt.GCXConfig = cfg + cleanup = chainCleanup(func() error { return removeIfExists(cfg) }, cleanup) rt.CustomCheckEnv = composeCheckEnv(plan, rt) rt.ParallelSafe, rt.ParallelDisabled = planSupportsParallel(plan) return composeFixture{suite: suite, cleanup: cleanup}, rt, nil @@ -570,10 +576,13 @@ func startFixture(ctx context.Context, plan discovery.Plan) (suiteFixture, fixtu ParallelSafe: false, ParallelDisabled: "k3d fixtures currently use shared clusters and kubectl port-forwards", } - if cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL); cfgErr == nil { - rt.GCXConfig = cfg + cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL) + if cfgErr != nil { + _ = ep.Stop(context.Background()) + return nil, fixtureRuntime{}, fmt.Errorf("write local gcx config: %w", cfgErr) } - return endpointFixture{ep: ep}, rt, nil + rt.GCXConfig = cfg + return endpointFixture{ep: ep, cleanup: func() error { return removeIfExists(cfg) }}, rt, nil default: return nil, fixtureRuntime{}, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) } @@ -657,11 +666,18 @@ func closeFixture(rep report.Reporter, plan discovery.Plan, fix suiteFixture) er } type endpointFixture struct { - ep *remote.Endpoint + ep *remote.Endpoint + cleanup func() error } func (e endpointFixture) Close() error { - return e.ep.Stop(context.Background()) + err := e.ep.Stop(context.Background()) + if e.cleanup != nil { + if cleanupErr := e.cleanup(); cleanupErr != nil && err == nil { + err = cleanupErr + } + } + return err } type composeFixture struct { @@ -713,6 +729,32 @@ func writeBuiltinLGTMCompose(sourceDir string) (string, error) { func grafanaURL() string { return "http://localhost:3000" } +// removeIfExists deletes path, ignoring a not-exist error so cleanup is +// idempotent. +func removeIfExists(path string) error { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// chainCleanup returns a cleanup that runs first then second, returning the +// first non-nil error. Either argument may be nil. +func chainCleanup(first, second func() error) func() error { + return func() error { + var err error + if first != nil { + err = first() + } + if second != nil { + if e := second(); e != nil && err == nil { + err = e + } + } + return err + } +} + func writeLocalGCXConfig(grafanaURL string) (string, error) { cfg := fmt.Sprintf(`current-context: local contexts: diff --git a/runner/cache_integration_test.go b/runner/cache_integration_test.go index 1e551f64..b0849b50 100644 --- a/runner/cache_integration_test.go +++ b/runner/cache_integration_test.go @@ -47,7 +47,7 @@ func TestCache_HitShortCircuits(t *testing.T) { r := New(exec, rep, Endpoint{GCXContext: "test"}, Options{ Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, - SeedSettleDelay: 1, + SeedSettleDelay: time.Nanosecond, }) store, _ := cache.New(t.TempDir(), 0, nil) @@ -70,7 +70,7 @@ func TestCache_HitShortCircuits(t *testing.T) { r2 := New(exec, rep, Endpoint{GCXContext: "test"}, Options{ Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, - SeedSettleDelay: 1, + SeedSettleDelay: time.Nanosecond, }).WithCache(store, CacheContext{GCXVersion: "v1", OatsVersion: "v2"}) rep.Emit(report.Event{Type: report.EventRunStart}) @@ -88,7 +88,7 @@ func TestCache_HitShortCircuits(t *testing.T) { } } -func TestCache_FailureEvictsStaleEntry(t *testing.T) { +func TestCache_FailingRunLeavesNoGreenRecord(t *testing.T) { c, _ := cachedRunnerCase(t) cacheDir := t.TempDir() store, _ := cache.New(cacheDir, 0, nil) @@ -103,14 +103,17 @@ func TestCache_FailureEvictsStaleEntry(t *testing.T) { t.Fatal(err) } - // Now the case fails. The runner must evict so the next run is honest. + // After the cached (green) run, invalidate the entry and rerun with output + // that fails the assertion. This asserts the failing run itself never + // records a green entry (the manual Evict below sets up the cache miss; it + // does not test runner-driven eviction). exec := &stubExec{stdout: "wrong content"} var buf bytes.Buffer rep := report.NewTextReporter(&buf, report.VerboseDefault) r := New(exec, rep, Endpoint{GCXContext: "test"}, Options{ Timeout: 30 * time.Millisecond, Interval: 5 * time.Millisecond, - SeedSettleDelay: 1, + SeedSettleDelay: time.Nanosecond, }).WithCache(store, CacheContext{GCXVersion: "v1", OatsVersion: "v2"}) rep.Emit(report.Event{Type: report.EventRunStart}) diff --git a/testhelpers/compose/compose.go b/testhelpers/compose/compose.go index 23d5d60b..f0f08d46 100644 --- a/testhelpers/compose/compose.go +++ b/testhelpers/compose/compose.go @@ -95,14 +95,15 @@ func (c *Compose) runDocker(cc command) error { return fmt.Errorf("failed to open docker stdout pipe: %w", err) } cmd.Stderr = cmd.Stdout + // Start before spawning the consumer: if Start fails the write end of + // the pipe never opens, so a consumer started earlier would block on + // the read forever and leak. + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start docker command: %w", err) + } wg := sync.WaitGroup{} wg.Add(1) go cc.logConsumer(stdout, &wg) - - err = cmd.Start() - if err != nil { - return fmt.Errorf("failed to start docker command: %w", err) - } wg.Wait() if err := cmd.Wait(); err != nil { return fmt.Errorf("failed to run docker command: %w", err) diff --git a/wait/wait.go b/wait/wait.go index 7407f228..20838ce6 100644 --- a/wait/wait.go +++ b/wait/wait.go @@ -37,8 +37,9 @@ type Options struct { } // Result is what Until and While return. Iterations counts how many poll -// attempts ran; LastFailures is the most recent failure set observed (nil on -// success, populated on failure). +// attempts ran; LastFailures is the most recent failure set observed — nil on +// success, and also nil when the run is cancelled or stopped early before +// observing a failure; populated when an assertion actually failed. type Result[F any] struct { OK bool Iterations int @@ -54,7 +55,7 @@ func Until[F any](ctx context.Context, opts Options, asserter Asserter[F]) Resul start := time.Now() deadline := start.Add(opts.Timeout) var timer *time.Timer - defer stopTimer(timer) + defer func() { stopTimer(timer) }() var last []F iter := 0 @@ -85,7 +86,7 @@ func While[F any](ctx context.Context, opts Options, asserter Asserter[F]) Resul start := time.Now() deadline := start.Add(opts.Timeout) var timer *time.Timer - defer stopTimer(timer) + defer func() { stopTimer(timer) }() iter := 0 for { diff --git a/wait/wait_test.go b/wait/wait_test.go index b454bada..9556bb49 100644 --- a/wait/wait_test.go +++ b/wait/wait_test.go @@ -46,7 +46,9 @@ func TestUntil_FailsAtDeadline(t *testing.T) { if len(r.LastFailures) == 0 { t.Errorf("LastFailures should carry the last asserter output") } - if elapsed := time.Since(start); elapsed > 60*time.Millisecond { + // Generous upper bound: guards against a runaway loop that ignores the + // deadline (which would be seconds), while tolerating CI scheduler jitter. + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { t.Fatalf("Until overshot timeout too far: %s", elapsed) } } @@ -78,7 +80,9 @@ func TestWhile_HoldsForEntireWindow(t *testing.T) { if r.Iterations < 2 { t.Errorf("expected multiple polls in 30ms with 5ms interval, got %d", r.Iterations) } - if elapsed := time.Since(start); elapsed > 60*time.Millisecond { + // Generous upper bound: guards against a runaway loop that ignores the + // deadline (which would be seconds), while tolerating CI scheduler jitter. + if elapsed := time.Since(start); elapsed > 500*time.Millisecond { t.Fatalf("While overshot timeout too far: %s", elapsed) } } From d2265a68929fba9bbca29e8feab8b90611214256 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 13:15:14 +0000 Subject: [PATCH 096/124] feat: add --fail-fast, metrics/value e2e coverage, and v2 assertion docs - cli: --fail-fast stops scheduling cases after the first failure - e2e: metrics+value pass/fail cases; fail-fast two-case suite - docs: README assertion reference, seed modes, custom-check contract; UPGRADING schema v2->v3 mappings; align example to attribute-list form Signed-off-by: Gregor Zeitlinger --- README.md | 146 ++++++++++++++++++ UPGRADING.md | 77 +++++++++ examples/smoke/cases/inline-seed.yaml | 3 +- internal/cli/main.go | 15 +- .../cases/assert/value-fail/files/oats.yaml | 16 ++ tests/e2e/cases/assert/value-fail/test.yaml | 14 ++ tests/e2e/cases/assert/value/files/oats.yaml | 15 ++ tests/e2e/cases/assert/value/test.yaml | 13 ++ .../cases/cli/fail-fast/files/01-fail.yaml | 13 ++ .../cases/cli/fail-fast/files/02-pass.yaml | 13 ++ tests/e2e/cases/cli/fail-fast/files/oats.toml | 11 ++ tests/e2e/cases/cli/fail-fast/test.yaml | 19 +++ 12 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/cases/assert/value-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/value-fail/test.yaml create mode 100644 tests/e2e/cases/assert/value/files/oats.yaml create mode 100644 tests/e2e/cases/assert/value/test.yaml create mode 100644 tests/e2e/cases/cli/fail-fast/files/01-fail.yaml create mode 100644 tests/e2e/cases/cli/fail-fast/files/02-pass.yaml create mode 100644 tests/e2e/cases/cli/fail-fast/files/oats.toml create mode 100644 tests/e2e/cases/cli/fail-fast/test.yaml diff --git a/README.md b/README.md index 0dab1a13..648c9960 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ oats --config oats.toml --suite smoke oats --config oats.toml --tags traces,logs oats --config oats.toml --gcx-context my-lgtm oats --config oats.toml --no-cache +oats --config oats.toml --fail-fast oats --format ndjson oats -v=1 oats -v=2 @@ -70,6 +71,7 @@ Key flags: - `--interval` - `--absent-timeout` - `--parallel` +- `--fail-fast` — stop scheduling further cases after the first case failure - `--gcx` - `--gcx-context` - `--version` @@ -118,6 +120,150 @@ expected: - script: ./verify.sh ``` +## Seed + +A case populates the stack before assertions run via one of two `seed.type` +modes: + +```yaml +# App-backed: the suite fixture boots an instrumented app, and `input` +# requests drive it so it emits telemetry. +seed: + type: app +input: + - path: /rolldice?rolls=5 # method defaults to GET +``` + +```yaml +# Inline-OTLP: the case carries its own payload, pushed as OTLP/HTTP JSON. +# No app, no SDK. Declare only the signals you need. +seed: + type: inline-otlp + traces: + - service: my-service + spans: + - name: seed-operation # kind defaults to INTERNAL, duration to 200ms + logs: + - service: my-service + body: seed-log-line # severity_number defaults to 9 (INFO) + metrics: + - service: my-service + name: seed_counter # monotonic sum → PromQL `seed_counter_total` + value: 42 +``` + +> Inline-OTLP can seed traces, logs, and metrics. **Profiles cannot be +> inline-seeded** — assert profiles against an app-backed fixture that produces +> them (e.g. an eBPF profiler or a pyroscope-instrumented app). + +## Assertions + +Every signal block under `expected` shares one assertion vocabulary, plus a +few signal-specific keys. Each entry first names a query, then the checks that +must hold against its result. + +Shared keys (valid on `traces`, `metrics`, `logs`, `profiles`): + +| Key | Meaning | +|-----|---------| +| `contains` | string (or list) that must appear in the query output | +| `not_contains` | string (or list) that must **not** appear | +| `regex` | RE2 pattern (or list) that must match the output | +| `match` | structural row match — list of `{match_type, name, attributes}` | +| `count` | comparison against the number of rows, e.g. `'== 1'`, `'>= 2'` | +| `absent` | the query must return nothing for the whole `--absent-timeout` window | + +`match` (and the trace-only `match_spans`) entries: + +```yaml +match: + - match_type: strict # "strict" (default) or "regexp" + name: seed-log-line # for regexp, an RE2 pattern + attributes: # optional; list of {key, value?} + - key: service_name + value: my-service # omit `value` to assert the key is merely present +``` + +Signal-specific keys: + +- `traces`: `traceql` (required), `match_spans` (span-row match, same shape as `match`) +- `metrics`: `promql` (required), `value` (compare the sample value, e.g. `'>= 1'`, `'== 42'`) +- `logs`: `logql` (required) +- `profiles`: `query` (required) + +Example covering several shapes: + +```yaml +expected: + traces: + - traceql: '{ resource.service.name = "my-service" }' + match_spans: + - match_type: regexp + name: '^GET /rolldice.*' + attributes: + - key: service.name + value: my-service + metrics: + - promql: 'seed_counter_total{service_name="my-service"}' + value: '>= 1' + logs: + - logql: '{service_name="my-service"}' + regex: '.*rolling the dice.*' + count: '>= 1' + profiles: + - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{service_name="my-service"}' + contains: main +``` + +### compose-logs + +For `compose` fixtures, `expected.compose-logs` greps the container logs +(`docker compose logs`) for each string — useful for asserting on output that +never reaches the OTLP pipeline: + +```yaml +expected: + compose-logs: + - app started ok +``` + +## Custom checks + +`expected.custom-checks` runs an arbitrary script and treats **exit code 0 as +pass**, non-zero as fail. Combined stdout+stderr is captured and printed on +failure. Like every assertion it is retried until it passes or `--timeout` +elapses. + +```yaml +expected: + custom-checks: + - script: ./verify.sh # resolved relative to the case file's directory +``` + +`script` may be a path (relative to the case dir, or absolute) or an inline +script beginning with a `#!` shebang. The process runs with its working +directory set to the case dir and inherits the parent environment plus these +OATS-provided variables: + +| Variable | Fixtures | Meaning | +|----------|----------|---------| +| `OATS_FIXTURE_TYPE` | all | `remote` / `compose` / `k3d` | +| `OATS_GRAFANA_URL` | all | base URL of the fixture's Grafana | +| `OATS_APP_URL` | remote, k3d | base URL of the app under test | +| `OATS_OTLP_HTTP` | compose, k3d | OTLP/HTTP endpoint | +| `OATS_PYROSCOPE_URL` | compose, k3d | Pyroscope base URL | +| `COMPOSE_PROJECT_NAME`, `COMPOSE_FILE`, `OATS_COMPOSE_FILE_ARGS` | compose | let the script run its own `docker compose` commands | + +A custom check that queries Grafana directly (replacing, for example, a legacy +`compose-logs` grep with a real LogQL query): + +```bash +#!/usr/bin/env bash +set -euo pipefail +curl -fsS "${OATS_GRAFANA_URL:?}/api/health" >/dev/null +echo "custom check ok" +``` + ## Notes - Cases inside one suite still run sequentially. diff --git a/UPGRADING.md b/UPGRADING.md index 50207c1b..0565bc04 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -23,6 +23,83 @@ removed. For one-off help migrating old cases, use: oats --migrate path/to/legacy.yaml ``` +### Case schema: `oats-schema-version: 2` → `3` + +The case-yaml assertion shape changed with the gcx-driven runner. Bump the tag +to `3` and update assertions. The `0.5.0` / `0.6.0` notes further down describe +the **version-2** shape (`equals`, `attribute-regexp`, `flamebearers`, +`regexp`); the mappings below take you from that shape to version 3. + +**Logs — regex.** Version 2's `regexp:` becomes `regex:` (scalar or list). +`contains` / `not_contains` are also first-class again: + +```diff + logs: + - logql: '{job="app"}' +- regexp: "error" ++ regex: "error" +``` + +**Traces — span matching.** Version 2's flat `equals` / `attributes` / +`attribute-regexp` become structured `match_spans` entries with an explicit +`match_type` and a list-of-`{key, value}` attribute form: + +```diff + traces: + - traceql: '{}' +- equals: "GET /api" +- attributes: +- http.method: "GET" +- attribute-regexp: +- http.route: "/api/.*" ++ match_spans: ++ - match_type: strict ++ name: "GET /api" ++ attributes: ++ - key: http.method ++ value: "GET" ++ - match_type: regexp ++ attributes: ++ - key: http.route ++ value: "/api/.*" +``` + +A span name matched by regex: + +```diff + traces: + - traceql: '{}' +- regexp: "GET /api.*" ++ match_spans: ++ - match_type: regexp ++ name: "GET /api.*" +``` + +**Profiles.** Version 2's `flamebearers:` block becomes the shared assertion +vocabulary keyed off `query`: + +```diff + profiles: + - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds' +- flamebearers: +- equals: "my-function" ++ match: ++ - match_type: strict ++ name: "my-function" +``` + +**`compose-logs`.** Still supported natively for `compose` fixtures +(`expected.compose-logs: [ ... ]`). `--migrate` does **not** convert it (it +emits a warning); either keep it as-is on a compose fixture, or replace it with +a `custom-checks` script that queries the backend directly — see the +[custom checks](README.md#custom-checks) contract. + +**`matrix`.** Not migrated automatically. `--migrate` flattens a single-entry +matrix and otherwise emits a hint; multi-entry matrices must be split into +separate cases by hand. + +See [README.md](README.md) for the full version-3 assertion reference. + ## 0.6.0 ⚠️ Breaking Changes - Migration Required: File Version Tag diff --git a/examples/smoke/cases/inline-seed.yaml b/examples/smoke/cases/inline-seed.yaml index d564bb4b..0f147065 100644 --- a/examples/smoke/cases/inline-seed.yaml +++ b/examples/smoke/cases/inline-seed.yaml @@ -12,4 +12,5 @@ expected: match_spans: - name: seed-operation attributes: - service.name: gcx-e2e-seed + - key: service.name + value: gcx-e2e-seed diff --git a/internal/cli/main.go b/internal/cli/main.go index e2982335..647627d9 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -16,6 +16,7 @@ // -v=1 / -v=2 / -v=3 Progressive verbosity (passes / commands / lifecycle) // --suite Comma-separated suite names to include // --tags Comma-separated tag any-match filter +// --fail-fast Stop scheduling further cases after the first failure package cli import ( @@ -119,6 +120,7 @@ func Run() int { appPort := flag.Int("app-port", 8080, "application port for driving case input requests") otlpHTTP := flag.String("otlp-http", "http://localhost:4318", "OTLP/HTTP base URL for inline-otlp seed mode") parallel := flag.Int("parallel", 1, "number of suites to run in parallel when fixture isolation allows it") + failFast := flag.Bool("fail-fast", false, "stop scheduling further cases after the first case failure") noCache := flag.Bool("no-cache", false, "disable the skip-when-unchanged cache for this run") cacheDir := flag.String("cache-dir", defaultCacheDir(), "directory for the skip-when-unchanged cache") @@ -199,6 +201,7 @@ func Run() int { noCache: *noCache, cacheDir: *cacheDir, cacheTTLDays: cfg.Cache.TTLDays, + failFast: *failFast, } totalPass, totalFail, runErr := runPlans(ctx, rep, plans, opts, *parallel) if runErr != nil { @@ -307,6 +310,7 @@ type runOptions struct { noCache bool cacheDir string cacheTTLDays int + failFast bool } func runPlans(ctx context.Context, rep report.Reporter, plans []discovery.Plan, opts runOptions, parallel int) (int, int, error) { @@ -330,6 +334,9 @@ func runPlans(ctx context.Context, rep report.Reporter, plans []discovery.Plan, if err != nil { return totalPass, totalFail, err } + if opts.failFast && totalFail > 0 { + return totalPass, totalFail, nil + } pass, fail, err := runPlansSequential(ctx, rep, serialPlans, opts) return totalPass + pass, totalFail + fail, err @@ -344,6 +351,9 @@ func runPlansSequential(ctx context.Context, rep report.Reporter, plans []discov if res.err != nil { return totalPass, totalFail, res.err } + if opts.failFast && totalFail > 0 { + break + } } return totalPass, totalFail, nil } @@ -369,7 +379,7 @@ func runPlansParallel(parent context.Context, rep report.Reporter, plans []disco defer wg.Done() for plan := range workCh { res := runPlan(ctx, rep, plan, opts) - if res.err != nil { + if res.err != nil || (opts.failFast && res.fail > 0) { cancel() } resultCh <- res @@ -465,6 +475,9 @@ func runPlan(ctx context.Context, rep report.Reporter, plan discovery.Plan, opts suitePass++ } else { suiteFail++ + if opts.failFast { + break + } } } diff --git a/tests/e2e/cases/assert/value-fail/files/oats.yaml b/tests/e2e/cases/assert/value-fail/files/oats.yaml new file mode 100644 index 00000000..ad43f26b --- /dev/null +++ b/tests/e2e/cases/assert/value-fail/files/oats.yaml @@ -0,0 +1,16 @@ +oats-schema-version: 3 +name: value-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + metrics: + - service: {{CASE_NAME}} + name: oats_value_bad + value: 42 +expected: + metrics: + # Seeded value is 42, so "> 100" never holds — the case must fail. + - promql: 'oats_value_bad_total{service_name="{{CASE_NAME}}"}' + value: '> 100' diff --git a/tests/e2e/cases/assert/value-fail/test.yaml b/tests/e2e/cases/assert/value-fail/test.yaml new file mode 100644 index 00000000..2ad03c92 --- /dev/null +++ b/tests/e2e/cases/assert/value-fail/test.yaml @@ -0,0 +1,14 @@ +name: value-fail +tags: [assert, value, metrics, fail] + +run: + args: + - --timeout + - 15s + - --interval + - 1s + +expect: + exit_code: 1 + stdout_contains: + - FAIL diff --git a/tests/e2e/cases/assert/value/files/oats.yaml b/tests/e2e/cases/assert/value/files/oats.yaml new file mode 100644 index 00000000..c82b29c5 --- /dev/null +++ b/tests/e2e/cases/assert/value/files/oats.yaml @@ -0,0 +1,15 @@ +oats-schema-version: 3 +name: value +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + metrics: + - service: {{CASE_NAME}} + name: oats_value_ok + value: 42 +expected: + metrics: + - promql: 'oats_value_ok_total{service_name="{{CASE_NAME}}"}' + value: '>= 1' diff --git a/tests/e2e/cases/assert/value/test.yaml b/tests/e2e/cases/assert/value/test.yaml new file mode 100644 index 00000000..f8efb209 --- /dev/null +++ b/tests/e2e/cases/assert/value/test.yaml @@ -0,0 +1,13 @@ +name: value +tags: [assert, value, metrics] + +run: + args: + - --timeout + - 30s + - --interval + - 1s + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/cli/fail-fast/files/01-fail.yaml b/tests/e2e/cases/cli/fail-fast/files/01-fail.yaml new file mode 100644 index 00000000..824a4b09 --- /dev/null +++ b/tests/e2e/cases/cli/fail-fast/files/01-fail.yaml @@ -0,0 +1,13 @@ +oats-schema-version: 3 +name: ff-first-fails +seed: + type: inline-otlp + logs: + - service: ff-first-fails + body: ff-line +expected: + logs: + # This token is never logged, so the first case fails and --fail-fast + # must stop the run before the second case executes. + - logql: '{service_name="ff-first-fails"}' + contains: definitely-absent-token diff --git a/tests/e2e/cases/cli/fail-fast/files/02-pass.yaml b/tests/e2e/cases/cli/fail-fast/files/02-pass.yaml new file mode 100644 index 00000000..15a296e8 --- /dev/null +++ b/tests/e2e/cases/cli/fail-fast/files/02-pass.yaml @@ -0,0 +1,13 @@ +oats-schema-version: 3 +name: ff-second-should-not-run +seed: + type: inline-otlp + logs: + - service: ff-second-should-not-run + body: ff-second-line +expected: + logs: + # Would pass on its own, but --fail-fast should prevent it from running + # at all after the first case fails. + - logql: '{service_name="ff-second-should-not-run"}' + contains: ff-second-line diff --git a/tests/e2e/cases/cli/fail-fast/files/oats.toml b/tests/e2e/cases/cli/fail-fast/files/oats.toml new file mode 100644 index 00000000..cde32081 --- /dev/null +++ b/tests/e2e/cases/cli/fail-fast/files/oats.toml @@ -0,0 +1,11 @@ +[meta] +version = 2 + +[[suite]] +name = "failfast" +cases = ["01-fail.yaml", "02-pass.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/fail-fast/test.yaml b/tests/e2e/cases/cli/fail-fast/test.yaml new file mode 100644 index 00000000..2219de87 --- /dev/null +++ b/tests/e2e/cases/cli/fail-fast/test.yaml @@ -0,0 +1,19 @@ +name: fail-fast +tags: [cli, fail-fast] + +run: + args: + - --fail-fast + - --timeout + - 5s + - --interval + - 1s + +expect: + exit_code: 1 + stdout_contains: + - FAIL + # With --fail-fast the runner stops after the first failing case, so the + # second case must never run and its name must not appear in the report. + stdout_not_contains: + - ff-second-should-not-run From 2f04f03252a10d99289eb7eb23ba42eea0f6fe5a Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 16:07:16 +0200 Subject: [PATCH 097/124] test(e2e): add profiles regression coverage; widen value timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - assert/profile + profile-fail: query LGTM's Pyroscope self-profile (runtime.mcall), no seeding needed — profiles can't be inline-seeded. Runs serially so the flamegraph is warm and CPU isn't contended. - assert/value: widen timeout to 60s (OTLP metric→Prometheus landing latency is 15-35s); validated against live otel-lgtm via the e2e harness Signed-off-by: Gregor Zeitlinger --- .../cases/assert/profile-fail/files/oats.yaml | 13 +++++++++++++ tests/e2e/cases/assert/profile-fail/test.yaml | 14 ++++++++++++++ .../e2e/cases/assert/profile/files/oats.yaml | 16 ++++++++++++++++ tests/e2e/cases/assert/profile/test.yaml | 19 +++++++++++++++++++ tests/e2e/cases/assert/value/test.yaml | 6 ++++-- 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 tests/e2e/cases/assert/profile-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/profile-fail/test.yaml create mode 100644 tests/e2e/cases/assert/profile/files/oats.yaml create mode 100644 tests/e2e/cases/assert/profile/test.yaml diff --git a/tests/e2e/cases/assert/profile-fail/files/oats.yaml b/tests/e2e/cases/assert/profile-fail/files/oats.yaml new file mode 100644 index 00000000..135051f3 --- /dev/null +++ b/tests/e2e/cases/assert/profile-fail/files/oats.yaml @@ -0,0 +1,13 @@ +oats-schema-version: 3 +name: profile-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: app +expected: + profiles: + # This frame is never present in Pyroscope's self-profile, so the case must + # fail — exercising the profiles assertion failure path. + - query: 'process_cpu:samples:count:cpu:nanoseconds{service_name="pyroscope"}' + contains: this_frame_does_not_exist_oats_xyz diff --git a/tests/e2e/cases/assert/profile-fail/test.yaml b/tests/e2e/cases/assert/profile-fail/test.yaml new file mode 100644 index 00000000..87ca2782 --- /dev/null +++ b/tests/e2e/cases/assert/profile-fail/test.yaml @@ -0,0 +1,14 @@ +name: profile-fail +tags: [assert, profile, profiles, fail] + +run: + args: + - --timeout + - 10s + - --interval + - 2s + +expect: + exit_code: 1 + stdout_contains: + - FAIL diff --git a/tests/e2e/cases/assert/profile/files/oats.yaml b/tests/e2e/cases/assert/profile/files/oats.yaml new file mode 100644 index 00000000..758371f5 --- /dev/null +++ b/tests/e2e/cases/assert/profile/files/oats.yaml @@ -0,0 +1,16 @@ +oats-schema-version: 3 +name: profile +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: app +expected: + profiles: + # otel-lgtm's Pyroscope self-profiles under service_name="pyroscope", so a + # CPU flamegraph is always present without seeding (oats cannot seed + # profiles). This exercises the oats -> gcx -> Pyroscope query + flamebearer + # assertion path. runtime.mcall is a Go scheduler frame present in every + # Go CPU profile. + - query: 'process_cpu:samples:count:cpu:nanoseconds{service_name="pyroscope"}' + contains: runtime.mcall diff --git a/tests/e2e/cases/assert/profile/test.yaml b/tests/e2e/cases/assert/profile/test.yaml new file mode 100644 index 00000000..44bade62 --- /dev/null +++ b/tests/e2e/cases/assert/profile/test.yaml @@ -0,0 +1,19 @@ +name: profile +tags: [assert, profile, profiles] + +# Run serially (after the parallel batch) so Pyroscope has been up long enough +# to have self-profiled a CPU flamegraph, and isn't starved by concurrent +# CPU-heavy cases. A cold Pyroscope needs ~30s to accumulate runtime frames. +execution: + mode: serial + +run: + args: + - --timeout + - 45s + - --interval + - 2s + +expect: + stdout_contains: + - PASS 1/1 diff --git a/tests/e2e/cases/assert/value/test.yaml b/tests/e2e/cases/assert/value/test.yaml index f8efb209..3ed3bbe9 100644 --- a/tests/e2e/cases/assert/value/test.yaml +++ b/tests/e2e/cases/assert/value/test.yaml @@ -3,10 +3,12 @@ tags: [assert, value, metrics] run: args: + # OTLP metric → Prometheus landing latency is 15-35s in otel-lgtm, so allow + # a generous ceiling; wait.Until returns as soon as the value appears. - --timeout - - 30s + - 60s - --interval - - 1s + - 2s expect: stdout_contains: From 6c1c56ac0198f55953d68a9b7fb38eb5fde487a3 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 16:46:26 +0200 Subject: [PATCH 098/124] feat(cli): cobra subcommands, cache clear; drop inert hermetic field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CLI moves to cobra: implicit-run root (bare `oats [flags]` still runs) plus run/list/migrate/cache clear/version subcommands. --list/--migrate kept as hidden deprecated flag aliases; verbosity is now -v/-vv/-vvv. - add `oats cache clear` (wipes the cache dir) - remove the `hermetic:` case field — it parsed but was never enforced - docs: README CLI section + UPGRADING (migrate subcommand, worked legacy→v3 example) Signed-off-by: Gregor Zeitlinger --- .github/renovate-tracked-deps.json | 3 + README.md | 27 ++- UPGRADING.md | 70 ++++++- casefile/case.go | 11 -- casefile/case_test.go | 12 -- go.mod | 3 + go.sum | 8 + internal/cli/main.go | 294 +++++++++++++++++++++-------- 8 files changed, 316 insertions(+), 112 deletions(-) diff --git a/.github/renovate-tracked-deps.json b/.github/renovate-tracked-deps.json index 9b62f9ba..3bd350bc 100644 --- a/.github/renovate-tracked-deps.json +++ b/.github/renovate-tracked-deps.json @@ -38,11 +38,14 @@ "github.com/google/uuid", "github.com/grpc-ecosystem/grpc-gateway/v2", "github.com/hashicorp/go-version", + "github.com/inconshreveable/mousetrap", "github.com/json-iterator/go", "github.com/modern-go/concurrent", "github.com/modern-go/reflect2", "github.com/onsi/gomega", "github.com/pmezard/go-difflib", + "github.com/spf13/cobra", + "github.com/spf13/pflag", "github.com/stretchr/testify", "go", "go.opentelemetry.io/auto/sdk", diff --git a/README.md b/README.md index 648c9960..e494430e 100644 --- a/README.md +++ b/README.md @@ -19,10 +19,9 @@ go install github.com/grafana/oats@latest # Print the CLI version bin/oats version -bin/oats -version # Print what would run -bin/oats --config examples/smoke/oats.toml --list +bin/oats list --config examples/smoke/oats.toml # Run bin/oats --config examples/smoke/oats.toml @@ -44,22 +43,32 @@ bin/oats --config examples/smoke/oats.toml - best-effort migration from legacy OATS yaml via: ```sh - oats --migrate path/to/legacy.yaml + oats migrate path/to/legacy.yaml ``` ## CLI +Commands: + +```sh +oats [flags] # run the suites (implicit; same as `oats run`) +oats run [flags] # run the suites +oats list --config oats.toml # print the run plan and exit +oats migrate legacy.yaml # convert one legacy yaml to the v3 shape +oats cache clear # delete all cached results +oats version # print the version +``` + +Run flags (on `oats` / `oats run`): + ```sh -oats --config oats.toml --list oats --config oats.toml --suite smoke oats --config oats.toml --tags traces,logs oats --config oats.toml --gcx-context my-lgtm oats --config oats.toml --no-cache oats --config oats.toml --fail-fast -oats --format ndjson -oats -v=1 -oats -v=2 -oats -v=3 +oats --config oats.toml --format ndjson +oats -v # -v / -vv / -vvv increase verbosity ``` Key flags: @@ -74,7 +83,7 @@ Key flags: - `--fail-fast` — stop scheduling further cases after the first case failure - `--gcx` - `--gcx-context` -- `--version` +- `-v` / `-vv` / `-vvv` — verbosity ## Config shape diff --git a/UPGRADING.md b/UPGRADING.md index 0565bc04..7b967a0e 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -20,7 +20,7 @@ The legacy “pass one or more old yaml files directly to `oats`” runner has b removed. For one-off help migrating old cases, use: ```sh -oats --migrate path/to/legacy.yaml +oats migrate path/to/legacy.yaml ``` ### Case schema: `oats-schema-version: 2` → `3` @@ -89,15 +89,79 @@ vocabulary keyed off `query`: ``` **`compose-logs`.** Still supported natively for `compose` fixtures -(`expected.compose-logs: [ ... ]`). `--migrate` does **not** convert it (it +(`expected.compose-logs: [ ... ]`). `oats migrate` does **not** convert it (it emits a warning); either keep it as-is on a compose fixture, or replace it with a `custom-checks` script that queries the backend directly — see the [custom checks](README.md#custom-checks) contract. -**`matrix`.** Not migrated automatically. `--migrate` flattens a single-entry +**`matrix`.** Not migrated automatically. `oats migrate` flattens a single-entry matrix and otherwise emits a hint; multi-entry matrices must be split into separate cases by hand. +### Worked example: a full legacy case → v3 + +A legacy (schema-version 2) case plus its `docker-compose`: + +```yaml +# oats.yaml (v2) +oats-schema-version: 2 +docker-compose: + files: + - ./docker-compose.yaml +input: + - path: /rolldice +expected: + traces: + - traceql: '{}' + spans: + - name: "GET /rolldice" + attributes: + http.request.method: "GET" + logs: + - logql: '{service_name="rolldice"}' + contains: ["rolling the dice"] +``` + +Run `oats migrate ./oats.yaml`. Split the result into an `oats.toml` and a v3 +case (the migrator prints both the case yaml and a suggested `[fixture]` block): + +```toml +# oats.toml +[meta] +version = 2 + +[[suite]] +name = "rolldice" +cases = ["cases/*.yaml"] +fixture = "compose-lgtm" + +[fixture.compose-lgtm] +type = "compose" +compose_file = "docker-compose.yaml" +``` + +```yaml +# cases/rolldice.yaml (v3) +oats-schema-version: 3 +name: rolldice +seed: + type: app +input: + - path: /rolldice +expected: + traces: + - traceql: '{}' + match_spans: + - match_type: strict + name: "GET /rolldice" + attributes: + - key: http.request.method + value: "GET" + logs: + - logql: '{service_name="rolldice"}' + contains: "rolling the dice" +``` + See [README.md](README.md) for the full version-3 assertion reference. ## 0.6.0 diff --git a/casefile/case.go b/casefile/case.go index 84cb0d22..23580fbe 100644 --- a/casefile/case.go +++ b/casefile/case.go @@ -32,7 +32,6 @@ type Case struct { OatsSchemaVersion int `yaml:"oats-schema-version"` Name string `yaml:"name"` Tags []string `yaml:"tags,omitempty"` - Hermetic *bool `yaml:"hermetic,omitempty"` // pointer: distinguish unset vs explicit false Interval time.Duration `yaml:"interval,omitempty"` Fixture *FixtureConfig `yaml:"fixture,omitempty"` @@ -438,16 +437,6 @@ func (f FixtureConfig) Validate(path string) error { return nil } -// IsHermetic reports whether this case promises not to interfere with siblings -// sharing the same fixture. Default is true; cases opt out by setting -// `hermetic: false`. -func (c *Case) IsHermetic() bool { - if c.Hermetic == nil { - return true - } - return *c.Hermetic -} - func validateTraceAssertion(idx int, a TraceAssertion) error { if err := validateMatchEntries(fmt.Sprintf("expected.traces[%d].match_spans", idx), a.MatchSpans); err != nil { return err diff --git a/casefile/case_test.go b/casefile/case_test.go index a06034a8..b9e3cfb9 100644 --- a/casefile/case_test.go +++ b/casefile/case_test.go @@ -366,15 +366,3 @@ expected: t.Fatalf("expected regexp error, got %v", err) } } - -func TestIsHermetic(t *testing.T) { - c := &Case{} - if !c.IsHermetic() { - t.Error("default should be hermetic") - } - f := false - c.Hermetic = &f - if c.IsHermetic() { - t.Error("explicit false should override") - } -} diff --git a/go.mod b/go.mod index cabb2e6d..3cf78c74 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.25.0 require ( github.com/BurntSushi/toml v1.6.0 github.com/onsi/gomega v1.42.1 + github.com/spf13/cobra v1.10.2 + github.com/spf13/pflag v1.0.9 github.com/stretchr/testify v1.11.1 go.opentelemetry.io/collector/pdata v1.61.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 @@ -24,6 +26,7 @@ require ( github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/go-version v1.9.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect diff --git a/go.sum b/go.sum index df7fe1ca..f7986659 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -24,6 +25,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF2 github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -43,6 +46,11 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= diff --git a/internal/cli/main.go b/internal/cli/main.go index 647627d9..c3b36505 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -5,24 +5,27 @@ // // Usage: // -// oats [flags] +// oats [flags] run the suites (implicit; same as `oats run`) +// oats run [flags] run the suites +// oats list print the run plan and exit +// oats migrate convert one legacy yaml to the v3 shape +// oats cache clear delete all cached results +// oats version print the version // -// Flags (subset): +// Run flags (subset): // // --config Path to oats.toml (default ./oats.toml) // --gcx Path to gcx binary (default "gcx" on PATH) -// --list Print the run plan and exit (no execution) // --format Output format: "text" (default) or "ndjson" -// -v=1 / -v=2 / -v=3 Progressive verbosity (passes / commands / lifecycle) // --suite Comma-separated suite names to include // --tags Comma-separated tag any-match filter // --fail-fast Stop scheduling further cases after the first failure +// -v / -vv / -vvv Progressive verbosity (passes / commands / lifecycle) package cli import ( "context" "encoding/json" - "flag" "fmt" "net" "net/http" @@ -36,6 +39,9 @@ import ( "syscall" "time" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/grafana/oats/cache" "github.com/grafana/oats/discovery" "github.com/grafana/oats/engine" @@ -97,83 +103,210 @@ func Main() { os.Exit(Run()) } +// Run builds the cobra command tree and executes it. The exit code is +// threaded through *exit so a run with failing cases returns 1 without cobra +// treating it as a usage error. func Run() int { - if len(os.Args) > 1 && os.Args[1] == "version" { - fmt.Println(Version) - return 0 - } - - configPath := flag.String("config", "oats.toml", "path to oats.toml") - gcxBin := flag.String("gcx", "gcx", "path to gcx binary (PATH-resolved if a bare name)") - listOnly := flag.Bool("list", false, "print the run plan and exit (no execution)") - versionOnly := flag.Bool("version", false, "print the oats version and exit") - migratePath := flag.String("migrate", "", "convert one legacy OATS yaml file and print the result to stdout") - format := flag.String("format", "text", "output format: text | ndjson") - suiteFilterStr := flag.String("suite", "", "comma-separated suite names") - tagFilterStr := flag.String("tags", "", "comma-separated tag any-match") - timeout := flag.Duration("timeout", 30*time.Second, "per-assertion timeout") - interval := flag.Duration("interval", 500*time.Millisecond, "polling interval") - absentTimeout := flag.Duration("absent-timeout", 10*time.Second, "absence-check window") - seedSettle := flag.Duration("seed-settle", 2*time.Second, "post-seed wait before first assertion") - gcxContextOverride := flag.String("gcx-context", "", "override the gcx --context value (otherwise derived from fixture endpoint)") - appHost := flag.String("app-host", "localhost", "application host for driving case input requests") - appPort := flag.Int("app-port", 8080, "application port for driving case input requests") - otlpHTTP := flag.String("otlp-http", "http://localhost:4318", "OTLP/HTTP base URL for inline-otlp seed mode") - parallel := flag.Int("parallel", 1, "number of suites to run in parallel when fixture isolation allows it") - failFast := flag.Bool("fail-fast", false, "stop scheduling further cases after the first case failure") - noCache := flag.Bool("no-cache", false, "disable the skip-when-unchanged cache for this run") - cacheDir := flag.String("cache-dir", defaultCacheDir(), "directory for the skip-when-unchanged cache") + var exit int + root := newRootCmd(&exit) + if err := root.Execute(); err != nil { + fmt.Fprintln(os.Stderr, "Error:", err) + if exit == 0 { + exit = 2 + } + } + return exit +} +func newRootCmd(exit *int) *cobra.Command { var verbose int - flag.IntVar(&verbose, "v", 0, "verbosity (0-3)") + root := &cobra.Command{ + Use: "oats", + Short: "OpenTelemetry Acceptance Tests — the gcx-driven runner", + SilenceUsage: true, + SilenceErrors: true, + // Bare `oats [flags]` is an implicit `run` for backward compatibility. + RunE: func(cmd *cobra.Command, _ []string) error { + return runAction(cmd, verbose, exit) + }, + } + root.PersistentFlags().CountVarP(&verbose, "verbose", "v", "increase verbosity (-v, -vv, -vvv)") + addRunFlags(root.Flags()) + + root.AddCommand( + newRunCmd(&verbose, exit), + newListCmd(), + newMigrateCmd(), + newCacheCmd(), + newVersionCmd(), + ) + return root +} + +// addRunFlags registers the flags shared by the implicit-run root and the +// explicit `run` subcommand. +func addRunFlags(fs *pflag.FlagSet) { + fs.String("config", "oats.toml", "path to oats.toml") + fs.String("gcx", "gcx", "path to gcx binary (PATH-resolved if a bare name)") + fs.String("format", "text", "output format: text | ndjson") + fs.String("suite", "", "comma-separated suite names") + fs.String("tags", "", "comma-separated tag any-match") + fs.Duration("timeout", 30*time.Second, "per-assertion timeout") + fs.Duration("interval", 500*time.Millisecond, "polling interval") + fs.Duration("absent-timeout", 10*time.Second, "absence-check window") + fs.Duration("seed-settle", 2*time.Second, "post-seed wait before first assertion") + fs.String("gcx-context", "", "override the gcx --context value (otherwise derived from fixture endpoint)") + fs.String("app-host", "localhost", "application host for driving case input requests") + fs.Int("app-port", 8080, "application port for driving case input requests") + fs.String("otlp-http", "http://localhost:4318", "OTLP/HTTP base URL for inline-otlp seed mode") + fs.Int("parallel", 1, "number of suites to run in parallel when fixture isolation allows it") + fs.Bool("fail-fast", false, "stop scheduling further cases after the first case failure") + fs.Bool("no-cache", false, "disable the skip-when-unchanged cache for this run") + fs.String("cache-dir", defaultCacheDir(), "directory for the skip-when-unchanged cache") + + // Deprecated flag aliases, superseded by the `list` and `migrate` + // subcommands. Hidden but still honored so existing invocations keep working. + fs.Bool("list", false, "deprecated: use `oats list`") + fs.String("migrate", "", "deprecated: use `oats migrate `") + _ = fs.MarkHidden("list") + _ = fs.MarkHidden("migrate") +} + +func newRunCmd(verbose, exit *int) *cobra.Command { + cmd := &cobra.Command{ + Use: "run", + Short: "Run the suites in oats.toml (default when no subcommand is given)", + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, _ []string) error { + return runAction(cmd, *verbose, exit) + }, + } + addRunFlags(cmd.Flags()) + return cmd +} + +func newListCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "Print the run plan and exit (no execution)", + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, _ []string) error { + return listAction(cmd) + }, + } + cmd.Flags().String("config", "oats.toml", "path to oats.toml") + return cmd +} - flag.Parse() +func newMigrateCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "migrate ", + Short: "Convert one legacy OATS yaml file and print the result to stdout", + Args: cobra.ExactArgs(1), + SilenceUsage: true, + SilenceErrors: true, + RunE: func(_ *cobra.Command, args []string) error { + return migrateAction(args[0]) + }, + } + return cmd +} - if *versionOnly { - fmt.Println(Version) - return 0 +func newCacheCmd() *cobra.Command { + cacheCmd := &cobra.Command{ + Use: "cache", + Short: "Manage the skip-when-unchanged cache", + } + clear := &cobra.Command{ + Use: "clear", + Short: "Delete all cached results", + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, _ []string) error { + dir, _ := cmd.Flags().GetString("cache-dir") + store, err := cache.New(dir, 0, nil) + if err != nil { + return err + } + if err := store.Clear(); err != nil { + return err + } + fmt.Println("cache cleared:", dir) + return nil + }, } + clear.Flags().String("cache-dir", defaultCacheDir(), "cache directory to clear") + cacheCmd.AddCommand(clear) + return cacheCmd +} - if *migratePath != "" { - out, warnings, err := migrate.ConvertFile(*migratePath) - if err != nil { - fmt.Fprintln(os.Stderr, err) - return 2 - } - for _, w := range warnings { - fmt.Fprintln(os.Stderr, "migrate warning:", w) - } - fmt.Print(string(out)) - return 0 +func newVersionCmd() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Print the oats version and exit", + RunE: func(_ *cobra.Command, _ []string) error { + fmt.Println(Version) + return nil + }, + } +} + +func listAction(cmd *cobra.Command) error { + configPath, _ := cmd.Flags().GetString("config") + cfg, err := discovery.Load(configPath) + if err != nil { + return err } + fmt.Print(cfg.Summary()) + return nil +} - cfg, err := discovery.Load(*configPath) +func migrateAction(path string) error { + out, warnings, err := migrate.ConvertFile(path) if err != nil { - fmt.Fprintln(os.Stderr, err) - return 2 + return err + } + for _, w := range warnings { + fmt.Fprintln(os.Stderr, "migrate warning:", w) } + fmt.Print(string(out)) + return nil +} - filter := discovery.Filter{ - Suites: splitCSV(*suiteFilterStr), - Tags: splitCSV(*tagFilterStr), +// runAction is the shared implementation behind the implicit-run root and the +// explicit `run` subcommand. +func runAction(cmd *cobra.Command, verbose int, exit *int) error { + fs := cmd.Flags() + + // Honor the deprecated --list / --migrate flags. + if list, _ := fs.GetBool("list"); list { + return listAction(cmd) + } + if migratePath, _ := fs.GetString("migrate"); migratePath != "" { + return migrateAction(migratePath) } - if *listOnly { - fmt.Print(cfg.Summary()) - return 0 + configPath, _ := fs.GetString("config") + cfg, err := discovery.Load(configPath) + if err != nil { + return err } + filter := discovery.Filter{ + Suites: splitCSV(flagStr(fs, "suite")), + Tags: splitCSV(flagStr(fs, "tags")), + } plans, err := cfg.PlanRun(filter) if err != nil { - fmt.Fprintln(os.Stderr, err) - return 2 + return err } if len(plans) == 0 { - fmt.Fprintln(os.Stderr, "no suites matched the filter") - return 2 + return fmt.Errorf("no suites matched the filter") } - rep := newReporter(os.Stdout, *format, verbosityFromInt(verbose)) + rep := newReporter(os.Stdout, flagStr(fs, "format"), verbosityFromInt(verbose)) defer func() { _ = rep.Close() }() rep = &lockedReporter{inner: rep} @@ -189,24 +322,23 @@ func Run() int { runStart := time.Now() opts := runOptions{ - gcxBin: *gcxBin, - gcxContextOverride: *gcxContextOverride, - appHost: *appHost, - appPort: *appPort, - otlpHTTP: *otlpHTTP, - timeout: *timeout, - interval: *interval, - absentTimeout: *absentTimeout, - seedSettle: *seedSettle, - noCache: *noCache, - cacheDir: *cacheDir, + gcxBin: flagStr(fs, "gcx"), + gcxContextOverride: flagStr(fs, "gcx-context"), + appHost: flagStr(fs, "app-host"), + appPort: flagInt(fs, "app-port"), + otlpHTTP: flagStr(fs, "otlp-http"), + timeout: flagDur(fs, "timeout"), + interval: flagDur(fs, "interval"), + absentTimeout: flagDur(fs, "absent-timeout"), + seedSettle: flagDur(fs, "seed-settle"), + noCache: flagBool(fs, "no-cache"), + cacheDir: flagStr(fs, "cache-dir"), cacheTTLDays: cfg.Cache.TTLDays, - failFast: *failFast, + failFast: flagBool(fs, "fail-fast"), } - totalPass, totalFail, runErr := runPlans(ctx, rep, plans, opts, *parallel) + totalPass, totalFail, runErr := runPlans(ctx, rep, plans, opts, flagInt(fs, "parallel")) if runErr != nil { - fmt.Fprintln(os.Stderr, runErr) - return 2 + return runErr } rep.Emit(report.Event{ @@ -217,9 +349,17 @@ func Run() int { }) if totalFail > 0 || ctx.Err() != nil { - return 1 + *exit = 1 } - return 0 + return nil +} + +func flagStr(fs *pflag.FlagSet, name string) string { v, _ := fs.GetString(name); return v } +func flagInt(fs *pflag.FlagSet, name string) int { v, _ := fs.GetInt(name); return v } +func flagBool(fs *pflag.FlagSet, name string) bool { v, _ := fs.GetBool(name); return v } +func flagDur(fs *pflag.FlagSet, name string) (d time.Duration) { + d, _ = fs.GetDuration(name) + return d } func newReporter(w *os.File, format string, v report.Verbosity) report.Reporter { From fc36f5cc802c499be6fdc6954a2490aec9145dfd Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 16:52:40 +0200 Subject: [PATCH 099/124] test(e2e): fail variants, ndjson, migrate, and cache-skip coverage - assert/{count,match-regexp,not-contains}-fail: FAIL-path coverage for the assertion vocab (not-contains-fail settles first so the forbidden string is present, otherwise not_contains trivially holds) - format/ndjson: assert the NDJSON reporter emits run.end + pass count - cli/migrate: drive `oats migrate` on a legacy yaml (no fixture needed) - cli/cache-skip: run a suite twice against one cache dir; second run SKIPs All validated against live otel-lgtm via the real e2e harness. Signed-off-by: Gregor Zeitlinger --- .../cases/assert/count-fail/files/oats.yaml | 17 +++++++++++++++++ tests/e2e/cases/assert/count-fail/test.yaml | 14 ++++++++++++++ .../assert/match-regexp-fail/files/oats.yaml | 17 +++++++++++++++++ .../cases/assert/match-regexp-fail/test.yaml | 14 ++++++++++++++ .../assert/not-contains-fail/files/oats.yaml | 15 +++++++++++++++ .../cases/assert/not-contains-fail/test.yaml | 18 ++++++++++++++++++ tests/e2e/cases/cli/cache-skip/files/case.yaml | 11 +++++++++++ tests/e2e/cases/cli/cache-skip/files/oats.toml | 11 +++++++++++ tests/e2e/cases/cli/cache-skip/test.yaml | 15 +++++++++++++++ tests/e2e/cases/cli/migrate/files/legacy.yaml | 10 ++++++++++ tests/e2e/cases/cli/migrate/test.yaml | 14 ++++++++++++++ tests/e2e/cases/format/ndjson/files/oats.yaml | 14 ++++++++++++++ tests/e2e/cases/format/ndjson/test.yaml | 16 ++++++++++++++++ 13 files changed, 186 insertions(+) create mode 100644 tests/e2e/cases/assert/count-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/count-fail/test.yaml create mode 100644 tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/match-regexp-fail/test.yaml create mode 100644 tests/e2e/cases/assert/not-contains-fail/files/oats.yaml create mode 100644 tests/e2e/cases/assert/not-contains-fail/test.yaml create mode 100644 tests/e2e/cases/cli/cache-skip/files/case.yaml create mode 100644 tests/e2e/cases/cli/cache-skip/files/oats.toml create mode 100644 tests/e2e/cases/cli/cache-skip/test.yaml create mode 100644 tests/e2e/cases/cli/migrate/files/legacy.yaml create mode 100644 tests/e2e/cases/cli/migrate/test.yaml create mode 100644 tests/e2e/cases/format/ndjson/files/oats.yaml create mode 100644 tests/e2e/cases/format/ndjson/test.yaml diff --git a/tests/e2e/cases/assert/count-fail/files/oats.yaml b/tests/e2e/cases/assert/count-fail/files/oats.yaml new file mode 100644 index 00000000..a4f66222 --- /dev/null +++ b/tests/e2e/cases/assert/count-fail/files/oats.yaml @@ -0,0 +1,17 @@ +oats-schema-version: 3 +name: count-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + # One line seeded, so "== 5" can never hold — the case must fail. + - logql: '{service_name="{{CASE_NAME}}"}' + count: '== 5' + match: + - name: seed-log-line diff --git a/tests/e2e/cases/assert/count-fail/test.yaml b/tests/e2e/cases/assert/count-fail/test.yaml new file mode 100644 index 00000000..0ba21448 --- /dev/null +++ b/tests/e2e/cases/assert/count-fail/test.yaml @@ -0,0 +1,14 @@ +name: count-fail +tags: [assert, count, fail] + +run: + args: + - --timeout + - 10s + - --interval + - 2s + +expect: + exit_code: 1 + stdout_contains: + - FAIL diff --git a/tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml b/tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml new file mode 100644 index 00000000..3f0980ca --- /dev/null +++ b/tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml @@ -0,0 +1,17 @@ +oats-schema-version: 3 +name: match-regexp-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + # The seeded body is "seed-log-line", which never matches "nomatch-.*". + - logql: '{service_name="{{CASE_NAME}}"}' + match: + - match_type: regexp + name: 'nomatch-.*' diff --git a/tests/e2e/cases/assert/match-regexp-fail/test.yaml b/tests/e2e/cases/assert/match-regexp-fail/test.yaml new file mode 100644 index 00000000..0689f882 --- /dev/null +++ b/tests/e2e/cases/assert/match-regexp-fail/test.yaml @@ -0,0 +1,14 @@ +name: match-regexp-fail +tags: [assert, match, regexp, fail] + +run: + args: + - --timeout + - 10s + - --interval + - 2s + +expect: + exit_code: 1 + stdout_contains: + - FAIL diff --git a/tests/e2e/cases/assert/not-contains-fail/files/oats.yaml b/tests/e2e/cases/assert/not-contains-fail/files/oats.yaml new file mode 100644 index 00000000..ec982de9 --- /dev/null +++ b/tests/e2e/cases/assert/not-contains-fail/files/oats.yaml @@ -0,0 +1,15 @@ +oats-schema-version: 3 +name: not-contains-fail +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + # The seeded body IS present, so not_contains must fail. + - logql: '{service_name="{{CASE_NAME}}"}' + not_contains: seed-log-line diff --git a/tests/e2e/cases/assert/not-contains-fail/test.yaml b/tests/e2e/cases/assert/not-contains-fail/test.yaml new file mode 100644 index 00000000..792a595a --- /dev/null +++ b/tests/e2e/cases/assert/not-contains-fail/test.yaml @@ -0,0 +1,18 @@ +name: not-contains-fail +tags: [assert, not_contains, fail] + +run: + args: + # Settle so the seeded log is ingested before the assertion runs, otherwise + # not_contains would trivially hold on an empty result. + - --seed-settle + - 6s + - --timeout + - 20s + - --interval + - 2s + +expect: + exit_code: 1 + stdout_contains: + - FAIL diff --git a/tests/e2e/cases/cli/cache-skip/files/case.yaml b/tests/e2e/cases/cli/cache-skip/files/case.yaml new file mode 100644 index 00000000..653c409f --- /dev/null +++ b/tests/e2e/cases/cli/cache-skip/files/case.yaml @@ -0,0 +1,11 @@ +oats-schema-version: 3 +name: cache-case +seed: + type: inline-otlp + logs: + - service: cache-case + body: cache-seed-line +expected: + logs: + - logql: '{service_name="cache-case"}' + contains: cache-seed-line diff --git a/tests/e2e/cases/cli/cache-skip/files/oats.toml b/tests/e2e/cases/cli/cache-skip/files/oats.toml new file mode 100644 index 00000000..a208e9e1 --- /dev/null +++ b/tests/e2e/cases/cli/cache-skip/files/oats.toml @@ -0,0 +1,11 @@ +[meta] +version = 2 + +[[suite]] +name = "cache" +cases = ["case.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/cache-skip/test.yaml b/tests/e2e/cases/cli/cache-skip/test.yaml new file mode 100644 index 00000000..eec7bcb6 --- /dev/null +++ b/tests/e2e/cases/cli/cache-skip/test.yaml @@ -0,0 +1,15 @@ +name: cache-skip +tags: [cli, cache] + +# Run the same suite twice against one cache dir; the second run must skip the +# unchanged case instead of re-running it. -v surfaces the SKIP line. +run: + shell: | + set -e + {{OATS}} --config files/oats.toml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v + echo "=== second run ===" + {{OATS}} --config files/oats.toml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v + +expect: + stdout_contains: + - "SKIP" diff --git a/tests/e2e/cases/cli/migrate/files/legacy.yaml b/tests/e2e/cases/cli/migrate/files/legacy.yaml new file mode 100644 index 00000000..fb495f67 --- /dev/null +++ b/tests/e2e/cases/cli/migrate/files/legacy.yaml @@ -0,0 +1,10 @@ +oats-schema-version: 2 +docker-compose: + files: + - ./docker-compose.yaml +input: + - path: /rolldice +expected: + logs: + - logql: '{service_name="rolldice"}' + regexp: "rolling the dice" diff --git a/tests/e2e/cases/cli/migrate/test.yaml b/tests/e2e/cases/cli/migrate/test.yaml new file mode 100644 index 00000000..1208d2a5 --- /dev/null +++ b/tests/e2e/cases/cli/migrate/test.yaml @@ -0,0 +1,14 @@ +name: migrate +tags: [cli, migrate] + +# Drives `oats migrate` directly (no fixture / LGTM needed). +run: + command: "{{OATS}}" + args: + - migrate + - "{{CASE_DIR}}/files/legacy.yaml" + +expect: + stdout_contains: + - "oats-schema-version: 3" + - "logql" diff --git a/tests/e2e/cases/format/ndjson/files/oats.yaml b/tests/e2e/cases/format/ndjson/files/oats.yaml new file mode 100644 index 00000000..3e419773 --- /dev/null +++ b/tests/e2e/cases/format/ndjson/files/oats.yaml @@ -0,0 +1,14 @@ +oats-schema-version: 3 +name: ndjson +fixture: + type: remote + endpoint: {{REMOTE_OTLP_HTTP}} +seed: + type: inline-otlp + logs: + - service: {{CASE_NAME}} + body: seed-log-line +expected: + logs: + - logql: '{service_name="{{CASE_NAME}}"}' + contains: seed-log-line diff --git a/tests/e2e/cases/format/ndjson/test.yaml b/tests/e2e/cases/format/ndjson/test.yaml new file mode 100644 index 00000000..bb3462e4 --- /dev/null +++ b/tests/e2e/cases/format/ndjson/test.yaml @@ -0,0 +1,16 @@ +name: ndjson +tags: [format, ndjson] + +run: + args: + - --format + - ndjson + - --timeout + - 30s + - --interval + - 2s + +expect: + stdout_contains: + - '"event":"run.end"' + - '"pass":1' From 606fdd4638805297b51ff47dd42cc8eb8b8b590c Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 17:01:25 +0200 Subject: [PATCH 100/124] test(e2e): app-backed seed round-trip via telemetrygen fixture/compose-app-seed boots LGTM + a real instrumented app (telemetrygen, a prebuilt public image) that exports OTLP traces, then asserts the span is queryable through gcx. Covers the seed:app path that inline-otlp cases don't. Validated against a live compose stack. Signed-off-by: Gregor Zeitlinger --- .../files/docker-compose.oats.yml | 18 ++++++++++++++++++ .../fixture/compose-app-seed/files/oats.yaml | 14 ++++++++++++++ .../cases/fixture/compose-app-seed/test.yaml | 18 ++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 tests/e2e/cases/fixture/compose-app-seed/files/docker-compose.oats.yml create mode 100644 tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml create mode 100644 tests/e2e/cases/fixture/compose-app-seed/test.yaml diff --git a/tests/e2e/cases/fixture/compose-app-seed/files/docker-compose.oats.yml b/tests/e2e/cases/fixture/compose-app-seed/files/docker-compose.oats.yml new file mode 100644 index 00000000..0e727f3a --- /dev/null +++ b/tests/e2e/cases/fixture/compose-app-seed/files/docker-compose.oats.yml @@ -0,0 +1,18 @@ +services: + # A real instrumented app: telemetrygen exports OTLP traces to the LGTM + # collector (added by `template: lgtm`). restart: always re-emits until LGTM + # is ready, so the trace reliably lands regardless of startup ordering. + telemetrygen: + image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest + command: + - traces + - --otlp-endpoint + - lgtm:4317 + - --otlp-insecure + - --service + - oats-app-e2e + - --traces + - "5" + restart: always + depends_on: + - lgtm diff --git a/tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml b/tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml new file mode 100644 index 00000000..7abb0d9a --- /dev/null +++ b/tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml @@ -0,0 +1,14 @@ +oats-schema-version: 3 +name: compose-app-seed +fixture: + type: compose + template: lgtm + compose_file: docker-compose.oats.yml +seed: + type: app +expected: + traces: + # telemetrygen emits a root span named "lets-go" under service oats-app-e2e. + - traceql: '{ resource.service.name = "oats-app-e2e" }' + match_spans: + - name: lets-go diff --git a/tests/e2e/cases/fixture/compose-app-seed/test.yaml b/tests/e2e/cases/fixture/compose-app-seed/test.yaml new file mode 100644 index 00000000..da929ce5 --- /dev/null +++ b/tests/e2e/cases/fixture/compose-app-seed/test.yaml @@ -0,0 +1,18 @@ +name: compose-app-seed +tags: [fixture, compose, seed, app] + +# Compose fixture boots LGTM + a real app (telemetrygen) that exports OTLP. +# Serial so it isn't starved while the compose stack builds and starts. +execution: + mode: serial + +run: + args: + - --timeout + - 90s + - --interval + - 3s + +expect: + stdout_contains: + - PASS 1/1 From 3a1c4ccc5ddeec967151ae35477bdfe57671ca0e Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 17:48:24 +0200 Subject: [PATCH 101/124] test(e2e): cover oats cache clear subcommand end-to-end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache clear subcommand had no end-to-end coverage; only the underlying Store.Clear() was unit-tested. Add a case that populates the cache with one green run, runs `oats cache clear`, then reruns and asserts the case re-executes (no SKIP line) — the sibling cache-skip case proves a rerun would skip without the clear. Signed-off-by: Gregor Zeitlinger --- .../e2e/cases/cli/cache-clear/files/case.yaml | 11 ++++++++++ .../e2e/cases/cli/cache-clear/files/oats.toml | 11 ++++++++++ tests/e2e/cases/cli/cache-clear/test.yaml | 21 +++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 tests/e2e/cases/cli/cache-clear/files/case.yaml create mode 100644 tests/e2e/cases/cli/cache-clear/files/oats.toml create mode 100644 tests/e2e/cases/cli/cache-clear/test.yaml diff --git a/tests/e2e/cases/cli/cache-clear/files/case.yaml b/tests/e2e/cases/cli/cache-clear/files/case.yaml new file mode 100644 index 00000000..4d4ff123 --- /dev/null +++ b/tests/e2e/cases/cli/cache-clear/files/case.yaml @@ -0,0 +1,11 @@ +oats-schema-version: 3 +name: cache-clear-case +seed: + type: inline-otlp + logs: + - service: cache-clear-case + body: cache-clear-seed-line +expected: + logs: + - logql: '{service_name="cache-clear-case"}' + contains: cache-clear-seed-line diff --git a/tests/e2e/cases/cli/cache-clear/files/oats.toml b/tests/e2e/cases/cli/cache-clear/files/oats.toml new file mode 100644 index 00000000..4eddddc1 --- /dev/null +++ b/tests/e2e/cases/cli/cache-clear/files/oats.toml @@ -0,0 +1,11 @@ +[meta] +version = 2 + +[[suite]] +name = "cache-clear" +cases = ["case.yaml"] +fixture = "remote-lgtm" + +[fixture.remote-lgtm] +type = "remote" +endpoint = "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/cache-clear/test.yaml b/tests/e2e/cases/cli/cache-clear/test.yaml new file mode 100644 index 00000000..2215260e --- /dev/null +++ b/tests/e2e/cases/cli/cache-clear/test.yaml @@ -0,0 +1,21 @@ +name: cache-clear +tags: [cli, cache] + +# Populate the cache with one green run, then `oats cache clear` it. The run +# after the clear must re-execute the case instead of skipping it: no SKIP line +# appears anywhere (the sibling cache-skip case proves a rerun WOULD skip +# without the clear). +run: + shell: | + set -e + {{OATS}} --config files/oats.toml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v + {{OATS}} cache clear --cache-dir "$PWD/cache" + echo "=== rerun after clear ===" + {{OATS}} --config files/oats.toml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v + +expect: + stdout_contains: + - "cache cleared:" + - "PASS" + stdout_not_contains: + - "SKIP" From e40c00a3a5cf55f3f63fe306ca5ecc61524f32e3 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 17:48:38 +0200 Subject: [PATCH 102/124] refactor(fixture): extract fixture lifecycle into its own package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move suite fixture boot/teardown out of internal/cli into a new pluggable fixture package (fixture.go, compose.go, k3d.go). The CLI keeps suite orchestration and reporting; the package owns standing up compose/k3d/remote backends, waiting for readiness, and parallel-safety. Exports Runtime, Handle, Start, WaitForReady, SupportsParallel; compose/k3d/ gcx-config/token/port helpers stay unexported and are tested white-box. Pure refactor, no behavior change — unit, integration, and compose/k3d/remote e2e all pass unchanged. The previously dead waitForGrafanaToken seam (only kept alive by test overrides, never called in production) now has a real unit test. Signed-off-by: Gregor Zeitlinger --- fixture/compose.go | 322 +++++++++++++++ fixture/fixture.go | 283 ++++++++++++++ fixture/fixture_test.go | 382 ++++++++++++++++++ fixture/k3d.go | 91 +++++ internal/cli/integration_test.go | 361 +---------------- internal/cli/main.go | 653 +------------------------------ 6 files changed, 1097 insertions(+), 995 deletions(-) create mode 100644 fixture/compose.go create mode 100644 fixture/fixture.go create mode 100644 fixture/fixture_test.go create mode 100644 fixture/k3d.go diff --git a/fixture/compose.go b/fixture/compose.go new file mode 100644 index 00000000..10a38ca9 --- /dev/null +++ b/fixture/compose.go @@ -0,0 +1,322 @@ +package fixture + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/grafana/oats/discovery" +) + +func startCompose(plan discovery.Plan) (Handle, Runtime, error) { + composeFiles, cleanup, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) + if err != nil { + return nil, Runtime{}, err + } + project := composeProjectName(plan) + suiteEnv := append([]string(nil), plan.Fixture.Env...) + suiteEnv = append(suiteEnv, "COMPOSE_PROJECT_NAME="+project) + suite, err := newComposeSuite(composeFiles, suiteEnv) + if err != nil { + if cleanup != nil { + _ = cleanup() + } + return nil, Runtime{}, err + } + if err := startSuiteFixture(suite); err != nil { + if cleanup != nil { + _ = cleanup() + } + return nil, Runtime{}, err + } + grafanaPort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "3000") + if err != nil { + _ = suite.Close() + if cleanup != nil { + _ = cleanup() + } + return nil, Runtime{}, err + } + otlpPort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "4318") + if err != nil { + _ = suite.Close() + if cleanup != nil { + _ = cleanup() + } + return nil, Runtime{}, err + } + pyroscopePort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "4040") + if err != nil { + _ = suite.Close() + if cleanup != nil { + _ = cleanup() + } + return nil, Runtime{}, err + } + rt := Runtime{ + GrafanaURL: "http://127.0.0.1:" + grafanaPort, + OTLPHTTP: "http://127.0.0.1:" + otlpPort, + PyroscopeURL: "http://127.0.0.1:" + pyroscopePort, + ComposeFiles: composeFiles, + ComposeProject: project, + } + cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL) + if cfgErr != nil { + if cleanup != nil { + _ = cleanup() + } + return nil, Runtime{}, fmt.Errorf("write local gcx config: %w", cfgErr) + } + rt.GCXConfig = cfg + cleanup = chainCleanup(func() error { return removeIfExists(cfg) }, cleanup) + rt.CustomCheckEnv = composeCheckEnv(plan, rt) + rt.ParallelSafe, rt.ParallelDisabled = SupportsParallel(plan) + return composeFixture{suite: suite, cleanup: cleanup}, rt, nil +} + +func resolveComposeFiles(sourceDir string, fixture discovery.FixtureConfig) ([]string, func() error, error) { + var files []string + var cleanup func() error + if fixture.Template == "lgtm" { + f, err := writeBuiltinLGTMCompose(sourceDir) + if err != nil { + return nil, nil, err + } + files = append(files, f) + cleanup = func() error { return os.Remove(f) } + } else if fixture.Template != "" { + return nil, nil, fmt.Errorf("unsupported compose fixture template %q", fixture.Template) + } + switch { + case fixture.ComposeFile != "": + files = append(files, filepath.Join(sourceDir, fixture.ComposeFile)) + case len(fixture.ComposeFiles) > 0: + for _, file := range fixture.ComposeFiles { + files = append(files, filepath.Join(sourceDir, file)) + } + case fixture.Template == "": + return nil, nil, fmt.Errorf("compose fixture requires compose_file, compose_files, or supported template") + } + return files, cleanup, nil +} + +func writeBuiltinLGTMCompose(sourceDir string) (string, error) { + if err := os.MkdirAll(sourceDir, 0o755); err != nil { + return "", err + } + f, err := os.CreateTemp(sourceDir, ".oats.lgtm.*.compose.yml") + if err != nil { + return "", err + } + path := f.Name() + const body = `services: + lgtm: + image: ${LGTM_IMAGE:-docker.io/grafana/otel-lgtm:latest} + ports: + - "127.0.0.1::3000" + - "127.0.0.1::4317" + - "127.0.0.1::4318" + - "127.0.0.1::3200" + - "127.0.0.1::4040" + - "127.0.0.1::9090" +` + if _, err := f.WriteString(body); err != nil { + _ = f.Close() + _ = os.Remove(path) + return "", err + } + if err := f.Close(); err != nil { + _ = os.Remove(path) + return "", err + } + return path, nil +} + +func composeCheckEnv(plan discovery.Plan, rt Runtime) []string { + files := rt.ComposeFiles + if len(files) == 0 { + return []string{"OATS_FIXTURE_TYPE=compose"} + } + return []string{ + "OATS_FIXTURE_TYPE=compose", + "COMPOSE_PROJECT_NAME=" + rt.ComposeProject, + "COMPOSE_FILE=" + strings.Join(files, string(os.PathListSeparator)), + "OATS_COMPOSE_FILE_ARGS=" + composeFileArgs(files), + "OATS_GRAFANA_URL=" + rt.GrafanaURL, + "OATS_OTLP_HTTP=" + rt.OTLPHTTP, + "OATS_PYROSCOPE_URL=" + rt.PyroscopeURL, + } +} + +func composeFileArgs(files []string) string { + var parts []string + for _, f := range files { + parts = append(parts, "-f", shellQuote(f)) + } + return strings.Join(parts, " ") +} + +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +func composeProjectName(plan discovery.Plan) string { + name := strings.ToLower(plan.Suite.Name) + if name == "" { + name = "oats" + } + var b strings.Builder + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + default: + b.WriteByte('-') + } + } + slug := strings.Trim(b.String(), "-") + if slug == "" { + slug = "oats" + } + if len(slug) > 32 { + slug = slug[:32] + } + return fmt.Sprintf("oats-%s-%d", slug, time.Now().UnixNano()) +} + +func extraComposeFiles(plan discovery.Plan) []string { + switch { + case plan.Fixture.ComposeFile != "": + return []string{filepath.Join(plan.FixtureSourceDir, plan.Fixture.ComposeFile)} + case len(plan.Fixture.ComposeFiles) > 0: + files := make([]string, 0, len(plan.Fixture.ComposeFiles)) + for _, file := range plan.Fixture.ComposeFiles { + files = append(files, filepath.Join(plan.FixtureSourceDir, file)) + } + return files + default: + return nil + } +} + +func composeFilePublishesFixedHostPorts(path string) (bool, error) { + data, err := os.ReadFile(path) + if err != nil { + return false, err + } + lines := strings.Split(string(data), "\n") + inPorts := false + portsIndent := 0 + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + indent := len(line) - len(strings.TrimLeft(line, " ")) + if inPorts && indent <= portsIndent { + inPorts = false + } + if strings.HasPrefix(trimmed, "ports:") { + inPorts = true + portsIndent = indent + continue + } + if !inPorts { + continue + } + if strings.Contains(trimmed, "published:") { + value := strings.TrimSpace(strings.TrimPrefix(trimmed, "published:")) + if value != "" && value != "0" { + return true, nil + } + continue + } + if !strings.HasPrefix(trimmed, "-") { + continue + } + value := strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "-")), `"'`) + if value == "" { + continue + } + if fixedShortPortMapping(value) { + return true, nil + } + } + return false, nil +} + +func fixedShortPortMapping(value string) bool { + if !strings.Contains(value, ":") { + return false + } + parts := strings.Split(value, ":") + if len(parts) < 2 { + return false + } + hostPart := strings.Trim(parts[len(parts)-2], "[]") + if _, err := strconv.Atoi(hostPart); err == nil && hostPart != "0" { + return true + } + return false +} + +func readComposeGrafanaToken(plan discovery.Plan) (string, error) { + files, _, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) + if err != nil { + return "", err + } + args := []string{"compose"} + for _, f := range files { + args = append(args, "-f", f) + } + args = append(args, "exec", "-T", "lgtm", "sh", "-c", "cat /tmp/grafana-sa-token 2>/dev/null || true") + cmd := exec.Command("docker", args...) + cmd.Env = append(cmd.Environ(), plan.Fixture.Env...) + out, err := cmd.Output() + if err != nil { + return "", err + } + return string(out), nil +} + +func dockerComposePort(files []string, env []string, service, containerPort string) (string, error) { + args := []string{"compose"} + for _, f := range files { + args = append(args, "-f", f) + } + args = append(args, "port", service, containerPort) + cmd := exec.Command("docker", args...) + cmd.Env = append(cmd.Environ(), env...) + out, err := cmd.Output() + if err != nil { + return "", err + } + host, port, err := splitDockerHostPort(strings.TrimSpace(string(out))) + if err != nil { + return "", err + } + if host == "" || port == "" { + return "", fmt.Errorf("invalid docker compose port output %q", strings.TrimSpace(string(out))) + } + return port, nil +} + +func splitDockerHostPort(addr string) (string, string, error) { + addr = strings.TrimSpace(addr) + if strings.HasPrefix(addr, "[") { + end := strings.Index(addr, "]") + if end < 0 || end+2 > len(addr) || addr[end+1] != ':' { + return "", "", fmt.Errorf("invalid address %q", addr) + } + return addr[1:end], addr[end+2:], nil + } + idx := strings.LastIndex(addr, ":") + if idx < 0 { + return "", "", fmt.Errorf("invalid address %q", addr) + } + return addr[:idx], addr[idx+1:], nil +} diff --git a/fixture/fixture.go b/fixture/fixture.go new file mode 100644 index 00000000..7da49cdb --- /dev/null +++ b/fixture/fixture.go @@ -0,0 +1,283 @@ +// Package fixture owns the lifecycle of the observability backends a suite +// runs against: booting a docker-compose or k3d stack, waiting for it to be +// ready, exposing the resolved endpoints as a Runtime, and tearing it down. +// The CLI orchestrates suites and reporting; this package abstracts "stand up +// the stack the cases need" behind a small, pluggable surface. +package fixture + +import ( + "context" + "fmt" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/grafana/oats/discovery" + "github.com/grafana/oats/testhelpers/compose" + "github.com/grafana/oats/testhelpers/kubernetes" + "github.com/grafana/oats/testhelpers/remote" +) + +// Seams overridable in tests. +var ( + newComposeSuite = func(files []string, env []string) (Handle, error) { + return compose.SuiteFiles(files, env) + } + newKubernetesEndpoint = func(plan discovery.Plan, ports remote.PortsConfig) *remote.Endpoint { + sourceDir := plan.FixtureSourceDir + if sourceDir == "" { + sourceDir = "." + } + model := &kubernetes.Kubernetes{ + Dir: plan.Fixture.K8sDir, + AppService: plan.Fixture.AppService, + AppDockerFile: plan.Fixture.AppDockerFile, + AppDockerContext: plan.Fixture.AppDockerContext, + AppDockerTag: plan.Fixture.AppDockerTag, + AppDockerPort: plan.Fixture.AppPort, + ImportImages: plan.Fixture.ImportImages, + } + return kubernetes.NewEndpoint("localhost", model, ports, plan.Suite.Name, sourceDir) + } + waitForGrafanaToken = waitForGrafanaTokenImpl + lookupComposePort = dockerComposePort +) + +// Runtime carries the resolved coordinates of a booted fixture back to the +// caller: where the backends live, the gcx config to talk to them, the compose +// project (for teardown/labels), and whether the fixture is parallel-safe. +type Runtime struct { + GrafanaURL string + OTLPHTTP string + PyroscopeURL string + CustomCheckEnv []string + ComposeFiles []string + ComposeProject string + GCXConfig string + ParallelSafe bool + ParallelDisabled string +} + +// Handle is a booted fixture that can be torn down. +type Handle interface { + Close() error +} + +type startableHandle interface { + Handle + Up() error +} + +// Start boots the fixture declared by the plan and returns a Handle for +// teardown plus the resolved Runtime. Remote/empty fixtures need no boot and +// return a nil Handle. +func Start(ctx context.Context, plan discovery.Plan) (Handle, Runtime, error) { + switch plan.Fixture.Type { + case "", "remote": + return nil, Runtime{ParallelSafe: true}, nil + case "compose": + return startCompose(plan) + case "k3d": + return startK3D(ctx, plan) + default: + return nil, Runtime{}, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) + } +} + +// WaitForReady blocks until the fixture's Grafana and OTLP endpoints answer, or +// the per-endpoint timeout elapses. Remote/empty fixtures are assumed ready. +func WaitForReady(plan discovery.Plan, rt Runtime) error { + switch plan.Fixture.Type { + case "compose", "k3d": + if err := waitForHTTP(rt.GrafanaURL+"/api/health", 2*time.Minute); err != nil { + return fmt.Errorf("wait for grafana: %w", err) + } + if err := waitForHTTP(rt.OTLPHTTP, 2*time.Minute); err != nil { + return fmt.Errorf("wait for otlp-http: %w", err) + } + } + return nil +} + +// SupportsParallel reports whether a suite on this fixture can run alongside +// other suites, and if not, a human-readable reason. +func SupportsParallel(plan discovery.Plan) (bool, string) { + switch plan.Fixture.Type { + case "", "remote": + return true, "" + case "compose": + if plan.Fixture.Template != "lgtm" { + return false, "compose fixtures are only parallel-safe when OATS owns the LGTM ports via template=lgtm" + } + for _, c := range plan.Cases { + if c.Seed.Type == "app" { + return false, "compose suites with app seeds still rely on shared fixed app ports" + } + } + for _, file := range extraComposeFiles(plan) { + if fixed, err := composeFilePublishesFixedHostPorts(file); err != nil { + return false, fmt.Sprintf("compose port inspection failed for %s: %v", file, err) + } else if fixed { + return false, fmt.Sprintf("compose file %s publishes fixed host ports", filepath.Base(file)) + } + } + return true, "" + case "k3d": + return false, "k3d fixtures currently use shared localhost port-forwards" + default: + return false, "fixture type is not parallel-safe" + } +} + +func startSuiteFixture(fix Handle) error { + startable, ok := fix.(startableHandle) + if !ok { + return fmt.Errorf("fixture does not support startup") + } + return startable.Up() +} + +type endpointFixture struct { + ep *remote.Endpoint + cleanup func() error +} + +func (e endpointFixture) Close() error { + err := e.ep.Stop(context.Background()) + if e.cleanup != nil { + if cleanupErr := e.cleanup(); cleanupErr != nil && err == nil { + err = cleanupErr + } + } + return err +} + +type composeFixture struct { + suite Handle + cleanup func() error +} + +func (c composeFixture) Close() error { + err := c.suite.Close() + if c.cleanup != nil { + if cleanupErr := c.cleanup(); cleanupErr != nil && err == nil { + err = cleanupErr + } + } + return err +} + +// removeIfExists deletes path, ignoring a not-exist error so cleanup is +// idempotent. +func removeIfExists(path string) error { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// chainCleanup returns a cleanup that runs first then second, returning the +// first non-nil error. Either argument may be nil. +func chainCleanup(first, second func() error) func() error { + return func() error { + var err error + if first != nil { + err = first() + } + if second != nil { + if e := second(); e != nil && err == nil { + err = e + } + } + return err + } +} + +func writeLocalGCXConfig(grafanaURL string) (string, error) { + cfg := fmt.Sprintf(`current-context: local +contexts: + local: + grafana: + server: %s + user: admin + password: admin + org-id: 1 + auth-method: basic + datasources: + prometheus: prometheus + loki: loki + tempo: tempo + pyroscope: pyroscope +`, grafanaURL) + f, err := os.CreateTemp("", "oats-gcx-*.yaml") + if err != nil { + return "", err + } + path := f.Name() + if _, err := f.WriteString(cfg); err != nil { + _ = f.Close() + _ = os.Remove(path) + return "", err + } + if err := f.Close(); err != nil { + _ = os.Remove(path) + return "", err + } + if err := os.Chmod(path, 0o600); err != nil { + _ = os.Remove(path) + return "", err + } + return path, nil +} + +func waitForGrafanaTokenImpl(plan discovery.Plan) (string, error) { + deadline := time.Now().Add(3 * time.Minute) + for time.Now().Before(deadline) { + var token string + var err error + switch plan.Fixture.Type { + case "compose": + token, err = readComposeGrafanaToken(plan) + case "k3d": + token, err = readK3DGrafanaToken() + default: + return "", nil + } + if err == nil && strings.TrimSpace(token) != "" { + return strings.TrimSpace(token), nil + } + time.Sleep(time.Second) + } + return "", fmt.Errorf("timed out waiting for Grafana service-account token") +} + +func findFreePort() (int, error) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return 0, err + } + defer func() { _ = ln.Close() }() + addr, ok := ln.Addr().(*net.TCPAddr) + if !ok { + return 0, fmt.Errorf("unexpected listener address %T", ln.Addr()) + } + return addr.Port, nil +} + +func waitForHTTP(url string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + resp, err := http.Get(url) //nolint:gosec + if err == nil { + _ = resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 500 { + return nil + } + } + time.Sleep(2 * time.Second) + } + return fmt.Errorf("timed out waiting for %s", url) +} diff --git a/fixture/fixture_test.go b/fixture/fixture_test.go new file mode 100644 index 00000000..b7b12e27 --- /dev/null +++ b/fixture/fixture_test.go @@ -0,0 +1,382 @@ +package fixture + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/grafana/oats/discovery" + "github.com/grafana/oats/runner" + "github.com/grafana/oats/testhelpers/remote" +) + +func TestResolveComposeFiles(t *testing.T) { + got, cleanup, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFile: "stack/compose.yml"}) + if err != nil { + t.Fatalf("resolveComposeFiles compose_file: %v", err) + } + if cleanup != nil { + t.Fatalf("unexpected cleanup for compose_file fixture") + } + if want := []string{"/tmp/work/stack/compose.yml"}; len(got) != 1 || got[0] != want[0] { + t.Fatalf("got %q want %q", got, want) + } + + got, cleanup, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFiles: []string{"a.yml", "b.yml"}}) + if err != nil { + t.Fatalf("resolveComposeFiles compose_files: %v", err) + } + if cleanup != nil { + t.Fatalf("unexpected cleanup for compose_files fixture") + } + if len(got) != 2 || got[0] != "/tmp/work/a.yml" || got[1] != "/tmp/work/b.yml" { + t.Fatalf("unexpected compose_files resolution: %v", got) + } + + got, cleanup, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "lgtm"}) + if err != nil { + t.Fatalf("resolveComposeFiles template=lgtm: %v", err) + } + if cleanup == nil { + t.Fatalf("expected cleanup for template=lgtm fixture") + } + defer func() { _ = cleanup() }() + if len(got) != 1 || !strings.Contains(filepath.Base(got[0]), ".oats.lgtm.") || !strings.HasSuffix(got[0], ".compose.yml") { + t.Fatalf("unexpected template=lgtm resolution: %v", got) + } +} + +func TestStart_ComposeLifecycle(t *testing.T) { + oldFactory := newComposeSuite + oldLookup := lookupComposePort + defer func() { newComposeSuite = oldFactory }() + defer func() { lookupComposePort = oldLookup }() + + var gotFiles, gotEnv []string + fake := &fakeHandle{} + newComposeSuite = func(files []string, env []string) (Handle, error) { + gotFiles = append([]string(nil), files...) + gotEnv = append([]string(nil), env...) + return fake, nil + } + lookupComposePort = func(files []string, env []string, service, containerPort string) (string, error) { + switch containerPort { + case "3000": + return "43000", nil + case "4318": + return "44318", nil + case "4040": + return "44040", nil + default: + return "", fmt.Errorf("unexpected port %s", containerPort) + } + } + + fix, _, err := Start(context.Background(), discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, + Fixture: discovery.FixtureConfig{ + Type: "compose", + ComposeFiles: []string{"a.yml", "b.yml"}, + Env: []string{"FOO=bar"}, + }, + FixtureSourceDir: "/tmp/work", + }) + if err != nil { + t.Fatalf("Start compose: %v", err) + } + if fake.upCalls != 1 { + t.Fatalf("expected Up once, got %d", fake.upCalls) + } + if want := []string{"/tmp/work/a.yml", "/tmp/work/b.yml"}; !equalStrings(gotFiles, want) { + t.Fatalf("compose files: got %v want %v", gotFiles, want) + } + if len(gotEnv) != 2 || gotEnv[0] != "FOO=bar" || !strings.HasPrefix(gotEnv[1], "COMPOSE_PROJECT_NAME=oats-smoke-") { + t.Fatalf("compose env: got %v", gotEnv) + } + if err := fix.Close(); err != nil { + t.Fatalf("fixture close: %v", err) + } + if fake.closeCalls != 1 { + t.Fatalf("expected Close once, got %d", fake.closeCalls) + } +} + +func TestStart_ComposeStartFailure(t *testing.T) { + oldFactory := newComposeSuite + defer func() { newComposeSuite = oldFactory }() + + newComposeSuite = func(files []string, env []string) (Handle, error) { + return &fakeHandle{upErr: fmt.Errorf("boom")}, nil + } + + _, _, err := Start(context.Background(), discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, + Fixture: discovery.FixtureConfig{Type: "compose", ComposeFile: "compose.yml"}, + FixtureSourceDir: "/tmp/work", + }) + if err == nil || !strings.Contains(err.Error(), "boom") { + t.Fatalf("expected compose startup error, got %v", err) + } +} + +func TestStart_K3DLifecycle(t *testing.T) { + oldFactory := newKubernetesEndpoint + defer func() { newKubernetesEndpoint = oldFactory }() + + var capturedPlan discovery.Plan + var starts, stops int + var capturedPorts remote.PortsConfig + newKubernetesEndpoint = func(plan discovery.Plan, ports remote.PortsConfig) *remote.Endpoint { + capturedPlan = plan + capturedPorts = ports + return remote.NewEndpoint("localhost", remote.PortsConfig{}, func(ctx context.Context) error { + starts++ + return nil + }, func(ctx context.Context) error { + stops++ + return nil + }, nil) + } + + fix, _, err := Start(context.Background(), discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, + Fixture: discovery.FixtureConfig{ + Type: "k3d", + K8sDir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerContext: ".", + AppDockerTag: "dice:test", + AppPort: 18080, + ImportImages: []string{"busybox:latest"}, + }, + FixtureSourceDir: "/tmp/work", + }) + if err != nil { + t.Fatalf("Start k3d: %v", err) + } + if starts != 1 { + t.Fatalf("expected one endpoint start, got %d", starts) + } + if capturedPlan.FixtureSourceDir != "/tmp/work" || capturedPlan.Suite.Name != "cluster-smoke" || capturedPlan.Fixture.AppPort != 18080 { + t.Fatalf("unexpected endpoint factory args: plan=%+v", capturedPlan) + } + if capturedPorts.GrafanaHTTPPort == 0 || capturedPorts.OTLPHTTPPort == 0 || capturedPorts.LokiHttpPort == 0 || capturedPorts.PrometheusHTTPPort == 0 || capturedPorts.TempoHTTPPort == 0 || capturedPorts.PyroscopeHttpPort == 0 { + t.Fatalf("expected allocated k3d ports, got %+v", capturedPorts) + } + if err := fix.Close(); err != nil { + t.Fatalf("fixture close: %v", err) + } + if stops != 1 { + t.Fatalf("expected one endpoint stop, got %d", stops) + } +} + +func TestStart_K3DStartFailure(t *testing.T) { + oldFactory := newKubernetesEndpoint + defer func() { newKubernetesEndpoint = oldFactory }() + + newKubernetesEndpoint = func(plan discovery.Plan, ports remote.PortsConfig) *remote.Endpoint { + return remote.NewEndpoint("localhost", remote.PortsConfig{}, func(ctx context.Context) error { + return fmt.Errorf("cluster boom") + }, func(ctx context.Context) error { + return nil + }, nil) + } + + _, _, err := Start(context.Background(), discovery.Plan{ + Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, + Fixture: discovery.FixtureConfig{ + Type: "k3d", + K8sDir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerContext: ".", + AppDockerTag: "dice:test", + AppPort: 18080, + }, + FixtureSourceDir: "/tmp/work", + }) + if err == nil || !strings.Contains(err.Error(), "cluster boom") { + t.Fatalf("expected k3d startup error, got %v", err) + } +} + +func TestK3DCheckEnv_UsesConfiguredPorts(t *testing.T) { + got := k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: 18080}, remote.PortsConfig{ + GrafanaHTTPPort: 13000, + OTLPHTTPPort: 14318, + PyroscopeHttpPort: 14040, + }) + for _, want := range []string{ + "OATS_APP_URL=http://localhost:18080", + "OATS_GRAFANA_URL=http://localhost:13000", + "OATS_OTLP_HTTP=http://localhost:14318", + "OATS_PYROSCOPE_URL=http://localhost:14040", + } { + if !containsString(got, want) { + t.Fatalf("k3dCheckEnv missing %q in %v", want, got) + } + } +} + +func TestResolveComposeFiles_UnsupportedTemplate(t *testing.T) { + _, _, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "weird"}) + if err == nil || !strings.Contains(err.Error(), `unsupported compose fixture template "weird"`) { + t.Fatalf("expected unsupported template error, got %v", err) + } +} + +func TestResolveComposeFiles_MissingConfig(t *testing.T) { + _, _, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose"}) + if err == nil || !strings.Contains(err.Error(), "compose fixture requires compose_file, compose_files, or supported template") { + t.Fatalf("expected missing compose config error, got %v", err) + } +} + +func TestComposeFilePublishesFixedHostPorts(t *testing.T) { + dir := t.TempDir() + fixed := filepath.Join(dir, "fixed.yml") + random := filepath.Join(dir, "random.yml") + writeFile(t, dir, "fixed.yml", `services: + app: + image: alpine + ports: + - "8080:8080" +`) + writeFile(t, dir, "random.yml", `services: + app: + image: alpine + ports: + - "8080" +`) + got, err := composeFilePublishesFixedHostPorts(fixed) + if err != nil { + t.Fatalf("composeFilePublishesFixedHostPorts fixed: %v", err) + } + if !got { + t.Fatalf("expected fixed host port detection for %s", fixed) + } + got, err = composeFilePublishesFixedHostPorts(random) + if err != nil { + t.Fatalf("composeFilePublishesFixedHostPorts random: %v", err) + } + if got { + t.Fatalf("did not expect fixed host port detection for %s", random) + } +} + +func TestSupportsParallel_ComposeTemplateLGTM(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats.toml", ` +[meta] +version = 2 + +[[suite]] +name = "parallel-safe" +cases = ["cases/*.yaml"] +fixture = "stack" + +[fixture.stack] +type = "compose" +template = "lgtm" +compose_file = "docker-compose.oats.yml" +`) + writeFile(t, dir, "docker-compose.oats.yml", `services: + app: + image: alpine + command: ["sh", "-c", "sleep 1"] +`) + writeFile(t, dir, "cases/a.yaml", `oats-schema-version: 3 +name: a +seed: + type: inline-otlp + logs: + - service: a + body: line +expected: + logs: + - logql: '{service_name="a"}' + contains: line +`) + + cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(discovery.Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + safe, reason := SupportsParallel(plans[0]) + if !safe { + t.Fatalf("expected plan to be parallel-safe, got false: %s", reason) + } +} + +// TestWaitForGrafanaToken_DefaultFixtureReturnsEmpty exercises the token seam +// for a fixture type that has no token to read: it returns immediately with an +// empty token and no error. +func TestWaitForGrafanaToken_DefaultFixtureReturnsEmpty(t *testing.T) { + token, err := waitForGrafanaToken(discovery.Plan{ + Fixture: discovery.FixtureConfig{Type: "remote"}, + }) + if err != nil { + t.Fatalf("waitForGrafanaToken: %v", err) + } + if token != "" { + t.Fatalf("expected empty token for remote fixture, got %q", token) + } +} + +func writeFile(t *testing.T, dir, rel, body string) { + t.Helper() + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +type fakeHandle struct { + upCalls int + closeCalls int + upErr error + closeErr error +} + +func (f *fakeHandle) Up() error { + f.upCalls++ + return f.upErr +} + +func (f *fakeHandle) Close() error { + f.closeCalls++ + return f.closeErr +} + +func equalStrings(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} + +func containsString(items []string, want string) bool { + for _, item := range items { + if item == want { + return true + } + } + return false +} diff --git a/fixture/k3d.go b/fixture/k3d.go new file mode 100644 index 00000000..f3310b43 --- /dev/null +++ b/fixture/k3d.go @@ -0,0 +1,91 @@ +package fixture + +import ( + "context" + "fmt" + "os/exec" + + "github.com/grafana/oats/discovery" + "github.com/grafana/oats/runner" + "github.com/grafana/oats/testhelpers/remote" +) + +func startK3D(ctx context.Context, plan discovery.Plan) (Handle, Runtime, error) { + ports, err := allocateK3DPorts() + if err != nil { + return nil, Runtime{}, err + } + ep := newKubernetesEndpoint(plan, ports) + if err := ep.Start(ctx); err != nil { + return nil, Runtime{}, err + } + rt := Runtime{ + GrafanaURL: fmt.Sprintf("http://localhost:%d", ports.GrafanaHTTPPort), + OTLPHTTP: fmt.Sprintf("http://localhost:%d", ports.OTLPHTTPPort), + PyroscopeURL: fmt.Sprintf("http://localhost:%d", ports.PyroscopeHttpPort), + CustomCheckEnv: k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: plan.Fixture.AppPort}, ports), + ParallelSafe: false, + ParallelDisabled: "k3d fixtures currently use shared clusters and kubectl port-forwards", + } + cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL) + if cfgErr != nil { + _ = ep.Stop(context.Background()) + return nil, Runtime{}, fmt.Errorf("write local gcx config: %w", cfgErr) + } + rt.GCXConfig = cfg + return endpointFixture{ep: ep, cleanup: func() error { return removeIfExists(cfg) }}, rt, nil +} + +func k3dCheckEnv(ep runner.Endpoint, ports remote.PortsConfig) []string { + return []string{ + "OATS_FIXTURE_TYPE=k3d", + "OATS_APP_URL=" + fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), + "OATS_GRAFANA_URL=" + fmt.Sprintf("http://localhost:%d", ports.GrafanaHTTPPort), + "OATS_OTLP_HTTP=" + fmt.Sprintf("http://localhost:%d", ports.OTLPHTTPPort), + "OATS_PYROSCOPE_URL=" + fmt.Sprintf("http://localhost:%d", ports.PyroscopeHttpPort), + } +} + +func allocateK3DPorts() (remote.PortsConfig, error) { + grafanaPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + otlpHTTPPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + lokiPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + promPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + tempoPort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + pyroscopePort, err := findFreePort() + if err != nil { + return remote.PortsConfig{}, err + } + return remote.PortsConfig{ + GrafanaHTTPPort: grafanaPort, + OTLPHTTPPort: otlpHTTPPort, + LokiHttpPort: lokiPort, + PrometheusHTTPPort: promPort, + TempoHTTPPort: tempoPort, + PyroscopeHttpPort: pyroscopePort, + }, nil +} + +func readK3DGrafanaToken() (string, error) { + cmd := exec.Command("kubectl", "exec", "deploy/lgtm", "--", "sh", "-c", "cat /tmp/grafana-sa-token 2>/dev/null || true") + out, err := cmd.Output() + if err != nil { + return "", err + } + return string(out), nil +} diff --git a/internal/cli/integration_test.go b/internal/cli/integration_test.go index 0ace6898..88bb6350 100644 --- a/internal/cli/integration_test.go +++ b/internal/cli/integration_test.go @@ -3,7 +3,6 @@ package cli import ( "bytes" "context" - "fmt" "net" "net/http" "net/http/httptest" @@ -17,57 +16,17 @@ import ( "github.com/grafana/oats/discovery" "github.com/grafana/oats/engine" + "github.com/grafana/oats/fixture" "github.com/grafana/oats/migrate" "github.com/grafana/oats/report" "github.com/grafana/oats/runner" - "github.com/grafana/oats/testhelpers/remote" ) -func TestResolveComposeFiles(t *testing.T) { - got, cleanup, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFile: "stack/compose.yml"}) - if err != nil { - t.Fatalf("resolveComposeFiles compose_file: %v", err) - } - if cleanup != nil { - t.Fatalf("unexpected cleanup for compose_file fixture") - } - if want := []string{"/tmp/work/stack/compose.yml"}; len(got) != 1 || got[0] != want[0] { - t.Fatalf("got %q want %q", got, want) - } - - got, cleanup, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFiles: []string{"a.yml", "b.yml"}}) - if err != nil { - t.Fatalf("resolveComposeFiles compose_files: %v", err) - } - if cleanup != nil { - t.Fatalf("unexpected cleanup for compose_files fixture") - } - if len(got) != 2 || got[0] != "/tmp/work/a.yml" || got[1] != "/tmp/work/b.yml" { - t.Fatalf("unexpected compose_files resolution: %v", got) - } - - got, cleanup, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "lgtm"}) - if err != nil { - t.Fatalf("resolveComposeFiles template=lgtm: %v", err) - } - if cleanup == nil { - t.Fatalf("expected cleanup for template=lgtm fixture") - } - defer func() { _ = cleanup() }() - if len(got) != 1 || !strings.Contains(filepath.Base(got[0]), ".oats.lgtm.") || !strings.HasSuffix(got[0], ".compose.yml") { - t.Fatalf("unexpected template=lgtm resolution: %v", got) - } -} - func TestResolveEndpoint_ComposeDefaults(t *testing.T) { - oldToken := waitForGrafanaToken - waitForGrafanaToken = func(plan discovery.Plan) (string, error) { return "tok", nil } - defer func() { waitForGrafanaToken = oldToken }() - ep, err := resolveEndpoint(discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, Fixture: discovery.FixtureConfig{Type: "compose", Template: "lgtm"}, - }, fixtureRuntime{GCXConfig: "/tmp/gcx.yaml", OTLPHTTP: "http://127.0.0.1:4318"}, "", "localhost", 8080, "http://localhost:4318") + }, fixture.Runtime{GCXConfig: "/tmp/gcx.yaml", OTLPHTTP: "http://127.0.0.1:4318"}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -77,14 +36,10 @@ func TestResolveEndpoint_ComposeDefaults(t *testing.T) { } func TestResolveEndpoint_K3DUsesFixtureAppPort(t *testing.T) { - oldToken := waitForGrafanaToken - waitForGrafanaToken = func(plan discovery.Plan) (string, error) { return "tok", nil } - defer func() { waitForGrafanaToken = oldToken }() - ep, err := resolveEndpoint(discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "cluster"}, Fixture: discovery.FixtureConfig{Type: "k3d", AppPort: 18080}, - }, fixtureRuntime{GCXConfig: "/tmp/gcx.yaml", OTLPHTTP: "http://127.0.0.1:4318"}, "", "localhost", 8080, "http://localhost:4318") + }, fixture.Runtime{GCXConfig: "/tmp/gcx.yaml", OTLPHTTP: "http://127.0.0.1:4318"}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -93,274 +48,6 @@ func TestResolveEndpoint_K3DUsesFixtureAppPort(t *testing.T) { } } -func TestStartFixture_ComposeLifecycle(t *testing.T) { - oldFactory := newComposeSuite - oldLookup := lookupComposePort - defer func() { newComposeSuite = oldFactory }() - defer func() { lookupComposePort = oldLookup }() - - var gotFiles, gotEnv []string - fake := &fakeSuiteFixture{} - newComposeSuite = func(files []string, env []string) (suiteFixture, error) { - gotFiles = append([]string(nil), files...) - gotEnv = append([]string(nil), env...) - return fake, nil - } - lookupComposePort = func(files []string, env []string, service, containerPort string) (string, error) { - switch containerPort { - case "3000": - return "43000", nil - case "4318": - return "44318", nil - case "4040": - return "44040", nil - default: - return "", fmt.Errorf("unexpected port %s", containerPort) - } - } - - fix, _, err := startFixture(context.Background(), discovery.Plan{ - Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, - Fixture: discovery.FixtureConfig{ - Type: "compose", - ComposeFiles: []string{"a.yml", "b.yml"}, - Env: []string{"FOO=bar"}, - }, - FixtureSourceDir: "/tmp/work", - }) - if err != nil { - t.Fatalf("startFixture compose: %v", err) - } - if fake.upCalls != 1 { - t.Fatalf("expected Up once, got %d", fake.upCalls) - } - if want := []string{"/tmp/work/a.yml", "/tmp/work/b.yml"}; !equalStrings(gotFiles, want) { - t.Fatalf("compose files: got %v want %v", gotFiles, want) - } - if len(gotEnv) != 2 || gotEnv[0] != "FOO=bar" || !strings.HasPrefix(gotEnv[1], "COMPOSE_PROJECT_NAME=oats-smoke-") { - t.Fatalf("compose env: got %v", gotEnv) - } - if err := fix.Close(); err != nil { - t.Fatalf("fixture close: %v", err) - } - if fake.closeCalls != 1 { - t.Fatalf("expected Close once, got %d", fake.closeCalls) - } -} - -func TestStartFixture_ComposeStartFailure(t *testing.T) { - oldFactory := newComposeSuite - defer func() { newComposeSuite = oldFactory }() - - newComposeSuite = func(files []string, env []string) (suiteFixture, error) { - return &fakeSuiteFixture{upErr: fmt.Errorf("boom")}, nil - } - - _, _, err := startFixture(context.Background(), discovery.Plan{ - Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, - Fixture: discovery.FixtureConfig{Type: "compose", ComposeFile: "compose.yml"}, - FixtureSourceDir: "/tmp/work", - }) - if err == nil || !strings.Contains(err.Error(), "boom") { - t.Fatalf("expected compose startup error, got %v", err) - } -} - -func TestStartFixture_K3DLifecycle(t *testing.T) { - oldFactory := newKubernetesEndpoint - defer func() { newKubernetesEndpoint = oldFactory }() - - var capturedPlan discovery.Plan - var starts, stops int - var capturedPorts remote.PortsConfig - newKubernetesEndpoint = func(plan discovery.Plan, ports remote.PortsConfig) *remote.Endpoint { - capturedPlan = plan - capturedPorts = ports - return remote.NewEndpoint("localhost", remote.PortsConfig{}, func(ctx context.Context) error { - starts++ - return nil - }, func(ctx context.Context) error { - stops++ - return nil - }, nil) - } - - fix, _, err := startFixture(context.Background(), discovery.Plan{ - Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, - Fixture: discovery.FixtureConfig{ - Type: "k3d", - K8sDir: "k8s", - AppService: "dice", - AppDockerFile: "Dockerfile", - AppDockerContext: ".", - AppDockerTag: "dice:test", - AppPort: 18080, - ImportImages: []string{"busybox:latest"}, - }, - FixtureSourceDir: "/tmp/work", - }) - if err != nil { - t.Fatalf("startFixture k3d: %v", err) - } - if starts != 1 { - t.Fatalf("expected one endpoint start, got %d", starts) - } - if capturedPlan.FixtureSourceDir != "/tmp/work" || capturedPlan.Suite.Name != "cluster-smoke" || capturedPlan.Fixture.AppPort != 18080 { - t.Fatalf("unexpected endpoint factory args: plan=%+v", capturedPlan) - } - if capturedPorts.GrafanaHTTPPort == 0 || capturedPorts.OTLPHTTPPort == 0 || capturedPorts.LokiHttpPort == 0 || capturedPorts.PrometheusHTTPPort == 0 || capturedPorts.TempoHTTPPort == 0 || capturedPorts.PyroscopeHttpPort == 0 { - t.Fatalf("expected allocated k3d ports, got %+v", capturedPorts) - } - if err := fix.Close(); err != nil { - t.Fatalf("fixture close: %v", err) - } - if stops != 1 { - t.Fatalf("expected one endpoint stop, got %d", stops) - } -} - -func TestStartFixture_K3DStartFailure(t *testing.T) { - oldFactory := newKubernetesEndpoint - defer func() { newKubernetesEndpoint = oldFactory }() - - newKubernetesEndpoint = func(plan discovery.Plan, ports remote.PortsConfig) *remote.Endpoint { - return remote.NewEndpoint("localhost", remote.PortsConfig{}, func(ctx context.Context) error { - return fmt.Errorf("cluster boom") - }, func(ctx context.Context) error { - return nil - }, nil) - } - - _, _, err := startFixture(context.Background(), discovery.Plan{ - Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, - Fixture: discovery.FixtureConfig{ - Type: "k3d", - K8sDir: "k8s", - AppService: "dice", - AppDockerFile: "Dockerfile", - AppDockerContext: ".", - AppDockerTag: "dice:test", - AppPort: 18080, - }, - FixtureSourceDir: "/tmp/work", - }) - if err == nil || !strings.Contains(err.Error(), "cluster boom") { - t.Fatalf("expected k3d startup error, got %v", err) - } -} - -func TestK3DCheckEnv_UsesConfiguredPorts(t *testing.T) { - got := k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: 18080}, remote.PortsConfig{ - GrafanaHTTPPort: 13000, - OTLPHTTPPort: 14318, - PyroscopeHttpPort: 14040, - }) - for _, want := range []string{ - "OATS_APP_URL=http://localhost:18080", - "OATS_GRAFANA_URL=http://localhost:13000", - "OATS_OTLP_HTTP=http://localhost:14318", - "OATS_PYROSCOPE_URL=http://localhost:14040", - } { - if !containsString(got, want) { - t.Fatalf("k3dCheckEnv missing %q in %v", want, got) - } - } -} - -func TestResolveComposeFiles_UnsupportedTemplate(t *testing.T) { - _, _, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "weird"}) - if err == nil || !strings.Contains(err.Error(), `unsupported compose fixture template "weird"`) { - t.Fatalf("expected unsupported template error, got %v", err) - } -} - -func TestResolveComposeFiles_MissingConfig(t *testing.T) { - _, _, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose"}) - if err == nil || !strings.Contains(err.Error(), "compose fixture requires compose_file, compose_files, or supported template") { - t.Fatalf("expected missing compose config error, got %v", err) - } -} - -func TestComposeFilePublishesFixedHostPorts(t *testing.T) { - dir := t.TempDir() - fixed := filepath.Join(dir, "fixed.yml") - random := filepath.Join(dir, "random.yml") - writeFile(t, dir, "fixed.yml", `services: - app: - image: alpine - ports: - - "8080:8080" -`) - writeFile(t, dir, "random.yml", `services: - app: - image: alpine - ports: - - "8080" -`) - got, err := composeFilePublishesFixedHostPorts(fixed) - if err != nil { - t.Fatalf("composeFilePublishesFixedHostPorts fixed: %v", err) - } - if !got { - t.Fatalf("expected fixed host port detection for %s", fixed) - } - got, err = composeFilePublishesFixedHostPorts(random) - if err != nil { - t.Fatalf("composeFilePublishesFixedHostPorts random: %v", err) - } - if got { - t.Fatalf("did not expect fixed host port detection for %s", random) - } -} - -func TestPlanSupportsParallel_ComposeTemplateLGTM(t *testing.T) { - dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "parallel-safe" -cases = ["cases/*.yaml"] -fixture = "stack" - -[fixture.stack] -type = "compose" -template = "lgtm" -compose_file = "docker-compose.oats.yml" -`) - writeFile(t, dir, "docker-compose.oats.yml", `services: - app: - image: alpine - command: ["sh", "-c", "sleep 1"] -`) - writeFile(t, dir, "cases/a.yaml", `oats-schema-version: 3 -name: a -seed: - type: inline-otlp - logs: - - service: a - body: line -expected: - logs: - - logql: '{service_name="a"}' - contains: line -`) - - cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) - if err != nil { - t.Fatalf("Load: %v", err) - } - plans, err := cfg.PlanRun(discovery.Filter{}) - if err != nil { - t.Fatalf("PlanRun: %v", err) - } - safe, reason := planSupportsParallel(plans[0]) - if !safe { - t.Fatalf("expected plan to be parallel-safe, got false: %s", reason) - } -} - func TestCloseFixture_EmitsTeardownEvent(t *testing.T) { rep := &recordingReporter{} fix := &fakeSuiteFixture{} @@ -625,7 +312,7 @@ expected: t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", appHost, appPort, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixture.Runtime{}, "", appHost, appPort, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -701,7 +388,7 @@ expected: t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixture.Runtime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -780,7 +467,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixture.Runtime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -868,7 +555,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixture.Runtime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -947,7 +634,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixture.Runtime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -1025,7 +712,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected one plan with one case, got %+v", plans) } - ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixture.Runtime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -1123,7 +810,7 @@ endpoint = "http://localhost:4318" t.Fatalf("expected k8s-only assertion to be filtered out, got %d traces", got) } - ep, err := resolveEndpoint(plans[0], fixtureRuntime{}, "", "localhost", 8080, "http://localhost:4318") + ep, err := resolveEndpoint(plans[0], fixture.Runtime{}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -1187,43 +874,15 @@ func splitHostPort(t *testing.T, addr string) (string, int) { } type fakeSuiteFixture struct { - upCalls int closeCalls int - upErr error closeErr error } -func (f *fakeSuiteFixture) Up() error { - f.upCalls++ - return f.upErr -} - func (f *fakeSuiteFixture) Close() error { f.closeCalls++ return f.closeErr } -func equalStrings(got, want []string) bool { - if len(got) != len(want) { - return false - } - for i := range got { - if got[i] != want[i] { - return false - } - } - return true -} - -func containsString(items []string, want string) bool { - for _, item := range items { - if item == want { - return true - } - } - return false -} - type recordingReporter struct { events []report.Event } diff --git a/internal/cli/main.go b/internal/cli/main.go index c3b36505..0fdb304e 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -27,13 +27,10 @@ import ( "context" "encoding/json" "fmt" - "net" - "net/http" "os" "os/exec" "os/signal" "path/filepath" - "strconv" "strings" "sync" "syscall" @@ -45,53 +42,15 @@ import ( "github.com/grafana/oats/cache" "github.com/grafana/oats/discovery" "github.com/grafana/oats/engine" + "github.com/grafana/oats/fixture" "github.com/grafana/oats/migrate" "github.com/grafana/oats/report" "github.com/grafana/oats/runner" - "github.com/grafana/oats/testhelpers/compose" - "github.com/grafana/oats/testhelpers/kubernetes" - "github.com/grafana/oats/testhelpers/remote" ) -var ( - // Version is the oats CLI version. Release builds can override this with - // -ldflags "-X github.com/grafana/oats/internal/cli.Version=vX.Y.Z". - Version = "dev" - - newComposeSuite = func(files []string, env []string) (suiteFixture, error) { - return compose.SuiteFiles(files, env) - } - newKubernetesEndpoint = func(plan discovery.Plan, ports remote.PortsConfig) *remote.Endpoint { - sourceDir := plan.FixtureSourceDir - if sourceDir == "" { - sourceDir = "." - } - model := &kubernetes.Kubernetes{ - Dir: plan.Fixture.K8sDir, - AppService: plan.Fixture.AppService, - AppDockerFile: plan.Fixture.AppDockerFile, - AppDockerContext: plan.Fixture.AppDockerContext, - AppDockerTag: plan.Fixture.AppDockerTag, - AppDockerPort: plan.Fixture.AppPort, - ImportImages: plan.Fixture.ImportImages, - } - return kubernetes.NewEndpoint("localhost", model, ports, plan.Suite.Name, sourceDir) - } - waitForGrafanaToken = waitForGrafanaTokenImpl - lookupComposePort = dockerComposePort -) - -type fixtureRuntime struct { - GrafanaURL string - OTLPHTTP string - PyroscopeURL string - CustomCheckEnv []string - ComposeFiles []string - ComposeProject string - GCXConfig string - ParallelSafe bool - ParallelDisabled string -} +// Version is the oats CLI version. Release builds can override this with +// -ldflags "-X github.com/grafana/oats/internal/cli.Version=vX.Y.Z". +var Version = "dev" type suiteResult struct { pass int @@ -386,7 +345,7 @@ func verbosityFromInt(n int) report.Verbosity { // resolveEndpoint maps a fixture config + an explicit override into the // concrete endpoint the runner needs. -func resolveEndpoint(plan discovery.Plan, rt fixtureRuntime, gcxContextOverride, appHost string, appPort int, otlpHTTP string) (runner.Endpoint, error) { +func resolveEndpoint(plan discovery.Plan, rt fixture.Runtime, gcxContextOverride, appHost string, appPort int, otlpHTTP string) (runner.Endpoint, error) { ep := runner.Endpoint{AppHost: appHost, AppPort: appPort, OTLPHTTP: otlpHTTP} switch plan.Fixture.Type { case "remote": @@ -463,7 +422,7 @@ func runPlans(ctx context.Context, rep report.Reporter, plans []discovery.Plan, var serialPlans, parallelPlans []discovery.Plan for _, plan := range plans { - if safe, _ := planSupportsParallel(plan); safe { + if safe, _ := fixture.SupportsParallel(plan); safe { parallelPlans = append(parallelPlans, plan) } else { serialPlans = append(serialPlans, plan) @@ -557,11 +516,11 @@ func runPlansParallel(parent context.Context, rep report.Reporter, plans []disco func runPlan(ctx context.Context, rep report.Reporter, plan discovery.Plan, opts runOptions) suiteResult { fixtureStart := emitFixtureStart(rep, plan) - fix, rt, err := startFixture(ctx, plan) + fix, rt, err := fixture.Start(ctx, plan) if err != nil { return suiteResult{err: fmt.Errorf("suite %q: %w", plan.Suite.Name, err)} } - if err := waitForFixtureReady(plan, rt); err != nil { + if err := fixture.WaitForReady(plan, rt); err != nil { if fix != nil { _ = closeFixture(rep, plan, fix) } @@ -635,146 +594,6 @@ func runPlan(ctx context.Context, rep report.Reporter, plan discovery.Plan, opts return suiteResult{pass: suitePass, fail: suiteFail} } -type suiteFixture interface { - Close() error -} - -type startableSuiteFixture interface { - suiteFixture - Up() error -} - -func startFixture(ctx context.Context, plan discovery.Plan) (suiteFixture, fixtureRuntime, error) { - switch plan.Fixture.Type { - case "", "remote": - return nil, fixtureRuntime{ParallelSafe: true}, nil - case "compose": - composeFiles, cleanup, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) - if err != nil { - return nil, fixtureRuntime{}, err - } - project := composeProjectName(plan) - suiteEnv := append([]string(nil), plan.Fixture.Env...) - suiteEnv = append(suiteEnv, "COMPOSE_PROJECT_NAME="+project) - suite, err := newComposeSuite(composeFiles, suiteEnv) - if err != nil { - if cleanup != nil { - _ = cleanup() - } - return nil, fixtureRuntime{}, err - } - if err := startSuiteFixture(suite); err != nil { - if cleanup != nil { - _ = cleanup() - } - return nil, fixtureRuntime{}, err - } - grafanaPort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "3000") - if err != nil { - _ = suite.Close() - if cleanup != nil { - _ = cleanup() - } - return nil, fixtureRuntime{}, err - } - otlpPort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "4318") - if err != nil { - _ = suite.Close() - if cleanup != nil { - _ = cleanup() - } - return nil, fixtureRuntime{}, err - } - pyroscopePort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "4040") - if err != nil { - _ = suite.Close() - if cleanup != nil { - _ = cleanup() - } - return nil, fixtureRuntime{}, err - } - rt := fixtureRuntime{ - GrafanaURL: "http://127.0.0.1:" + grafanaPort, - OTLPHTTP: "http://127.0.0.1:" + otlpPort, - PyroscopeURL: "http://127.0.0.1:" + pyroscopePort, - ComposeFiles: composeFiles, - ComposeProject: project, - } - cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL) - if cfgErr != nil { - if cleanup != nil { - _ = cleanup() - } - return nil, fixtureRuntime{}, fmt.Errorf("write local gcx config: %w", cfgErr) - } - rt.GCXConfig = cfg - cleanup = chainCleanup(func() error { return removeIfExists(cfg) }, cleanup) - rt.CustomCheckEnv = composeCheckEnv(plan, rt) - rt.ParallelSafe, rt.ParallelDisabled = planSupportsParallel(plan) - return composeFixture{suite: suite, cleanup: cleanup}, rt, nil - case "k3d": - ports, err := allocateK3DPorts() - if err != nil { - return nil, fixtureRuntime{}, err - } - ep := newKubernetesEndpoint(plan, ports) - if err := ep.Start(ctx); err != nil { - return nil, fixtureRuntime{}, err - } - rt := fixtureRuntime{ - GrafanaURL: fmt.Sprintf("http://localhost:%d", ports.GrafanaHTTPPort), - OTLPHTTP: fmt.Sprintf("http://localhost:%d", ports.OTLPHTTPPort), - PyroscopeURL: fmt.Sprintf("http://localhost:%d", ports.PyroscopeHttpPort), - CustomCheckEnv: k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: plan.Fixture.AppPort}, ports), - ParallelSafe: false, - ParallelDisabled: "k3d fixtures currently use shared clusters and kubectl port-forwards", - } - cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL) - if cfgErr != nil { - _ = ep.Stop(context.Background()) - return nil, fixtureRuntime{}, fmt.Errorf("write local gcx config: %w", cfgErr) - } - rt.GCXConfig = cfg - return endpointFixture{ep: ep, cleanup: func() error { return removeIfExists(cfg) }}, rt, nil - default: - return nil, fixtureRuntime{}, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) - } -} - -func resolveComposeFiles(sourceDir string, fixture discovery.FixtureConfig) ([]string, func() error, error) { - var files []string - var cleanup func() error - if fixture.Template == "lgtm" { - f, err := writeBuiltinLGTMCompose(sourceDir) - if err != nil { - return nil, nil, err - } - files = append(files, f) - cleanup = func() error { return os.Remove(f) } - } else if fixture.Template != "" { - return nil, nil, fmt.Errorf("unsupported compose fixture template %q", fixture.Template) - } - switch { - case fixture.ComposeFile != "": - files = append(files, filepath.Join(sourceDir, fixture.ComposeFile)) - case len(fixture.ComposeFiles) > 0: - for _, file := range fixture.ComposeFiles { - files = append(files, filepath.Join(sourceDir, file)) - } - case fixture.Template == "": - return nil, nil, fmt.Errorf("compose fixture requires compose_file, compose_files, or supported template") - } - return files, cleanup, nil -} - -func startSuiteFixture(fix suiteFixture) error { - startable, ok := fix.(startableSuiteFixture) - if !ok { - return fmt.Errorf("fixture does not support startup") - } - return startable.Up() -} - func emitFixtureStart(rep report.Reporter, plan discovery.Plan) time.Time { start := time.Now() if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { @@ -801,7 +620,7 @@ func emitFixtureReady(rep report.Reporter, plan discovery.Plan, start time.Time) } } -func closeFixture(rep report.Reporter, plan discovery.Plan, fix suiteFixture) error { +func closeFixture(rep report.Reporter, plan discovery.Plan, fix fixture.Handle) error { start := time.Now() if err := fix.Close(); err != nil { return err @@ -818,462 +637,8 @@ func closeFixture(rep report.Reporter, plan discovery.Plan, fix suiteFixture) er return nil } -type endpointFixture struct { - ep *remote.Endpoint - cleanup func() error -} - -func (e endpointFixture) Close() error { - err := e.ep.Stop(context.Background()) - if e.cleanup != nil { - if cleanupErr := e.cleanup(); cleanupErr != nil && err == nil { - err = cleanupErr - } - } - return err -} - -type composeFixture struct { - suite suiteFixture - cleanup func() error -} - -func (c composeFixture) Close() error { - err := c.suite.Close() - if c.cleanup != nil { - if cleanupErr := c.cleanup(); cleanupErr != nil && err == nil { - err = cleanupErr - } - } - return err -} - -func writeBuiltinLGTMCompose(sourceDir string) (string, error) { - if err := os.MkdirAll(sourceDir, 0o755); err != nil { - return "", err - } - f, err := os.CreateTemp(sourceDir, ".oats.lgtm.*.compose.yml") - if err != nil { - return "", err - } - path := f.Name() - const body = `services: - lgtm: - image: ${LGTM_IMAGE:-docker.io/grafana/otel-lgtm:latest} - ports: - - "127.0.0.1::3000" - - "127.0.0.1::4317" - - "127.0.0.1::4318" - - "127.0.0.1::3200" - - "127.0.0.1::4040" - - "127.0.0.1::9090" -` - if _, err := f.WriteString(body); err != nil { - _ = f.Close() - _ = os.Remove(path) - return "", err - } - if err := f.Close(); err != nil { - _ = os.Remove(path) - return "", err - } - return path, nil -} - func grafanaURL() string { return "http://localhost:3000" } -// removeIfExists deletes path, ignoring a not-exist error so cleanup is -// idempotent. -func removeIfExists(path string) error { - if err := os.Remove(path); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -// chainCleanup returns a cleanup that runs first then second, returning the -// first non-nil error. Either argument may be nil. -func chainCleanup(first, second func() error) func() error { - return func() error { - var err error - if first != nil { - err = first() - } - if second != nil { - if e := second(); e != nil && err == nil { - err = e - } - } - return err - } -} - -func writeLocalGCXConfig(grafanaURL string) (string, error) { - cfg := fmt.Sprintf(`current-context: local -contexts: - local: - grafana: - server: %s - user: admin - password: admin - org-id: 1 - auth-method: basic - datasources: - prometheus: prometheus - loki: loki - tempo: tempo - pyroscope: pyroscope -`, grafanaURL) - f, err := os.CreateTemp("", "oats-gcx-*.yaml") - if err != nil { - return "", err - } - path := f.Name() - if _, err := f.WriteString(cfg); err != nil { - _ = f.Close() - _ = os.Remove(path) - return "", err - } - if err := f.Close(); err != nil { - _ = os.Remove(path) - return "", err - } - if err := os.Chmod(path, 0o600); err != nil { - _ = os.Remove(path) - return "", err - } - return path, nil -} - -func composeCheckEnv(plan discovery.Plan, rt fixtureRuntime) []string { - files := rt.ComposeFiles - if len(files) == 0 { - return []string{"OATS_FIXTURE_TYPE=compose"} - } - return []string{ - "OATS_FIXTURE_TYPE=compose", - "COMPOSE_PROJECT_NAME=" + rt.ComposeProject, - "COMPOSE_FILE=" + strings.Join(files, string(os.PathListSeparator)), - "OATS_COMPOSE_FILE_ARGS=" + composeFileArgs(files), - "OATS_GRAFANA_URL=" + rt.GrafanaURL, - "OATS_OTLP_HTTP=" + rt.OTLPHTTP, - "OATS_PYROSCOPE_URL=" + rt.PyroscopeURL, - } -} - -func composeFileArgs(files []string) string { - var parts []string - for _, f := range files { - parts = append(parts, "-f", shellQuote(f)) - } - return strings.Join(parts, " ") -} - -func shellQuote(s string) string { - return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" -} - -func k3dCheckEnv(ep runner.Endpoint, ports remote.PortsConfig) []string { - return []string{ - "OATS_FIXTURE_TYPE=k3d", - "OATS_APP_URL=" + fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), - "OATS_GRAFANA_URL=" + fmt.Sprintf("http://localhost:%d", ports.GrafanaHTTPPort), - "OATS_OTLP_HTTP=" + fmt.Sprintf("http://localhost:%d", ports.OTLPHTTPPort), - "OATS_PYROSCOPE_URL=" + fmt.Sprintf("http://localhost:%d", ports.PyroscopeHttpPort), - } -} - -func waitForGrafanaTokenImpl(plan discovery.Plan) (string, error) { - deadline := time.Now().Add(3 * time.Minute) - for time.Now().Before(deadline) { - var token string - var err error - switch plan.Fixture.Type { - case "compose": - token, err = readComposeGrafanaToken(plan) - case "k3d": - token, err = readK3DGrafanaToken() - default: - return "", nil - } - if err == nil && strings.TrimSpace(token) != "" { - return strings.TrimSpace(token), nil - } - time.Sleep(time.Second) - } - return "", fmt.Errorf("timed out waiting for Grafana service-account token") -} - -func waitForFixtureReady(plan discovery.Plan, rt fixtureRuntime) error { - switch plan.Fixture.Type { - case "compose", "k3d": - if err := waitForHTTP(rt.GrafanaURL+"/api/health", 2*time.Minute); err != nil { - return fmt.Errorf("wait for grafana: %w", err) - } - if err := waitForHTTP(rt.OTLPHTTP, 2*time.Minute); err != nil { - return fmt.Errorf("wait for otlp-http: %w", err) - } - } - return nil -} - -func planSupportsParallel(plan discovery.Plan) (bool, string) { - switch plan.Fixture.Type { - case "", "remote": - return true, "" - case "compose": - if plan.Fixture.Template != "lgtm" { - return false, "compose fixtures are only parallel-safe when OATS owns the LGTM ports via template=lgtm" - } - for _, c := range plan.Cases { - if c.Seed.Type == "app" { - return false, "compose suites with app seeds still rely on shared fixed app ports" - } - } - for _, file := range extraComposeFiles(plan) { - if fixed, err := composeFilePublishesFixedHostPorts(file); err != nil { - return false, fmt.Sprintf("compose port inspection failed for %s: %v", file, err) - } else if fixed { - return false, fmt.Sprintf("compose file %s publishes fixed host ports", filepath.Base(file)) - } - } - return true, "" - case "k3d": - return false, "k3d fixtures currently use shared localhost port-forwards" - default: - return false, "fixture type is not parallel-safe" - } -} - -func allocateK3DPorts() (remote.PortsConfig, error) { - grafanaPort, err := findFreePort() - if err != nil { - return remote.PortsConfig{}, err - } - otlpHTTPPort, err := findFreePort() - if err != nil { - return remote.PortsConfig{}, err - } - lokiPort, err := findFreePort() - if err != nil { - return remote.PortsConfig{}, err - } - promPort, err := findFreePort() - if err != nil { - return remote.PortsConfig{}, err - } - tempoPort, err := findFreePort() - if err != nil { - return remote.PortsConfig{}, err - } - pyroscopePort, err := findFreePort() - if err != nil { - return remote.PortsConfig{}, err - } - return remote.PortsConfig{ - GrafanaHTTPPort: grafanaPort, - OTLPHTTPPort: otlpHTTPPort, - LokiHttpPort: lokiPort, - PrometheusHTTPPort: promPort, - TempoHTTPPort: tempoPort, - PyroscopeHttpPort: pyroscopePort, - }, nil -} - -func findFreePort() (int, error) { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - return 0, err - } - defer func() { _ = ln.Close() }() - addr, ok := ln.Addr().(*net.TCPAddr) - if !ok { - return 0, fmt.Errorf("unexpected listener address %T", ln.Addr()) - } - return addr.Port, nil -} - -func composeProjectName(plan discovery.Plan) string { - name := strings.ToLower(plan.Suite.Name) - if name == "" { - name = "oats" - } - var b strings.Builder - for _, r := range name { - switch { - case r >= 'a' && r <= 'z', r >= '0' && r <= '9': - b.WriteRune(r) - default: - b.WriteByte('-') - } - } - slug := strings.Trim(b.String(), "-") - if slug == "" { - slug = "oats" - } - if len(slug) > 32 { - slug = slug[:32] - } - return fmt.Sprintf("oats-%s-%d", slug, time.Now().UnixNano()) -} - -func extraComposeFiles(plan discovery.Plan) []string { - switch { - case plan.Fixture.ComposeFile != "": - return []string{filepath.Join(plan.FixtureSourceDir, plan.Fixture.ComposeFile)} - case len(plan.Fixture.ComposeFiles) > 0: - files := make([]string, 0, len(plan.Fixture.ComposeFiles)) - for _, file := range plan.Fixture.ComposeFiles { - files = append(files, filepath.Join(plan.FixtureSourceDir, file)) - } - return files - default: - return nil - } -} - -func composeFilePublishesFixedHostPorts(path string) (bool, error) { - data, err := os.ReadFile(path) - if err != nil { - return false, err - } - lines := strings.Split(string(data), "\n") - inPorts := false - portsIndent := 0 - for _, line := range lines { - trimmed := strings.TrimSpace(line) - if trimmed == "" || strings.HasPrefix(trimmed, "#") { - continue - } - indent := len(line) - len(strings.TrimLeft(line, " ")) - if inPorts && indent <= portsIndent { - inPorts = false - } - if strings.HasPrefix(trimmed, "ports:") { - inPorts = true - portsIndent = indent - continue - } - if !inPorts { - continue - } - if strings.Contains(trimmed, "published:") { - value := strings.TrimSpace(strings.TrimPrefix(trimmed, "published:")) - if value != "" && value != "0" { - return true, nil - } - continue - } - if !strings.HasPrefix(trimmed, "-") { - continue - } - value := strings.Trim(strings.TrimSpace(strings.TrimPrefix(trimmed, "-")), `"'`) - if value == "" { - continue - } - if fixedShortPortMapping(value) { - return true, nil - } - } - return false, nil -} - -func fixedShortPortMapping(value string) bool { - if !strings.Contains(value, ":") { - return false - } - parts := strings.Split(value, ":") - if len(parts) < 2 { - return false - } - hostPart := strings.Trim(parts[len(parts)-2], "[]") - if _, err := strconv.Atoi(hostPart); err == nil && hostPart != "0" { - return true - } - return false -} - -func waitForHTTP(url string, timeout time.Duration) error { - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - resp, err := http.Get(url) //nolint:gosec - if err == nil { - _ = resp.Body.Close() - if resp.StatusCode >= 200 && resp.StatusCode < 500 { - return nil - } - } - time.Sleep(2 * time.Second) - } - return fmt.Errorf("timed out waiting for %s", url) -} - -func readComposeGrafanaToken(plan discovery.Plan) (string, error) { - files, _, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) - if err != nil { - return "", err - } - args := []string{"compose"} - for _, f := range files { - args = append(args, "-f", f) - } - args = append(args, "exec", "-T", "lgtm", "sh", "-c", "cat /tmp/grafana-sa-token 2>/dev/null || true") - cmd := exec.Command("docker", args...) - cmd.Env = append(cmd.Environ(), plan.Fixture.Env...) - out, err := cmd.Output() - if err != nil { - return "", err - } - return string(out), nil -} - -func dockerComposePort(files []string, env []string, service, containerPort string) (string, error) { - args := []string{"compose"} - for _, f := range files { - args = append(args, "-f", f) - } - args = append(args, "port", service, containerPort) - cmd := exec.Command("docker", args...) - cmd.Env = append(cmd.Environ(), env...) - out, err := cmd.Output() - if err != nil { - return "", err - } - host, port, err := splitDockerHostPort(strings.TrimSpace(string(out))) - if err != nil { - return "", err - } - if host == "" || port == "" { - return "", fmt.Errorf("invalid docker compose port output %q", strings.TrimSpace(string(out))) - } - return port, nil -} - -func splitDockerHostPort(addr string) (string, string, error) { - addr = strings.TrimSpace(addr) - if strings.HasPrefix(addr, "[") { - end := strings.Index(addr, "]") - if end < 0 || end+2 > len(addr) || addr[end+1] != ':' { - return "", "", fmt.Errorf("invalid address %q", addr) - } - return addr[1:end], addr[end+2:], nil - } - idx := strings.LastIndex(addr, ":") - if idx < 0 { - return "", "", fmt.Errorf("invalid address %q", addr) - } - return addr[:idx], addr[idx+1:], nil -} - -func readK3DGrafanaToken() (string, error) { - cmd := exec.Command("kubectl", "exec", "deploy/lgtm", "--", "sh", "-c", "cat /tmp/grafana-sa-token 2>/dev/null || true") - out, err := cmd.Output() - if err != nil { - return "", err - } - return string(out), nil -} - func splitCSV(s string) []string { if strings.TrimSpace(s) == "" { return nil From 427279d2d24cd9c8d67669cac8749172141feebe Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 18:19:57 +0200 Subject: [PATCH 103/124] fix: address Copilot review comments - compose: start the docker command before spawning the stdout-reader goroutine (and close the pipe on Start error) so a failed Start can't leak a goroutine blocked on ReadString. - seed: make inline-otlp seeding context-aware (http.NewRequestWithContext, ctx threaded from Runner.RunCase) so a cancelled run stops seeding instead of delaying teardown. - yaml: propagate the filepath.Abs error in readTestCaseDefinition instead of panicking on the user-facing migrate path; document LoadTestCaseDefinition as a migrate-only shim, not supported public API. - go.mod: bump the go directive to 1.26.4 to match the toolchain mise pins. - wait: correct the Result.LastFailures docstring (it is returned on cancel/timeout when a prior asserter call reported failures). - casefile: rename TestValidate_RejectsPresentFalse to TestValidate_RejectsDuplicateAttributeKeys to match what it asserts. Signed-off-by: Gregor Zeitlinger --- casefile/case_test.go | 4 ++-- go.mod | 2 +- runner/runner.go | 6 +++--- seed/seed.go | 32 ++++++++++++++++++++------------ seed/seed_test.go | 9 +++++---- testhelpers/compose/compose.go | 11 +++++++---- wait/wait.go | 6 ++++-- yaml/testcase.go | 9 ++++++++- 8 files changed, 50 insertions(+), 29 deletions(-) diff --git a/casefile/case_test.go b/casefile/case_test.go index b9e3cfb9..c4dcc6ab 100644 --- a/casefile/case_test.go +++ b/casefile/case_test.go @@ -326,10 +326,10 @@ expected: } } -func TestValidate_RejectsPresentFalse(t *testing.T) { +func TestValidate_RejectsDuplicateAttributeKeys(t *testing.T) { _, err := Parse([]byte(` oats-schema-version: 3 -name: bad present +name: duplicate attribute keys seed: type: app compose: x.yml diff --git a/go.mod b/go.mod index 3cf78c74..509b98ed 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/grafana/oats -go 1.25.0 +go 1.26.4 require ( github.com/BurntSushi/toml v1.6.0 diff --git a/runner/runner.go b/runner/runner.go index 2aa41468..c0b3e7de 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -201,7 +201,7 @@ func (r *Runner) RunCase(ctx context.Context, c *casefile.Case) bool { } // Seed. - if err := r.seedCase(c); err != nil { + if err := r.seedCase(ctx, c); err != nil { r.failCase(c, "seed: "+err.Error(), "") r.reporter.Emit(report.Event{ Type: report.EventCaseFail, @@ -437,7 +437,7 @@ func trimOutput(s string) string { return s } -func (r *Runner) seedCase(c *casefile.Case) error { +func (r *Runner) seedCase(ctx context.Context, c *casefile.Case) error { switch c.Seed.Type { case "app": // External fixture is responsible for booting the app. Runner @@ -451,7 +451,7 @@ func (r *Runner) seedCase(c *casefile.Case) error { if err != nil { return err } - return r.seeder.Send(payload) + return r.seeder.Send(ctx, payload) } return fmt.Errorf("unknown seed type %q", c.Seed.Type) } diff --git a/seed/seed.go b/seed/seed.go index 67d64b6d..fe800567 100644 --- a/seed/seed.go +++ b/seed/seed.go @@ -15,6 +15,7 @@ package seed import ( "bytes" + "context" "crypto/rand" "encoding/hex" "fmt" @@ -79,30 +80,32 @@ func (s *Sender) httpClient() *http.Client { // Send pushes all signals declared in p. Returns the first error encountered, // but processes signals in a fixed order (traces, logs, metrics) so that a // partial send leaves the backend in a predictable state for assertions. -func (s *Sender) Send(p Payload) error { +// Cancelling ctx aborts in-flight and remaining requests so seeding does not +// outlive a cancelled run. +func (s *Sender) Send(ctx context.Context, p Payload) error { if s.OTLPEndpoint == "" { return fmt.Errorf("seed: OTLPEndpoint is empty") } now := time.Now() for _, t := range p.Traces { - if err := s.sendTrace(t, now); err != nil { + if err := s.sendTrace(ctx, t, now); err != nil { return fmt.Errorf("seed traces: %w", err) } } for _, l := range p.Logs { - if err := s.sendLog(l, now); err != nil { + if err := s.sendLog(ctx, l, now); err != nil { return fmt.Errorf("seed logs: %w", err) } } for _, m := range p.Metrics { - if err := s.sendMetric(m, now); err != nil { + if err := s.sendMetric(ctx, m, now); err != nil { return fmt.Errorf("seed metrics: %w", err) } } return nil } -func (s *Sender) sendTrace(t Trace, now time.Time) error { +func (s *Sender) sendTrace(ctx context.Context, t Trace, now time.Time) error { dur := t.Span.Duration if dur == 0 { dur = 200 * time.Millisecond @@ -132,10 +135,10 @@ func (s *Sender) sendTrace(t Trace, now time.Time) error { }] }] }`, t.Service, mustRandHex(16), mustRandHex(8), t.Span.Name, kind, start, end) - return s.post("/v1/traces", body) + return s.post(ctx, "/v1/traces", body) } -func (s *Sender) sendLog(l Log, now time.Time) error { +func (s *Sender) sendLog(ctx context.Context, l Log, now time.Time) error { sev := l.SeverityNumber if sev == 0 { sev = 9 @@ -160,10 +163,10 @@ func (s *Sender) sendLog(l Log, now time.Time) error { }] }] }`, l.Service, now.UnixNano(), sev, sevText, l.Body) - return s.post("/v1/logs", body) + return s.post(ctx, "/v1/logs", body) } -func (s *Sender) sendMetric(m Metric, now time.Time) error { +func (s *Sender) sendMetric(ctx context.Context, m Metric, now time.Time) error { end := now.UnixNano() start := now.Add(-time.Second).UnixNano() body := fmt.Sprintf(`{ @@ -188,11 +191,16 @@ func (s *Sender) sendMetric(m Metric, now time.Time) error { }] }] }`, m.Service, m.Name, start, end, m.Value) - return s.post("/v1/metrics", body) + return s.post(ctx, "/v1/metrics", body) } -func (s *Sender) post(path, body string) error { - resp, err := s.httpClient().Post(s.OTLPEndpoint+path, "application/json", bytes.NewBufferString(body)) +func (s *Sender) post(ctx context.Context, path, body string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.OTLPEndpoint+path, bytes.NewBufferString(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := s.httpClient().Do(req) if err != nil { return err } diff --git a/seed/seed_test.go b/seed/seed_test.go index 10b1ccfb..644592f3 100644 --- a/seed/seed_test.go +++ b/seed/seed_test.go @@ -1,6 +1,7 @@ package seed import ( + "context" "encoding/json" "io" "net/http" @@ -34,7 +35,7 @@ func TestSender_SendTracesLogsMetrics(t *testing.T) { defer srv.Close() s := &Sender{OTLPEndpoint: srv.URL} - err := s.Send(Payload{ + err := s.Send(context.Background(), Payload{ Traces: []Trace{{ Service: "svc-a", Span: SpanFields{Name: "do-thing"}, @@ -79,7 +80,7 @@ func TestSender_SendTracesLogsMetrics(t *testing.T) { func TestSender_EmptyEndpointFailsLoudly(t *testing.T) { s := &Sender{} - if err := s.Send(Payload{}); err == nil { + if err := s.Send(context.Background(), Payload{}); err == nil { t.Fatal("expected an error for empty endpoint") } } @@ -92,7 +93,7 @@ func TestSender_BackendErrorPropagates(t *testing.T) { defer srv.Close() s := &Sender{OTLPEndpoint: srv.URL} - err := s.Send(Payload{Traces: []Trace{{Service: "x", Span: SpanFields{Name: "y"}}}}) + err := s.Send(context.Background(), Payload{Traces: []Trace{{Service: "x", Span: SpanFields{Name: "y"}}}}) if err == nil { t.Fatal("expected an error from 400 response") } @@ -107,7 +108,7 @@ func TestSender_SpanDefaultsApply(t *testing.T) { before := time.Now() s := &Sender{OTLPEndpoint: srv.URL} - err := s.Send(Payload{Traces: []Trace{{ + err := s.Send(context.Background(), Payload{Traces: []Trace{{ Service: "svc", Span: SpanFields{Name: "n"}, }}}) diff --git a/testhelpers/compose/compose.go b/testhelpers/compose/compose.go index f0f08d46..9d9cd738 100644 --- a/testhelpers/compose/compose.go +++ b/testhelpers/compose/compose.go @@ -115,6 +115,13 @@ func (c *Compose) runDocker(cc command) error { return fmt.Errorf("failed to open docker stdout pipe: %w", err) } cmd.Stderr = cmd.Stdout + // Start the command before spawning the reader: if Start fails the + // write end of the pipe never opens, and a reader started earlier would + // block forever on ReadString and leak. + if err := cmd.Start(); err != nil { + _ = stdout.Close() + return fmt.Errorf("failed to start docker command: %w", err) + } wg := sync.WaitGroup{} wg.Add(1) go func() { @@ -127,10 +134,6 @@ func (c *Compose) runDocker(cc command) error { } }() - err = cmd.Start() - if err != nil { - return fmt.Errorf("failed to start docker command: %w", err) - } err = cmd.Wait() wg.Wait() if err != nil { diff --git a/wait/wait.go b/wait/wait.go index 20838ce6..5583c2ca 100644 --- a/wait/wait.go +++ b/wait/wait.go @@ -38,8 +38,10 @@ type Options struct { // Result is what Until and While return. Iterations counts how many poll // attempts ran; LastFailures is the most recent failure set observed — nil on -// success, and also nil when the run is cancelled or stopped early before -// observing a failure; populated when an assertion actually failed. +// success and when no asserter call ever reported failures (including a run +// cancelled or stopped before any failure was observed). If any call did +// report failures, those are returned even when the run was ultimately +// cancelled or timed out. type Result[F any] struct { OK bool Iterations int diff --git a/yaml/testcase.go b/yaml/testcase.go index b41c5524..7bed715b 100644 --- a/yaml/testcase.go +++ b/yaml/testcase.go @@ -37,6 +37,10 @@ func ReadTestCases(input []string, evaluateIgnoreFile bool) ([]model.TestCase, e // LoadTestCaseDefinition reads one legacy OATS test case definition file, // resolving includes exactly like the v1 runner does. Template files return // (nil, nil), matching readTestCaseDefinition's semantics. +// +// It is exported solely so the `migrate` package can reuse legacy parsing; it +// is not part of the supported OATS Go API and may change or move without +// notice. func LoadTestCaseDefinition(path string) (*model.TestCaseDefinition, error) { return readTestCaseDefinition(path, false) } @@ -147,7 +151,10 @@ func readTestCase(testBase, filePath string) (*model.TestCase, error) { } func readTestCaseDefinition(filePath string, templateMode bool) (*model.TestCaseDefinition, error) { - filePath = absolutePath(filePath) + filePath, err := filepath.Abs(filePath) + if err != nil { + return nil, fmt.Errorf("failed to resolve path %s: %w", filePath, err) + } content, err := os.ReadFile(filePath) if err != nil { return nil, fmt.Errorf("failed to read file %s: %w", filePath, err) From 217de8fa6d02ab28d9e4047f507467dd57d1aeea Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 16:41:50 +0000 Subject: [PATCH 104/124] fix: bound waitForHTTP probes and close compose pipe on start failure - fixture: give waitForHTTP an http.Client with a 10s timeout so a target that accepts TCP but never responds can't block a single probe past the overall readiness deadline. - compose: close the StdoutPipe read end when cmd.Start() fails in the logConsumer path, matching the background path and avoiding an FD leak. Addresses Copilot review comments. Signed-off-by: Gregor Zeitlinger --- fixture/fixture.go | 5 ++++- testhelpers/compose/compose.go | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/fixture/fixture.go b/fixture/fixture.go index 7da49cdb..a86d219f 100644 --- a/fixture/fixture.go +++ b/fixture/fixture.go @@ -268,9 +268,12 @@ func findFreePort() (int, error) { } func waitForHTTP(url string, timeout time.Duration) error { + // Bound each probe so a target that accepts TCP but never responds can't + // block a single GET past the overall deadline. + client := &http.Client{Timeout: 10 * time.Second} deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { - resp, err := http.Get(url) //nolint:gosec + resp, err := client.Get(url) //nolint:gosec if err == nil { _ = resp.Body.Close() if resp.StatusCode >= 200 && resp.StatusCode < 500 { diff --git a/testhelpers/compose/compose.go b/testhelpers/compose/compose.go index 9d9cd738..fbd6c214 100644 --- a/testhelpers/compose/compose.go +++ b/testhelpers/compose/compose.go @@ -99,6 +99,7 @@ func (c *Compose) runDocker(cc command) error { // the pipe never opens, so a consumer started earlier would block on // the read forever and leak. if err := cmd.Start(); err != nil { + _ = stdout.Close() return fmt.Errorf("failed to start docker command: %w", err) } wg := sync.WaitGroup{} From 67e01cbcf39483f190135d3e318df83e1c1dc82f Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 17:00:12 +0000 Subject: [PATCH 105/124] test(e2e): assert profile flamegraph on runtime. prefix, not runtime.mcall The assert/profile case matched runtime.mcall, a single scheduler frame only sampled when goroutines happen to be scheduled mid-sample. Under CI load it was frequently absent from Pyroscope's self-profile, flaking the case (repeated core shard failures). Match the broader runtime. prefix instead: any non-empty Go CPU profile contains many runtime.* frames, so the assertion still proves a real flamegraph came back through oats -> gcx -> Pyroscope without the single-frame fragility. Timeout unchanged. Signed-off-by: Gregor Zeitlinger --- tests/e2e/cases/assert/profile/files/oats.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/e2e/cases/assert/profile/files/oats.yaml b/tests/e2e/cases/assert/profile/files/oats.yaml index 758371f5..6d6eb421 100644 --- a/tests/e2e/cases/assert/profile/files/oats.yaml +++ b/tests/e2e/cases/assert/profile/files/oats.yaml @@ -10,7 +10,10 @@ expected: # otel-lgtm's Pyroscope self-profiles under service_name="pyroscope", so a # CPU flamegraph is always present without seeding (oats cannot seed # profiles). This exercises the oats -> gcx -> Pyroscope query + flamebearer - # assertion path. runtime.mcall is a Go scheduler frame present in every - # Go CPU profile. + # assertion path. We assert on the "runtime." prefix rather than a single + # frame: any non-empty Go CPU profile contains many runtime.* frames (GC, + # scheduler, netpoll, ...), whereas a specific frame like runtime.mcall is + # only sampled when goroutines happen to be scheduled mid-sample and flakes + # under CI load. - query: 'process_cpu:samples:count:cpu:nanoseconds{service_name="pyroscope"}' - contains: runtime.mcall + contains: runtime. From ef0c0fb9cfe632f21174cbdd9729a7b50402b968 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Tue, 7 Jul 2026 17:14:19 +0000 Subject: [PATCH 106/124] fix: preserve original path in readTestCaseDefinition error filepath.Abs returns an empty string on error, and the previous code overwrote filePath with it before formatting the error, yielding "failed to resolve path : ..." and losing the original input. Keep the original filePath for the message and only assign the resolved path on success. Addresses Copilot review comment. Signed-off-by: Gregor Zeitlinger --- yaml/testcase.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yaml/testcase.go b/yaml/testcase.go index 7bed715b..4eb15812 100644 --- a/yaml/testcase.go +++ b/yaml/testcase.go @@ -151,10 +151,11 @@ func readTestCase(testBase, filePath string) (*model.TestCase, error) { } func readTestCaseDefinition(filePath string, templateMode bool) (*model.TestCaseDefinition, error) { - filePath, err := filepath.Abs(filePath) + absPath, err := filepath.Abs(filePath) if err != nil { return nil, fmt.Errorf("failed to resolve path %s: %w", filePath, err) } + filePath = absPath content, err := os.ReadFile(filePath) if err != nil { return nil, fmt.Errorf("failed to read file %s: %w", filePath, err) From c8afa6420a318173ec66897ab578639b31f761e8 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 11:47:40 +0200 Subject: [PATCH 107/124] remove deprecated files Signed-off-by: Gregor Zeitlinger --- CURRENT.md | 1 - yaml/README.md | 3 --- 2 files changed, 4 deletions(-) delete mode 100644 CURRENT.md delete mode 100644 yaml/README.md diff --git a/CURRENT.md b/CURRENT.md deleted file mode 100644 index d4c35ac5..00000000 --- a/CURRENT.md +++ /dev/null @@ -1 +0,0 @@ -# Current OATS docs now live in [README.md](README.md) diff --git a/yaml/README.md b/yaml/README.md deleted file mode 100644 index 07b47070..00000000 --- a/yaml/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Declarative YAML tests - -See [OpenTelemetry Acceptance Tests (OATs)](../README.md) for more information. From 008b321853fa8e275ecdf55e84ca681f79ed8f74 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 11:50:30 +0200 Subject: [PATCH 108/124] remove deprecated files Signed-off-by: Gregor Zeitlinger --- .mise/tasks/e2e-test.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100755 .mise/tasks/e2e-test.sh diff --git a/.mise/tasks/e2e-test.sh b/.mise/tasks/e2e-test.sh deleted file mode 100755 index a9b16b96..00000000 --- a/.mise/tasks/e2e-test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -#MISE description="Run Integration tests" - -set -euo pipefail - -mise run build -./oats -timeout 5m tests/e2e From d323aca3c5f4d544896883854bd8c316fe0fe7a7 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 10:56:35 +0000 Subject: [PATCH 109/124] docs: restructure README, add case-reference and CI guides Slim README to an overview + install + quickstart + CLI (flags as a table), and move the exhaustive reference to docs/case-reference.md (config, fixtures, seed modes incl. app-vs-inline-otlp rationale, assertions, custom checks, parallelism + its memory cost). Add docs/ci.md covering mise/aqua + gh-release install and a result-caching pattern (pin versions, path-gate, optional actions/cache) with the floating-tag false-green caveat. Install uses `mise use --pin`; repoint moved UPGRADING anchors to docs/case-reference.md. Signed-off-by: Gregor Zeitlinger --- README.md | 321 ++++++++++------------------------------- UPGRADING.md | 6 +- docs/case-reference.md | 240 ++++++++++++++++++++++++++++++ docs/ci.md | 119 +++++++++++++++ 4 files changed, 442 insertions(+), 244 deletions(-) create mode 100644 docs/case-reference.md create mode 100644 docs/ci.md diff --git a/README.md b/README.md index e494430e..56a2a7ef 100644 --- a/README.md +++ b/README.md @@ -1,108 +1,16 @@ # OpenTelemetry Acceptance Tests (OATs) -OATs is a declarative acceptance-test framework for OpenTelemetry. +OATs is a declarative acceptance-test framework for OpenTelemetry. You describe, +in yaml, the telemetry an instrumented app *should* produce — traces, logs, +metrics, profiles — and OATS drives the app (or seeds telemetry directly), then +asserts against a real observability stack (Grafana, Loki, Tempo, Prometheus, +Pyroscope) via [`gcx`](https://github.com/grafana/gcx). -The `oats` binary is the gcx-driven CLI. Legacy direct-yaml invocation has been -removed; upgrades now use the current `oats.toml` + case-yaml flow. - -## Install - -```sh -go install github.com/grafana/oats@latest -``` - -## Quick start - -```sh -# Build local dev binaries (oats + gcx) into ./bin -./scripts/build-local-tools.sh - -# Print the CLI version -bin/oats version - -# Print what would run -bin/oats list --config examples/smoke/oats.toml - -# Run -bin/oats --config examples/smoke/oats.toml -``` - -## Layout - -- `examples/smoke/` — small runnable examples -- `examples/fixtures/` — richer compose / k3d fixture examples -- `UPGRADING.md` — migration notes for older repos - -## Current scope - -- traces / logs / metrics / profiles via `gcx` -- structural collector-style `match_spans` / `match` -- app-backed and inline-OTLP seed modes -- remote / compose / k3d fixtures -- custom checks -- best-effort migration from legacy OATS yaml via: - - ```sh - oats migrate path/to/legacy.yaml - ``` - -## CLI - -Commands: - -```sh -oats [flags] # run the suites (implicit; same as `oats run`) -oats run [flags] # run the suites -oats list --config oats.toml # print the run plan and exit -oats migrate legacy.yaml # convert one legacy yaml to the v3 shape -oats cache clear # delete all cached results -oats version # print the version -``` - -Run flags (on `oats` / `oats run`): - -```sh -oats --config oats.toml --suite smoke -oats --config oats.toml --tags traces,logs -oats --config oats.toml --gcx-context my-lgtm -oats --config oats.toml --no-cache -oats --config oats.toml --fail-fast -oats --config oats.toml --format ndjson -oats -v # -v / -vv / -vvv increase verbosity -``` - -Key flags: - -- `--config` -- `--suite` -- `--tags` -- `--timeout` -- `--interval` -- `--absent-timeout` -- `--parallel` -- `--fail-fast` — stop scheduling further cases after the first case failure -- `--gcx` -- `--gcx-context` -- `-v` / `-vv` / `-vvv` — verbosity - -## Config shape - -```toml -[meta] -version = 2 - -[[suite]] -cases = ["examples/smoke/cases/*.yaml"] - -[cache] -ttl_days = 7 -``` - -Case yaml: +A case reads like the outcome you care about: ```yaml oats-schema-version: 3 -name: rolldice traces have route attribute +name: rolldice traces have a route attribute fixture: type: compose @@ -119,165 +27,96 @@ expected: - traceql: '{ span.http.route = "/rolldice" }' match_spans: - name: "GET /rolldice" - metrics: - - promql: 'dice_lib_rolls_counter_total{service_name="dice-server"}' - value: '>= 0' - logs: - - logql: '{service_name="dice-server"}' - contains: Received request - custom-checks: - - script: ./verify.sh ``` -## Seed - -A case populates the stack before assertions run via one of two `seed.type` -modes: - -```yaml -# App-backed: the suite fixture boots an instrumented app, and `input` -# requests drive it so it emits telemetry. -seed: - type: app -input: - - path: /rolldice?rolls=5 # method defaults to GET -``` - -```yaml -# Inline-OTLP: the case carries its own payload, pushed as OTLP/HTTP JSON. -# No app, no SDK. Declare only the signals you need. -seed: - type: inline-otlp - traces: - - service: my-service - spans: - - name: seed-operation # kind defaults to INTERNAL, duration to 200ms - logs: - - service: my-service - body: seed-log-line # severity_number defaults to 9 (INFO) - metrics: - - service: my-service - name: seed_counter # monotonic sum → PromQL `seed_counter_total` - value: 42 -``` - -> Inline-OTLP can seed traces, logs, and metrics. **Profiles cannot be -> inline-seeded** — assert profiles against an app-backed fixture that produces -> them (e.g. an eBPF profiler or a pyroscope-instrumented app). - -## Assertions - -Every signal block under `expected` shares one assertion vocabulary, plus a -few signal-specific keys. Each entry first names a query, then the checks that -must hold against its result. - -Shared keys (valid on `traces`, `metrics`, `logs`, `profiles`): +## Install -| Key | Meaning | -|-----|---------| -| `contains` | string (or list) that must appear in the query output | -| `not_contains` | string (or list) that must **not** appear | -| `regex` | RE2 pattern (or list) that must match the output | -| `match` | structural row match — list of `{match_type, name, attributes}` | -| `count` | comparison against the number of rows, e.g. `'== 1'`, `'>= 2'` | -| `absent` | the query must return nothing for the whole `--absent-timeout` window | +1. Install [mise](https://mise.jdx.dev/). -`match` (and the trace-only `match_spans`) entries: +2. Pin `oats` and `gcx` into your repo — mise resolves the latest releases and + locks them into `mise.toml`: -```yaml -match: - - match_type: strict # "strict" (default) or "regexp" - name: seed-log-line # for regexp, an RE2 pattern - attributes: # optional; list of {key, value?} - - key: service_name - value: my-service # omit `value` to assert the key is merely present -``` + ```sh + mise use --pin aqua:grafana/oats aqua:grafana/gcx + ``` -Signal-specific keys: +For CI, see [docs/ci.md](docs/ci.md). -- `traces`: `traceql` (required), `match_spans` (span-row match, same shape as `match`) -- `metrics`: `promql` (required), `value` (compare the sample value, e.g. `'>= 1'`, `'== 42'`) -- `logs`: `logql` (required) -- `profiles`: `query` (required) +Without mise: -Example covering several shapes: +- **GitHub releases** — download for your OS/arch from the + [oats](https://github.com/grafana/oats/releases) and + [gcx](https://github.com/grafana/gcx/releases) release pages. +- **go install** (oats from source) — `go install github.com/grafana/oats@latest`. -```yaml -expected: - traces: - - traceql: '{ resource.service.name = "my-service" }' - match_spans: - - match_type: regexp - name: '^GET /rolldice.*' - attributes: - - key: service.name - value: my-service - metrics: - - promql: 'seed_counter_total{service_name="my-service"}' - value: '>= 1' - logs: - - logql: '{service_name="my-service"}' - regex: '.*rolling the dice.*' - count: '>= 1' - profiles: - - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{service_name="my-service"}' - contains: main -``` +`oats` drives assertions through `gcx`; for fixture-backed runs OATS can +bootstrap gcx itself, but pinning it explicitly keeps runs reproducible. -### compose-logs +## Quick start -For `compose` fixtures, `expected.compose-logs` greps the container logs -(`docker compose logs`) for each string — useful for asserting on output that -never reaches the OTLP pipeline: +```sh +# Print the run plan without executing +oats list --config examples/smoke/oats.toml -```yaml -expected: - compose-logs: - - app started ok +# Run it +oats --config examples/smoke/oats.toml ``` -## Custom checks - -`expected.custom-checks` runs an arbitrary script and treats **exit code 0 as -pass**, non-zero as fail. Combined stdout+stderr is captured and printed on -failure. Like every assertion it is retried until it passes or `--timeout` -elapses. +To hack on OATS itself (builds local `oats` + `gcx` into `./bin`): -```yaml -expected: - custom-checks: - - script: ./verify.sh # resolved relative to the case file's directory +```sh +./scripts/build-local-tools.sh +bin/oats version ``` -`script` may be a path (relative to the case dir, or absolute) or an inline -script beginning with a `#!` shebang. The process runs with its working -directory set to the case dir and inherits the parent environment plus these -OATS-provided variables: - -| Variable | Fixtures | Meaning | -|----------|----------|---------| -| `OATS_FIXTURE_TYPE` | all | `remote` / `compose` / `k3d` | -| `OATS_GRAFANA_URL` | all | base URL of the fixture's Grafana | -| `OATS_APP_URL` | remote, k3d | base URL of the app under test | -| `OATS_OTLP_HTTP` | compose, k3d | OTLP/HTTP endpoint | -| `OATS_PYROSCOPE_URL` | compose, k3d | Pyroscope base URL | -| `COMPOSE_PROJECT_NAME`, `COMPOSE_FILE`, `OATS_COMPOSE_FILE_ARGS` | compose | let the script run its own `docker compose` commands | - -A custom check that queries Grafana directly (replacing, for example, a legacy -`compose-logs` grep with a real LogQL query): +## CLI -```bash -#!/usr/bin/env bash -set -euo pipefail -curl -fsS "${OATS_GRAFANA_URL:?}/api/health" >/dev/null -echo "custom check ok" +```sh +oats [flags] # run the suites (implicit; same as `oats run`) +oats run [flags] # run the suites +oats list --config oats.toml # print the run plan and exit +oats migrate legacy.yaml # convert one legacy yaml to the v3 shape +oats cache clear # delete all cached results +oats version # print the version ``` -## Notes - -- Cases inside one suite still run sequentially. -- Suites can run in parallel with `--parallel N` when fixture isolation allows - it. Today that mainly means remote suites and compose suites where OATS owns - the LGTM ports (`template = "lgtm"`). -- Case-local `fixture:` blocks cover the common one-case-per-suite shape. -- OATS owns local LGTM bootstrapping and gcx bootstrap for fixture-backed runs. +Common flags: + +| Flag | Default | Meaning | +|------|---------|---------| +| `--config` | `oats.toml` | path to the config file | +| `--suite` | all | comma-separated suite names to run | +| `--tags` | all | comma-separated tags; a case runs if it matches any | +| `--timeout` | `30s` | per-assertion timeout — each assertion is retried until it passes or this elapses | +| `--interval` | `500ms` | polling interval between assertion retries | +| `--absent-timeout` | `10s` | window an `absent` assertion must stay empty | +| `--parallel` | `1` | suites to run concurrently, when fixture isolation allows | +| `--fail-fast` | `false` | stop scheduling further cases after the first case failure | +| `--no-cache` | `false` | disable the skip-when-unchanged cache for this run | +| `--format` | `text` | output format: `text` or `ndjson` | +| `--gcx` | `gcx` | path to the gcx binary | +| `--gcx-context` | derived | override the gcx context (otherwise derived from the fixture endpoint) | +| `-v` / `-vv` / `-vvv` | — | increase verbosity | + +Run `oats --help` for the full list, including the inline-OTLP seed host/port +overrides. + +## What it covers + +- traces / logs / metrics / profiles, queried through `gcx` +- structural collector-style row matching (`match` / `match_spans`) +- app-backed and inline-OTLP seed modes +- remote / compose / k3d fixtures +- custom-check scripts +- best-effort migration from legacy OATS yaml (`oats migrate path/to/legacy.yaml`) + +## Documentation + +- **[docs/case-reference.md](docs/case-reference.md)** — the full case + config + shape: fixtures, seed modes, the assertion vocabulary, custom checks +- **[docs/ci.md](docs/ci.md)** — installing and running OATS in CI, plus result + caching and its caveats +- **[UPGRADING.md](UPGRADING.md)** — migrating older (schema-2) repos to v3 +- **[AGENTS.md](AGENTS.md)** — for contributors and coding agents working *on* + OATS (build, layout, conventions) +- `examples/smoke/`, `examples/fixtures/` — small runnable examples diff --git a/UPGRADING.md b/UPGRADING.md index 7b967a0e..85e983e2 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -14,7 +14,7 @@ The root `oats` binary now runs the gcx-driven current CLI only. That means upgrades now require migrating to the current: - `oats.toml` suite/discovery file -- current case yaml shape documented in [README.md](README.md) +- current case yaml shape documented in [docs/case-reference.md](docs/case-reference.md) The legacy “pass one or more old yaml files directly to `oats`” runner has been removed. For one-off help migrating old cases, use: @@ -92,7 +92,7 @@ vocabulary keyed off `query`: (`expected.compose-logs: [ ... ]`). `oats migrate` does **not** convert it (it emits a warning); either keep it as-is on a compose fixture, or replace it with a `custom-checks` script that queries the backend directly — see the -[custom checks](README.md#custom-checks) contract. +[custom checks](docs/case-reference.md#custom-checks) contract. **`matrix`.** Not migrated automatically. `oats migrate` flattens a single-entry matrix and otherwise emits a hint; multi-entry matrices must be split into @@ -162,7 +162,7 @@ expected: contains: "rolling the dice" ``` -See [README.md](README.md) for the full version-3 assertion reference. +See [docs/case-reference.md](docs/case-reference.md) for the full version-3 assertion reference. ## 0.6.0 diff --git a/docs/case-reference.md b/docs/case-reference.md new file mode 100644 index 00000000..ec3a1116 --- /dev/null +++ b/docs/case-reference.md @@ -0,0 +1,240 @@ +# Case reference + +The full shape of an OATS project: the `oats.toml` config, the case yaml, seed +modes, the assertion vocabulary, and custom checks. For a quick start and the +CLI summary see the [README](../README.md); for running in CI see +[ci.md](ci.md). + +## oats.toml + +```toml +[meta] +version = 2 # oats.toml schema version (distinct from a case's oats-schema-version) + +[[suite]] +cases = ["examples/smoke/cases/*.yaml"] + +[cache] +ttl_days = 7 # skip-when-unchanged TTL; 0 → default (7 days) +``` + +A suite may pin its own fixture, or let each case carry a case-local `fixture:` +block (the common one-case-per-suite shape). + +## Case yaml + +```yaml +oats-schema-version: 3 +name: rolldice traces have route attribute + +fixture: + type: compose + template: lgtm + compose_file: docker-compose.oats.yml + +seed: + type: app +input: + - path: /rolldice?rolls=5 + +expected: + traces: + - traceql: '{ span.http.route = "/rolldice" }' + match_spans: + - name: "GET /rolldice" + metrics: + - promql: 'dice_lib_rolls_counter_total{service_name="dice-server"}' + value: '>= 0' + logs: + - logql: '{service_name="dice-server"}' + contains: Received request + custom-checks: + - script: ./verify.sh +``` + +## Fixtures + +A fixture is the observability stack a case runs against — Grafana + Loki + +Tempo + Prometheus + Pyroscope, optionally plus the app under test. `fixture.type` +selects how it is stood up: + +| Type | Meaning | +|------|---------| +| `remote` | point at an already-running stack (`endpoint:` / a gcx context) | +| `compose` | OATS boots a docker-compose stack; `template: lgtm` lets OATS own the LGTM ports | +| `k3d` | OATS boots a k3d (k3s-in-docker) cluster | + +`compose` and `k3d` fixtures are booted, waited on for readiness, and torn down +by OATS. `remote` fixtures are assumed ready. + +## Seed + +A case populates the stack before assertions run via one of two `seed.type` +modes: + +- **`app`** is the default for acceptance tests: drive your real instrumented + app and assert on what it actually emits end-to-end (SDK → collector → + backend). This is what most consumer repos want. +- **`inline-otlp`** carries a hand-written OTLP payload and pushes it straight + at the OTLP endpoint — no app, no SDK. Reach for it when there is no app to + drive: to exercise the pipeline or backend in isolation (collector + processor/transform config, ingestion, query behaviour), or to pin an exact + payload shape a test depends on. OATS's own e2e suite leans on it heavily for + precisely this — deterministic telemetry without booting an instrumented app. + +```yaml +# App-backed: the suite fixture boots an instrumented app, and `input` +# requests drive it so it emits telemetry. +seed: + type: app +input: + - path: /rolldice?rolls=5 # method defaults to GET +``` + +```yaml +# Inline-OTLP: the case carries its own payload, pushed as OTLP/HTTP JSON. +# No app, no SDK. Declare only the signals you need. +seed: + type: inline-otlp + traces: + - service: my-service + spans: + - name: seed-operation # kind defaults to INTERNAL, duration to 200ms + logs: + - service: my-service + body: seed-log-line # severity_number defaults to 9 (INFO) + metrics: + - service: my-service + name: seed_counter # monotonic sum → PromQL `seed_counter_total` + value: 42 +``` + +> Inline-OTLP can seed traces, logs, and metrics. **Profiles cannot be +> inline-seeded** — assert profiles against an app-backed fixture that produces +> them (e.g. an eBPF profiler or a pyroscope-instrumented app). + +## Assertions + +Every signal block under `expected` shares one assertion vocabulary, plus a +few signal-specific keys. Each entry first names a query, then the checks that +must hold against its result. Every assertion is retried until it passes or +`--timeout` elapses. + +Shared keys (valid on `traces`, `metrics`, `logs`, `profiles`): + +| Key | Meaning | +|-----|---------| +| `contains` | string (or list) that must appear in the query output | +| `not_contains` | string (or list) that must **not** appear | +| `regex` | RE2 pattern (or list) that must match the output | +| `match` | structural row match — list of `{match_type, name, attributes}` | +| `count` | comparison against the number of rows, e.g. `'== 1'`, `'>= 2'` | +| `absent` | the query must return nothing for the whole `--absent-timeout` window | + +`match` (and the trace-only `match_spans`) entries: + +```yaml +match: + - match_type: strict # "strict" (default) or "regexp" + name: seed-log-line # for regexp, an RE2 pattern + attributes: # optional; list of {key, value?} + - key: service_name + value: my-service # omit `value` to assert the key is merely present +``` + +Signal-specific keys: + +- `traces`: `traceql` (required), `match_spans` (span-row match, same shape as `match`) +- `metrics`: `promql` (required), `value` (compare the sample value, e.g. `'>= 1'`, `'== 42'`) +- `logs`: `logql` (required) +- `profiles`: `query` (required) + +Example covering several shapes: + +```yaml +expected: + traces: + - traceql: '{ resource.service.name = "my-service" }' + match_spans: + - match_type: regexp + name: '^GET /rolldice.*' + attributes: + - key: service.name + value: my-service + metrics: + - promql: 'seed_counter_total{service_name="my-service"}' + value: '>= 1' + logs: + - logql: '{service_name="my-service"}' + regex: '.*rolling the dice.*' + count: '>= 1' + profiles: + - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{service_name="my-service"}' + contains: main +``` + +### compose-logs + +For `compose` fixtures, `expected.compose-logs` greps the container logs +(`docker compose logs`) for each string — useful for asserting on output that +never reaches the OTLP pipeline: + +```yaml +expected: + compose-logs: + - app started ok +``` + +## Custom checks + +`expected.custom-checks` runs an arbitrary script and treats **exit code 0 as +pass**, non-zero as fail. Combined stdout+stderr is captured and printed on +failure. Like every assertion it is retried until it passes or `--timeout` +elapses. + +```yaml +expected: + custom-checks: + - script: ./verify.sh # resolved relative to the case file's directory +``` + +`script` may be a path (relative to the case dir, or absolute) or an inline +script beginning with a `#!` shebang. The process runs with its working +directory set to the case dir and inherits the parent environment plus these +OATS-provided variables: + +| Variable | Fixtures | Meaning | +|----------|----------|---------| +| `OATS_FIXTURE_TYPE` | all | `remote` / `compose` / `k3d` | +| `OATS_GRAFANA_URL` | all | base URL of the fixture's Grafana | +| `OATS_APP_URL` | remote, k3d | base URL of the app under test | +| `OATS_OTLP_HTTP` | compose, k3d | OTLP/HTTP endpoint | +| `OATS_PYROSCOPE_URL` | compose, k3d | Pyroscope base URL | +| `COMPOSE_PROJECT_NAME`, `COMPOSE_FILE`, `OATS_COMPOSE_FILE_ARGS` | compose | let the script run its own `docker compose` commands | + +A custom check that queries Grafana directly (replacing, for example, a legacy +`compose-logs` grep with a real LogQL query): + +```bash +#!/usr/bin/env bash +set -euo pipefail +curl -fsS "${OATS_GRAFANA_URL:?}/api/health" >/dev/null +echo "custom check ok" +``` + +## Running in parallel + +- Cases inside one suite run sequentially. +- Suites can run concurrently with `--parallel N`, but only where fixture + isolation allows it: remote suites, and `template = "lgtm"` compose suites + without app seeds. k3d suites and app-seed compose suites always run serially + regardless of the flag. +- **Memory, not CPU, is the limit for compose parallelism.** Each parallel + `template = "lgtm"` suite boots its *own* LGTM stack — a unique compose project + with dynamically allocated host ports — so suites can neither collide on ports + nor see each other's telemetry. That hermetic isolation costs one full LGTM + container per concurrent suite (Grafana + Loki + Tempo + Mimir + Prometheus + + Pyroscope + collector, on the order of ~1 GB each). `--parallel` therefore + defaults to `1`; raise it deliberately, sized to available RAM rather than core + count. Remote suites boot nothing and parallelize cheaply. +- OATS owns local LGTM bootstrapping and gcx bootstrap for fixture-backed runs. diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 00000000..45f3c1e4 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,119 @@ +# Running OATS in CI + +OATS is two binaries: `oats` (the runner) and `gcx` (the assertion engine it +drives). This guide covers installing both reproducibly and wiring a fast, +correct CI job. The reference model is +[opentelemetry-java-examples](https://github.com/open-telemetry/opentelemetry-java-examples/blob/main/.github/workflows/oats-tests.yml), +which already gates by path and runs everything through mise. + +## Install + +### mise (recommended) + +Pin both tools with the [aqua](https://mise.jdx.dev/dev-tools/backends/aqua.html) +backend — mise resolves the latest releases and locks them into `mise.toml`: + +```sh +mise use --pin aqua:grafana/oats aqua:grafana/gcx +``` + +which produces a pinned `mise.toml`: + +```toml +[tools] +"aqua:grafana/oats" = "0.7.0" +"aqua:grafana/gcx" = "0.4.3" +``` + +Pinning is what makes CI caching safe (see below): a pinned release is an +immutable artifact, so `gcx --version` is byte-stable across runs and a version +bump is an explicit `mise.toml` diff. Renovate keeps the pins current. + +### GitHub releases (no mise) + +Download the binary for your OS/arch: + +- oats — +- gcx — + +Pin to a specific tag; avoid tracking a floating "latest" if you rely on the +result cache. + +### go install (oats from source) + +```sh +go install github.com/grafana/oats@latest +``` + +`oats` can bootstrap `gcx` itself for fixture-backed runs, but pinning gcx +explicitly (mise/release) is recommended so runs are reproducible and the cache +stays valid. + +## The workflow shape + +```yaml +on: + pull_request: + paths: # coarse gate: skip the whole job when nothing relevant changed + - .github/workflows/oats-tests.yml + - mise.toml + - "my-example/**" + workflow_dispatch: + +jobs: + oats: + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: jdx/mise-action@v2 # installs pinned oats + gcx from mise.toml + - run: oats --config oats.toml +``` + +The `paths` filter is the cheapest and most important optimization: if a PR +touches nothing the tests depend on, the job never runs. This alone covers most +of the saving. + +## Result caching (optional) + +`oats` keeps a skip-when-unchanged cache under `--cache-dir`. A case is skipped +when `(case yaml, fixture config, gcx version, oats version)` hashes to a +previous green run within the TTL. Within a job that *did* trigger, this skips +the cases a PR didn't actually affect. + +To persist it across CI runs: + +```yaml + - uses: actions/cache@v4 + with: + path: ~/.cache/oats + key: oats-${{ hashFiles('mise.toml') }} + - run: oats --config oats.toml --cache-dir ~/.cache/oats +``` + +Keying on `hashFiles('mise.toml')` scopes the cache to a gcx/oats version pair, +so a version bump starts a fresh cache — no stale skips from version drift. + +### Correctness caveat — floating image tags + +The cache key hashes the fixture **config** (`fixture.type`, compose file paths, +image tags, env, endpoint …), **not** the fixture's **contents**. If the app +under test is baked into an image with a floating tag (`:latest`), rebuilding +that image under the same tag produces an identical key — so a case can be +skipped against stale bytes (a false green). + +You are safe when either holds: + +- the app is **built fresh from source in the same PR** and the case yaml lives + in the changed directory (the path gate + case hash already cover it — this is + the java-examples case), or +- the image is **pinned by digest** (`image@sha256:…`), so a new image changes + the fixture config and therefore the key. + +If neither holds, don't persist the cache in CI until the image digest is folded +into the cache key (`cache.Key.Extra` exists for exactly this; the CLI does not +set it today). + +Caching is opt-in — a clear pin + path gate is the baseline recommendation, and +the cache is a bonus on top. From 8e0c072d67df6afe822e831c6cc10c9bfedf58f5 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 11:40:55 +0000 Subject: [PATCH 110/124] feat(fixture): make app-seed compose suites parallel-safe Enable `--parallel` for compose app-seed suites by giving the app an ephemeral host port instead of a shared fixed one. When a compose fixture sets app_service + app_port, startCompose discovers the host port docker published for that service (via the existing lookupComposePort path) and resolveEndpoint drives the app on it. SupportsParallel then treats an app-seed suite as parallel-safe only when app_service + app_port are set; otherwise it still shares the fixed --app-port and runs serially. No schema change (app_service/app_port already existed). Documented in case-reference.md. Signed-off-by: Gregor Zeitlinger --- docs/case-reference.md | 17 ++++++++++++++--- fixture/compose.go | 22 ++++++++++++++++++++++ fixture/fixture.go | 10 ++++++++-- fixture/fixture_test.go | 27 +++++++++++++++++++++++++++ internal/cli/main.go | 3 +++ 5 files changed, 74 insertions(+), 5 deletions(-) diff --git a/docs/case-reference.md b/docs/case-reference.md index ec3a1116..31bef512 100644 --- a/docs/case-reference.md +++ b/docs/case-reference.md @@ -67,6 +67,14 @@ selects how it is stood up: `compose` and `k3d` fixtures are booted, waited on for readiness, and torn down by OATS. `remote` fixtures are assumed ready. +For a `compose` fixture driven by a `seed: app` case, set `app_service` (the +compose service name of the app) and `app_port` (the app's container port). OATS +then discovers the host port docker published for that service, so the app can +bind an **ephemeral** host port (`127.0.0.1::`) rather than a fixed one +— which is what lets app-seed suites run under `--parallel` without colliding. +Omit them and the app is driven on the fixed `--app-port` (default 8080), which +forces the suite to run serially. + ## Seed A case populates the stack before assertions run via one of two `seed.type` @@ -226,9 +234,12 @@ echo "custom check ok" - Cases inside one suite run sequentially. - Suites can run concurrently with `--parallel N`, but only where fixture - isolation allows it: remote suites, and `template = "lgtm"` compose suites - without app seeds. k3d suites and app-seed compose suites always run serially - regardless of the flag. + isolation allows it: remote suites, and `template = "lgtm"` compose suites that + publish **no fixed host ports**. An app-seed compose suite qualifies when its + app binds an *ephemeral* host port (`127.0.0.1::`) and the fixture sets + `app_service` + `app_port` so OATS can discover the published port; an app-seed + suite without `app_service`, or any compose file that binds a fixed host port, + runs serially. k3d suites always run serially. - **Memory, not CPU, is the limit for compose parallelism.** Each parallel `template = "lgtm"` suite boots its *own* LGTM stack — a unique compose project with dynamically allocated host ports — so suites can neither collide on ports diff --git a/fixture/compose.go b/fixture/compose.go index 10a38ca9..dec48f0a 100644 --- a/fixture/compose.go +++ b/fixture/compose.go @@ -64,6 +64,28 @@ func startCompose(plan discovery.Plan) (Handle, Runtime, error) { ComposeFiles: composeFiles, ComposeProject: project, } + // When the fixture names an app service, resolve the port docker published + // for it. This lets the app bind an ephemeral host port (127.0.0.1::) + // instead of a fixed one, which is what makes app-seed suites parallel-safe. + if plan.Fixture.AppService != "" && plan.Fixture.AppPort > 0 { + appPort, portErr := lookupComposePort(composeFiles, suiteEnv, plan.Fixture.AppService, strconv.Itoa(plan.Fixture.AppPort)) + if portErr != nil { + _ = suite.Close() + if cleanup != nil { + _ = cleanup() + } + return nil, Runtime{}, fmt.Errorf("resolve app host port for service %q: %w", plan.Fixture.AppService, portErr) + } + p, convErr := strconv.Atoi(appPort) + if convErr != nil { + _ = suite.Close() + if cleanup != nil { + _ = cleanup() + } + return nil, Runtime{}, fmt.Errorf("invalid app host port %q for service %q: %w", appPort, plan.Fixture.AppService, convErr) + } + rt.AppHostPort = p + } cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL) if cfgErr != nil { if cleanup != nil { diff --git a/fixture/fixture.go b/fixture/fixture.go index a86d219f..96a16ca7 100644 --- a/fixture/fixture.go +++ b/fixture/fixture.go @@ -53,6 +53,7 @@ type Runtime struct { GrafanaURL string OTLPHTTP string PyroscopeURL string + AppHostPort int CustomCheckEnv []string ComposeFiles []string ComposeProject string @@ -113,8 +114,13 @@ func SupportsParallel(plan discovery.Plan) (bool, string) { return false, "compose fixtures are only parallel-safe when OATS owns the LGTM ports via template=lgtm" } for _, c := range plan.Cases { - if c.Seed.Type == "app" { - return false, "compose suites with app seeds still rely on shared fixed app ports" + // App seeds are parallel-safe only when OATS can give the app an + // ephemeral host port instead of a shared fixed one — which requires + // fixture.app_service (+ app_port) so the published port can be + // discovered. Without it the app falls back to the fixed --app-port + // and parallel suites would collide. + if c.Seed.Type == "app" && (plan.Fixture.AppService == "" || plan.Fixture.AppPort == 0) { + return false, "compose app-seed suites need fixture.app_service and app_port so OATS can publish an ephemeral app port; otherwise they share a fixed app port" } } for _, file := range extraComposeFiles(plan) { diff --git a/fixture/fixture_test.go b/fixture/fixture_test.go index b7b12e27..f449970f 100644 --- a/fixture/fixture_test.go +++ b/fixture/fixture_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/grafana/oats/casefile" "github.com/grafana/oats/discovery" "github.com/grafana/oats/runner" "github.com/grafana/oats/testhelpers/remote" @@ -317,6 +318,32 @@ expected: } } +// TestSupportsParallel_AppSeedRequiresAppService checks the gate that makes +// app-seed compose suites parallel-safe: only when fixture.app_service and +// app_port are set (so OATS can publish + discover an ephemeral app port) is the +// suite safe; otherwise it would share a fixed app port with its peers. +func TestSupportsParallel_AppSeedRequiresAppService(t *testing.T) { + appCase := &casefile.Case{Seed: casefile.Seed{Type: "app"}} + + // app seed, no app_service → not parallel-safe. + noService := discovery.Plan{ + Fixture: discovery.FixtureConfig{Type: "compose", Template: "lgtm"}, + Cases: []*casefile.Case{appCase}, + } + if safe, reason := SupportsParallel(noService); safe { + t.Fatalf("app-seed without app_service should not be parallel-safe, got safe (%s)", reason) + } + + // app seed with app_service + app_port → parallel-safe. + withService := discovery.Plan{ + Fixture: discovery.FixtureConfig{Type: "compose", Template: "lgtm", AppService: "app", AppPort: 8080}, + Cases: []*casefile.Case{appCase}, + } + if safe, reason := SupportsParallel(withService); !safe { + t.Fatalf("app-seed with app_service+app_port should be parallel-safe: %s", reason) + } +} + // TestWaitForGrafanaToken_DefaultFixtureReturnsEmpty exercises the token seam // for a fixture type that has no token to read: it returns immediately with an // empty token and no error. diff --git a/internal/cli/main.go b/internal/cli/main.go index 0fdb304e..f176e823 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -366,6 +366,9 @@ func resolveEndpoint(plan discovery.Plan, rt fixture.Runtime, gcxContextOverride if rt.OTLPHTTP != "" { ep.OTLPHTTP = rt.OTLPHTTP } + if rt.AppHostPort > 0 { + ep.AppPort = rt.AppHostPort + } ep.CustomCheckEnv = append(ep.CustomCheckEnv, rt.CustomCheckEnv...) case "k3d": if rt.GCXConfig != "" { From 5f101fddda4f6ec5cf2b184351d3a6a22a8e040e Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 12:53:18 +0000 Subject: [PATCH 111/124] refactor(fixture): nested type-grouped fixture config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the flat, duplicated FixtureConfig (defined twice — toml in discovery, yaml in casefile — with a hand-written mapping and two validators) with a single casefile.FixtureConfig carrying dual toml+yaml tags. Group fields under nested compose/k3d/remote blocks and drop the `type` discriminator: the block that is set selects the kind (Kind()), and validation enforces exactly one. Inside the compose block, compose_file/compose_files become file/files (the block name makes the prefix redundant). This is a breaking schema change; v3 is unreleased. Migrated examples, the e2e case suite, README/case-reference/UPGRADING docs, and the `oats migrate` hint emitters to the nested shape. Also folds in the resolve-path cleanup: a single `fail` teardown closure in startCompose (replacing repeated suite.Close()+cleanup() blocks) and a uniform Runtime.AppHostPort read in resolveEndpoint for every fixture type. Signed-off-by: Gregor Zeitlinger --- README.md | 6 +- UPGRADING.md | 5 +- casefile/case.go | 119 +++++++++++++----- discovery/discovery.go | 81 ++---------- discovery/discovery_test.go | 57 ++++++--- docs/case-reference.md | 42 +++++-- examples/fixtures/oats.toml | 8 +- examples/smoke/oats.toml | 3 +- fixture/compose.go | 90 ++++++------- fixture/fixture.go | 29 ++--- fixture/fixture_test.go | 87 +++++++------ fixture/k3d.go | 4 +- internal/cli/integration_test.go | 44 +++---- internal/cli/main.go | 33 ++--- migrate/migrate.go | 10 +- migrate/migrate_test.go | 16 ++- .../cases/assert/absent-fail/files/oats.yaml | 4 +- tests/e2e/cases/assert/absent/files/oats.yaml | 4 +- .../assert/contains-fail/files/oats.yaml | 4 +- .../e2e/cases/assert/contains/files/oats.yaml | 4 +- .../cases/assert/count-fail/files/oats.yaml | 4 +- tests/e2e/cases/assert/count/files/oats.yaml | 4 +- .../cases/assert/match-fail/files/oats.yaml | 4 +- .../assert/match-regexp-fail/files/oats.yaml | 4 +- .../cases/assert/match-regexp/files/oats.yaml | 4 +- .../assert/match-spans-fail/files/oats.yaml | 4 +- .../assert/not-contains-fail/files/oats.yaml | 4 +- .../cases/assert/not-contains/files/oats.yaml | 4 +- .../cases/assert/profile-fail/files/oats.yaml | 4 +- .../e2e/cases/assert/profile/files/oats.yaml | 4 +- .../cases/assert/regex-fail/files/oats.yaml | 4 +- tests/e2e/cases/assert/regex/files/oats.yaml | 4 +- .../cases/assert/value-fail/files/oats.yaml | 4 +- tests/e2e/cases/assert/value/files/oats.yaml | 4 +- .../e2e/cases/cli/cache-clear/files/oats.toml | 3 +- .../e2e/cases/cli/cache-skip/files/oats.toml | 3 +- tests/e2e/cases/cli/fail-fast/files/oats.toml | 3 +- .../custom/custom-check-fail/files/oats.yaml | 4 +- .../cases/custom/custom-check/files/oats.yaml | 4 +- .../fixture/compose-app-seed/files/oats.yaml | 6 +- .../fixture/compose-logs-fail/files/oats.yaml | 6 +- .../fixture/compose-logs/files/oats.yaml | 6 +- .../cases/fixture/k3d-fail/files/oats.yaml | 14 +-- .../cases/fixture/k3d-smoke/files/oats.yaml | 14 +-- .../fixture/parallel-compose/files/oats.toml | 5 +- .../fixture/remote-basic/files/oats.yaml | 4 +- .../e2e/cases/fixture/remote-basic/test.yaml | 4 +- tests/e2e/cases/format/ndjson/files/oats.yaml | 4 +- .../cases/seed/inline-otlp/files/oats.yaml | 4 +- 49 files changed, 407 insertions(+), 383 deletions(-) diff --git a/README.md b/README.md index 56a2a7ef..0b29910b 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,9 @@ oats-schema-version: 3 name: rolldice traces have a route attribute fixture: - type: compose - template: lgtm - compose_file: docker-compose.oats.yml + compose: + template: lgtm + file: docker-compose.oats.yml seed: type: app diff --git a/UPGRADING.md b/UPGRADING.md index 85e983e2..bcbc7acc 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -135,9 +135,8 @@ name = "rolldice" cases = ["cases/*.yaml"] fixture = "compose-lgtm" -[fixture.compose-lgtm] -type = "compose" -compose_file = "docker-compose.yaml" +[fixture.compose-lgtm.compose] +file = "docker-compose.yaml" ``` ```yaml diff --git a/casefile/case.go b/casefile/case.go index 23580fbe..0c30d078 100644 --- a/casefile/case.go +++ b/casefile/case.go @@ -57,21 +57,67 @@ type Seed struct { Vars map[string]any `yaml:"vars,omitempty"` } +// FixtureConfig declares how a suite stands up the backends its cases run +// against. Exactly one of the type-specific blocks is set; the block that is +// present selects the fixture kind. Dual toml/yaml tags let the same struct +// load from oats.toml (BurntSushi/toml) and from a case yaml (yaml.v3). type FixtureConfig struct { - Type string `yaml:"type,omitempty"` - Template string `yaml:"template,omitempty"` - ComposeFile string `yaml:"compose_file,omitempty"` - ComposeFiles []string `yaml:"compose_files,omitempty"` - Env []string `yaml:"env,omitempty"` - K8sDir string `yaml:"k8s_dir,omitempty"` - AppService string `yaml:"app_service,omitempty"` - AppDockerFile string `yaml:"app_docker_file,omitempty"` - AppDockerContext string `yaml:"app_docker_context,omitempty"` - AppDockerTag string `yaml:"app_docker_tag,omitempty"` - AppPort int `yaml:"app_port,omitempty"` - ImportImages []string `yaml:"import_images,omitempty"` - PoolSize int `yaml:"pool_size,omitempty"` - Endpoint string `yaml:"endpoint,omitempty"` + Compose *ComposeFixture `toml:"compose,omitempty" yaml:"compose,omitempty"` + K3D *K3DFixture `toml:"k3d,omitempty" yaml:"k3d,omitempty"` + Remote *RemoteFixture `toml:"remote,omitempty" yaml:"remote,omitempty"` +} + +// ComposeFixture boots a docker-compose stack. template selects a built-in +// stack ("lgtm"); file/files point at compose files relative to the fixture +// source dir. app_service + app_port let OATS publish and discover an +// ephemeral host port for the application under test. +type ComposeFixture struct { + Template string `toml:"template,omitempty" yaml:"template,omitempty"` + File string `toml:"file,omitempty" yaml:"file,omitempty"` + Files []string `toml:"files,omitempty" yaml:"files,omitempty"` + Env []string `toml:"env,omitempty" yaml:"env,omitempty"` + AppService string `toml:"app_service,omitempty" yaml:"app_service,omitempty"` + AppPort int `toml:"app_port,omitempty" yaml:"app_port,omitempty"` +} + +// K3DFixture boots a k3d cluster and builds/imports the application image. +type K3DFixture struct { + K8sDir string `toml:"k8s_dir,omitempty" yaml:"k8s_dir,omitempty"` + AppService string `toml:"app_service,omitempty" yaml:"app_service,omitempty"` + AppDockerFile string `toml:"app_docker_file,omitempty" yaml:"app_docker_file,omitempty"` + AppDockerContext string `toml:"app_docker_context,omitempty" yaml:"app_docker_context,omitempty"` + AppDockerTag string `toml:"app_docker_tag,omitempty" yaml:"app_docker_tag,omitempty"` + AppPort int `toml:"app_port,omitempty" yaml:"app_port,omitempty"` + ImportImages []string `toml:"import_images,omitempty" yaml:"import_images,omitempty"` + PoolSize int `toml:"pool_size,omitempty" yaml:"pool_size,omitempty"` +} + +// RemoteFixture points at an already-running stack; OATS boots nothing. +type RemoteFixture struct { + Endpoint string `toml:"endpoint,omitempty" yaml:"endpoint,omitempty"` +} + +// Kind returns "compose"/"k3d"/"remote", or "" when no block is set. Exactly +// one block should be set (enforced by Validate). +func (f FixtureConfig) Kind() string { + switch { + case f.Compose != nil: + return "compose" + case f.K3D != nil: + return "k3d" + case f.Remote != nil: + return "remote" + default: + return "" + } +} + +// HasManagedApp reports whether OATS manages the compose app endpoint (so it +// can publish + discover an ephemeral host port): a compose block with +// app_service + app_port. It drives both the parallel-safety gate and app-port +// discovery, so the two stay in lockstep. +func (f FixtureConfig) HasManagedApp() bool { + return f.Compose != nil && f.Compose.AppService != "" && f.Compose.AppPort > 0 } type SeedTrace struct { @@ -412,27 +458,38 @@ func (c *Case) Validate() error { return nil } -func (f FixtureConfig) Validate(path string) error { - switch f.Type { - case "compose": - if f.Template == "" && f.ComposeFile == "" && len(f.ComposeFiles) == 0 { - return fmt.Errorf("%s: type=compose requires template, compose_file, or compose_files", path) +// Validate enforces that exactly one fixture block is set and that the set +// block carries the fields its kind requires. label names the fixture in +// error messages (the fixture name from oats.toml, or "fixture" for a +// case-level block). +func (f FixtureConfig) Validate(label string) error { + set := 0 + for _, present := range []bool{f.Compose != nil, f.K3D != nil, f.Remote != nil} { + if present { + set++ + } + } + if set != 1 { + return fmt.Errorf("fixture %q: set exactly one of compose/k3d/remote", label) + } + switch { + case f.Compose != nil: + c := f.Compose + if c.Template == "" && c.File == "" && len(c.Files) == 0 { + return fmt.Errorf("fixture %q: compose requires template, file, or files", label) } - if f.ComposeFile != "" && len(f.ComposeFiles) > 0 { - return fmt.Errorf("%s: use compose_file or compose_files, not both", path) + if c.File != "" && len(c.Files) > 0 { + return fmt.Errorf("fixture %q: compose sets file or files, not both", label) } - case "k3d": - if f.K8sDir == "" || f.AppService == "" || f.AppDockerFile == "" || f.AppDockerTag == "" || f.AppPort == 0 { - return fmt.Errorf("%s: type=k3d requires k8s_dir, app_service, app_docker_file, app_docker_tag, and app_port", path) + case f.K3D != nil: + k := f.K3D + if k.K8sDir == "" || k.AppService == "" || k.AppDockerFile == "" || k.AppDockerTag == "" || k.AppPort == 0 { + return fmt.Errorf("fixture %q: k3d requires k8s_dir, app_service, app_docker_file, app_docker_tag, and app_port", label) } - case "remote": - if f.Endpoint == "" { - return fmt.Errorf("%s: type=remote requires endpoint", path) + case f.Remote != nil: + if f.Remote.Endpoint == "" { + return fmt.Errorf("fixture %q: remote requires endpoint", label) } - case "": - return fmt.Errorf("%s.type: required (compose | k3d | remote)", path) - default: - return fmt.Errorf("%s.type: unknown value %q (expected compose, k3d, or remote)", path, f.Type) } return nil } diff --git a/discovery/discovery.go b/discovery/discovery.go index 778e6b38..ada00c3f 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -24,11 +24,11 @@ import ( // directly so a misnamed key surfaces as a "field not defined" error from // the toml decoder rather than a silent miss. type RootConfig struct { - Meta Meta `toml:"meta"` - Cases []string `toml:"cases"` - Suites []SuiteConfig `toml:"suite"` - Fixture map[string]FixtureConfig `toml:"fixture"` - Cache CacheConfig `toml:"cache,omitempty"` + Meta Meta `toml:"meta"` + Cases []string `toml:"cases"` + Suites []SuiteConfig `toml:"suite"` + Fixture map[string]casefile.FixtureConfig `toml:"fixture"` + Cache CacheConfig `toml:"cache,omitempty"` // SourceDir is the directory of the loaded oats.toml. Case glob // expressions resolve relative to it. @@ -46,23 +46,6 @@ type SuiteConfig struct { Tags []string `toml:"tags,omitempty"` } -type FixtureConfig struct { - Type string `toml:"type"` // "compose" | "k3d" | "remote" - Template string `toml:"template,omitempty"` - ComposeFile string `toml:"compose_file,omitempty"` - ComposeFiles []string `toml:"compose_files,omitempty"` - Env []string `toml:"env,omitempty"` - K8sDir string `toml:"k8s_dir,omitempty"` - AppService string `toml:"app_service,omitempty"` - AppDockerFile string `toml:"app_docker_file,omitempty"` - AppDockerContext string `toml:"app_docker_context,omitempty"` - AppDockerTag string `toml:"app_docker_tag,omitempty"` - AppPort int `toml:"app_port,omitempty"` - ImportImages []string `toml:"import_images,omitempty"` - PoolSize int `toml:"pool_size,omitempty"` - Endpoint string `toml:"endpoint,omitempty"` // remote only -} - type CacheConfig struct { TTLDays int `toml:"ttl_days,omitempty"` // zero → use runtime default } @@ -121,27 +104,8 @@ func (c *RootConfig) Validate() error { } } for name, f := range c.Fixture { - switch f.Type { - case "compose": - if f.Template == "" && f.ComposeFile == "" && len(f.ComposeFiles) == 0 { - return fmt.Errorf("fixture %q: type=compose requires template, compose_file, or compose_files", name) - } - if f.ComposeFile != "" && len(f.ComposeFiles) > 0 { - return fmt.Errorf("fixture %q: use compose_file or compose_files, not both", name) - } - case "k3d": - if f.K8sDir == "" || f.AppService == "" || f.AppDockerFile == "" || f.AppDockerTag == "" || f.AppPort == 0 { - return fmt.Errorf("fixture %q: type=k3d requires k8s_dir, app_service, app_docker_file, app_docker_tag, and app_port", name) - } - // PoolSize=0 means "single ephemeral cluster" — valid. - case "remote": - if f.Endpoint == "" { - return fmt.Errorf("fixture %q: type=remote requires endpoint", name) - } - case "": - return fmt.Errorf("fixture %q: type is required (compose | k3d | remote)", name) - default: - return fmt.Errorf("fixture %q: unknown type %q", name, f.Type) + if err := f.Validate(name); err != nil { + return err } } return nil @@ -158,7 +122,7 @@ type Filter struct { // Plan is one suite plus the cases it expanded to. type Plan struct { Suite SuiteConfig - Fixture FixtureConfig + Fixture casefile.FixtureConfig FixtureSourceDir string Cases []*casefile.Case } @@ -328,12 +292,12 @@ func dirLabel(dir string) string { return base } -func (c *RootConfig) resolveSuiteFixture(suite SuiteConfig, cases []*casefile.Case) (FixtureConfig, string, error) { +func (c *RootConfig) resolveSuiteFixture(suite SuiteConfig, cases []*casefile.Case) (casefile.FixtureConfig, string, error) { if suite.Fixture != "" { return c.Fixture[suite.Fixture], c.SourceDir, nil } var ( - fixture FixtureConfig + fixture casefile.FixtureConfig sourceDir string seen bool ) @@ -341,41 +305,22 @@ func (c *RootConfig) resolveSuiteFixture(suite SuiteConfig, cases []*casefile.Ca if tc.Fixture == nil { continue } - next := fixtureConfigFromCase(*tc.Fixture) + next := *tc.Fixture nextSourceDir := filepath.Dir(tc.SourcePath) if !seen { fixture, sourceDir, seen = next, nextSourceDir, true continue } if !reflect.DeepEqual(fixture, next) || sourceDir != nextSourceDir { - return FixtureConfig{}, "", fmt.Errorf("suite omits fixture but cases do not agree on one shared fixture") + return casefile.FixtureConfig{}, "", fmt.Errorf("suite omits fixture but cases do not agree on one shared fixture") } } if !seen { - return FixtureConfig{}, "", nil + return casefile.FixtureConfig{}, "", nil } return fixture, sourceDir, nil } -func fixtureConfigFromCase(f casefile.FixtureConfig) FixtureConfig { - return FixtureConfig{ - Type: f.Type, - Template: f.Template, - ComposeFile: f.ComposeFile, - ComposeFiles: append([]string(nil), f.ComposeFiles...), - Env: append([]string(nil), f.Env...), - K8sDir: f.K8sDir, - AppService: f.AppService, - AppDockerFile: f.AppDockerFile, - AppDockerContext: f.AppDockerContext, - AppDockerTag: f.AppDockerTag, - AppPort: f.AppPort, - ImportImages: append([]string(nil), f.ImportImages...), - PoolSize: f.PoolSize, - Endpoint: f.Endpoint, - } -} - // Summary renders a single line per plan suitable for `oats list`. It does // not load any cases — useful for a dry-run before deciding to expand globs. func (c *RootConfig) Summary() string { diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 62944cd7..6a1d93bf 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/grafana/oats/casefile" ) func writeFile(t *testing.T, dir, rel, body string) { @@ -41,8 +43,7 @@ cases = ["cases/*.yaml"] fixture = "lgtm-shared" tags = ["traces"] -[fixture.lgtm-shared] -type = "compose" +[fixture.lgtm-shared.compose] template = "lgtm" `) writeFile(t, dir, "cases/a.yaml", strings.Replace(validCaseYAML, "%s", "case-a", 1)) @@ -66,7 +67,7 @@ template = "lgtm" if plans[0].Cases[0].Name != "case-a" { t.Errorf("sort order: got %q first", plans[0].Cases[0].Name) } - if plans[0].Fixture.Type != "compose" { + if plans[0].Fixture.Kind() != "compose" { t.Errorf("fixture resolved wrong: %+v", plans[0].Fixture) } } @@ -173,17 +174,34 @@ cases = ["b.yaml"] } } -func TestValidate_BadFixtureType(t *testing.T) { +func TestValidate_EmptyFixtureRejected(t *testing.T) { + cfg := &RootConfig{ + Meta: Meta{Version: 2}, + Suites: []SuiteConfig{{ + Name: "s", Cases: []string{"a.yaml"}, Fixture: "x", + }}, + Fixture: map[string]casefile.FixtureConfig{"x": {}}, + } + err := cfg.Validate() + if err == nil || !strings.Contains(err.Error(), "set exactly one of compose/k3d/remote") { + t.Errorf("expected empty-fixture error, got %v", err) + } +} + +func TestValidate_MultipleFixtureBlocksRejected(t *testing.T) { cfg := &RootConfig{ Meta: Meta{Version: 2}, Suites: []SuiteConfig{{ Name: "s", Cases: []string{"a.yaml"}, Fixture: "x", }}, - Fixture: map[string]FixtureConfig{"x": {Type: "weird"}}, + Fixture: map[string]casefile.FixtureConfig{"x": { + Compose: &casefile.ComposeFixture{Template: "lgtm"}, + Remote: &casefile.RemoteFixture{Endpoint: "http://localhost:4318"}, + }}, } err := cfg.Validate() - if err == nil || !strings.Contains(err.Error(), `unknown type "weird"`) { - t.Errorf("expected unknown-type error, got %v", err) + if err == nil || !strings.Contains(err.Error(), "set exactly one of compose/k3d/remote") { + t.Errorf("expected multiple-block error, got %v", err) } } @@ -204,7 +222,7 @@ func TestValidate_RemoteRequiresEndpoint(t *testing.T) { cfg := &RootConfig{ Meta: Meta{Version: 2}, Suites: []SuiteConfig{{Name: "s", Cases: []string{"a.yaml"}}}, - Fixture: map[string]FixtureConfig{"r": {Type: "remote"}}, + Fixture: map[string]casefile.FixtureConfig{"r": {Remote: &casefile.RemoteFixture{}}}, } if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "endpoint") { t.Errorf("expected endpoint error, got %v", err) @@ -217,15 +235,16 @@ func TestValidate_ComposeFilesConflict(t *testing.T) { Suites: []SuiteConfig{{ Name: "s", Cases: []string{"a.yaml"}, Fixture: "c", }}, - Fixture: map[string]FixtureConfig{"c": { - Type: "compose", - ComposeFile: "one.yml", - ComposeFiles: []string{ - "two.yml", + Fixture: map[string]casefile.FixtureConfig{"c": { + Compose: &casefile.ComposeFixture{ + File: "one.yml", + Files: []string{ + "two.yml", + }, }, }}, } - if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "compose_file or compose_files") { + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "compose sets file or files, not both") { t.Errorf("expected compose conflict error, got %v", err) } } @@ -236,11 +255,11 @@ func TestValidate_K3DRequiresFields(t *testing.T) { Suites: []SuiteConfig{{ Name: "s", Cases: []string{"a.yaml"}, Fixture: "k", }}, - Fixture: map[string]FixtureConfig{"k": { - Type: "k3d", + Fixture: map[string]casefile.FixtureConfig{"k": { + K3D: &casefile.K3DFixture{}, }}, } - if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "type=k3d requires") { + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "k3d requires") { t.Errorf("expected k3d field error, got %v", err) } } @@ -310,10 +329,10 @@ func TestExampleV2FixturesConfigLoads(t *testing.T) { if len(plans) != 2 { t.Fatalf("expected two suites, got %+v", plans) } - if plans[0].Suite.Name != "compose-app" || plans[0].Fixture.Type != "compose" || len(plans[0].Cases) != 1 { + if plans[0].Suite.Name != "compose-app" || plans[0].Fixture.Kind() != "compose" || len(plans[0].Cases) != 1 { t.Fatalf("unexpected first plan: %+v", plans[0]) } - if plans[1].Suite.Name != "k3d-app" || plans[1].Fixture.Type != "k3d" || len(plans[1].Cases) != 1 { + if plans[1].Suite.Name != "k3d-app" || plans[1].Fixture.Kind() != "k3d" || len(plans[1].Cases) != 1 { t.Fatalf("unexpected second plan: %+v", plans[1]) } if plans[1].Cases[0].Name != "k3d fixture app smoke" { diff --git a/docs/case-reference.md b/docs/case-reference.md index 31bef512..58261044 100644 --- a/docs/case-reference.md +++ b/docs/case-reference.md @@ -28,9 +28,9 @@ oats-schema-version: 3 name: rolldice traces have route attribute fixture: - type: compose - template: lgtm - compose_file: docker-compose.oats.yml + compose: + template: lgtm + file: docker-compose.oats.yml seed: type: app @@ -55,23 +55,43 @@ expected: ## Fixtures A fixture is the observability stack a case runs against — Grafana + Loki + -Tempo + Prometheus + Pyroscope, optionally plus the app under test. `fixture.type` -selects how it is stood up: +Tempo + Prometheus + Pyroscope, optionally plus the app under test. Set exactly +one of three nested blocks; the block you set selects how the stack is stood up +(there is no separate `type` field): -| Type | Meaning | -|------|---------| +| Block | Meaning | +|-------|---------| | `remote` | point at an already-running stack (`endpoint:` / a gcx context) | | `compose` | OATS boots a docker-compose stack; `template: lgtm` lets OATS own the LGTM ports | | `k3d` | OATS boots a k3d (k3s-in-docker) cluster | +```yaml +fixture: + remote: + endpoint: http://localhost:4318 +--- +fixture: + compose: + template: lgtm + file: docker-compose.oats.yml # or files: [a.yml, b.yml] +--- +fixture: + k3d: + k8s_dir: k8s + app_service: rolldice + app_docker_file: Dockerfile + app_port: 8080 +``` + `compose` and `k3d` fixtures are booted, waited on for readiness, and torn down by OATS. `remote` fixtures are assumed ready. For a `compose` fixture driven by a `seed: app` case, set `app_service` (the -compose service name of the app) and `app_port` (the app's container port). OATS -then discovers the host port docker published for that service, so the app can -bind an **ephemeral** host port (`127.0.0.1::`) rather than a fixed one -— which is what lets app-seed suites run under `--parallel` without colliding. +compose service name of the app) and `app_port` (the app's container port) inside +the `compose` block. OATS then discovers the host port docker published for that +service, so the app can bind an **ephemeral** host port (`127.0.0.1::`) +rather than a fixed one — which is what lets app-seed suites run under `--parallel` +without colliding. Omit them and the app is driven on the fixed `--app-port` (default 8080), which forces the suite to run serially. diff --git a/examples/fixtures/oats.toml b/examples/fixtures/oats.toml index cdc142ba..e3ab4a85 100644 --- a/examples/fixtures/oats.toml +++ b/examples/fixtures/oats.toml @@ -13,13 +13,11 @@ cases = ["cases/k3d/*.yaml"] fixture = "k3d-dev" tags = ["k3d", "app"] -[fixture.compose-dev] -type = "compose" -compose_files = ["docker-compose.yml", "docker-compose.override.yml"] +[fixture.compose-dev.compose] +files = ["docker-compose.yml", "docker-compose.override.yml"] env = ["OTEL_EXPORTER_OTLP_ENDPOINT=http://lgtm:4318"] -[fixture.k3d-dev] -type = "k3d" +[fixture.k3d-dev.k3d] k8s_dir = "k8s" app_service = "rolldice" app_docker_file = "Dockerfile" diff --git a/examples/smoke/oats.toml b/examples/smoke/oats.toml index 16c82f12..4c8b105e 100644 --- a/examples/smoke/oats.toml +++ b/examples/smoke/oats.toml @@ -7,6 +7,5 @@ cases = ["cases/*.yaml"] fixture = "local-lgtm" tags = ["traces", "logs", "metrics"] -[fixture.local-lgtm] -type = "remote" +[fixture.local-lgtm.remote] endpoint = "http://localhost:4318" diff --git a/fixture/compose.go b/fixture/compose.go index dec48f0a..52861aba 100644 --- a/fixture/compose.go +++ b/fixture/compose.go @@ -9,16 +9,18 @@ import ( "strings" "time" + "github.com/grafana/oats/casefile" "github.com/grafana/oats/discovery" ) func startCompose(plan discovery.Plan) (Handle, Runtime, error) { - composeFiles, cleanup, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) + compose := plan.Fixture.Compose + composeFiles, cleanup, err := resolveComposeFiles(plan.FixtureSourceDir, compose) if err != nil { return nil, Runtime{}, err } project := composeProjectName(plan) - suiteEnv := append([]string(nil), plan.Fixture.Env...) + suiteEnv := append([]string(nil), compose.Env...) suiteEnv = append(suiteEnv, "COMPOSE_PROJECT_NAME="+project) suite, err := newComposeSuite(composeFiles, suiteEnv) if err != nil { @@ -33,29 +35,25 @@ func startCompose(plan discovery.Plan) (Handle, Runtime, error) { } return nil, Runtime{}, err } - grafanaPort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "3000") - if err != nil { + // fail tears the started suite (and any file cleanup) down before returning. + fail := func(err error) (Handle, Runtime, error) { _ = suite.Close() if cleanup != nil { _ = cleanup() } return nil, Runtime{}, err } + grafanaPort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "3000") + if err != nil { + return fail(err) + } otlpPort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "4318") if err != nil { - _ = suite.Close() - if cleanup != nil { - _ = cleanup() - } - return nil, Runtime{}, err + return fail(err) } pyroscopePort, err := lookupComposePort(composeFiles, suiteEnv, "lgtm", "4040") if err != nil { - _ = suite.Close() - if cleanup != nil { - _ = cleanup() - } - return nil, Runtime{}, err + return fail(err) } rt := Runtime{ GrafanaURL: "http://127.0.0.1:" + grafanaPort, @@ -64,34 +62,24 @@ func startCompose(plan discovery.Plan) (Handle, Runtime, error) { ComposeFiles: composeFiles, ComposeProject: project, } - // When the fixture names an app service, resolve the port docker published - // for it. This lets the app bind an ephemeral host port (127.0.0.1::) - // instead of a fixed one, which is what makes app-seed suites parallel-safe. - if plan.Fixture.AppService != "" && plan.Fixture.AppPort > 0 { - appPort, portErr := lookupComposePort(composeFiles, suiteEnv, plan.Fixture.AppService, strconv.Itoa(plan.Fixture.AppPort)) + // When the fixture manages the app (app_service + app_port), resolve the host + // port docker published for it. This lets the app bind an ephemeral host port + // (127.0.0.1::) instead of a fixed one, which is what makes app-seed + // suites parallel-safe. + if plan.Fixture.HasManagedApp() { + appPort, portErr := lookupComposePort(composeFiles, suiteEnv, compose.AppService, strconv.Itoa(compose.AppPort)) if portErr != nil { - _ = suite.Close() - if cleanup != nil { - _ = cleanup() - } - return nil, Runtime{}, fmt.Errorf("resolve app host port for service %q: %w", plan.Fixture.AppService, portErr) + return fail(fmt.Errorf("resolve app host port for service %q: %w", compose.AppService, portErr)) } p, convErr := strconv.Atoi(appPort) if convErr != nil { - _ = suite.Close() - if cleanup != nil { - _ = cleanup() - } - return nil, Runtime{}, fmt.Errorf("invalid app host port %q for service %q: %w", appPort, plan.Fixture.AppService, convErr) + return fail(fmt.Errorf("invalid app host port %q for service %q: %w", appPort, compose.AppService, convErr)) } rt.AppHostPort = p } cfg, cfgErr := writeLocalGCXConfig(rt.GrafanaURL) if cfgErr != nil { - if cleanup != nil { - _ = cleanup() - } - return nil, Runtime{}, fmt.Errorf("write local gcx config: %w", cfgErr) + return fail(fmt.Errorf("write local gcx config: %w", cfgErr)) } rt.GCXConfig = cfg cleanup = chainCleanup(func() error { return removeIfExists(cfg) }, cleanup) @@ -100,28 +88,28 @@ func startCompose(plan discovery.Plan) (Handle, Runtime, error) { return composeFixture{suite: suite, cleanup: cleanup}, rt, nil } -func resolveComposeFiles(sourceDir string, fixture discovery.FixtureConfig) ([]string, func() error, error) { +func resolveComposeFiles(sourceDir string, compose *casefile.ComposeFixture) ([]string, func() error, error) { var files []string var cleanup func() error - if fixture.Template == "lgtm" { + if compose.Template == "lgtm" { f, err := writeBuiltinLGTMCompose(sourceDir) if err != nil { return nil, nil, err } files = append(files, f) cleanup = func() error { return os.Remove(f) } - } else if fixture.Template != "" { - return nil, nil, fmt.Errorf("unsupported compose fixture template %q", fixture.Template) + } else if compose.Template != "" { + return nil, nil, fmt.Errorf("unsupported compose fixture template %q", compose.Template) } switch { - case fixture.ComposeFile != "": - files = append(files, filepath.Join(sourceDir, fixture.ComposeFile)) - case len(fixture.ComposeFiles) > 0: - for _, file := range fixture.ComposeFiles { + case compose.File != "": + files = append(files, filepath.Join(sourceDir, compose.File)) + case len(compose.Files) > 0: + for _, file := range compose.Files { files = append(files, filepath.Join(sourceDir, file)) } - case fixture.Template == "": - return nil, nil, fmt.Errorf("compose fixture requires compose_file, compose_files, or supported template") + case compose.Template == "": + return nil, nil, fmt.Errorf("compose fixture requires file, files, or supported template") } return files, cleanup, nil } @@ -211,12 +199,13 @@ func composeProjectName(plan discovery.Plan) string { } func extraComposeFiles(plan discovery.Plan) []string { + compose := plan.Fixture.Compose switch { - case plan.Fixture.ComposeFile != "": - return []string{filepath.Join(plan.FixtureSourceDir, plan.Fixture.ComposeFile)} - case len(plan.Fixture.ComposeFiles) > 0: - files := make([]string, 0, len(plan.Fixture.ComposeFiles)) - for _, file := range plan.Fixture.ComposeFiles { + case compose.File != "": + return []string{filepath.Join(plan.FixtureSourceDir, compose.File)} + case len(compose.Files) > 0: + files := make([]string, 0, len(compose.Files)) + for _, file := range compose.Files { files = append(files, filepath.Join(plan.FixtureSourceDir, file)) } return files @@ -287,7 +276,8 @@ func fixedShortPortMapping(value string) bool { } func readComposeGrafanaToken(plan discovery.Plan) (string, error) { - files, _, err := resolveComposeFiles(plan.FixtureSourceDir, plan.Fixture) + compose := plan.Fixture.Compose + files, _, err := resolveComposeFiles(plan.FixtureSourceDir, compose) if err != nil { return "", err } @@ -297,7 +287,7 @@ func readComposeGrafanaToken(plan discovery.Plan) (string, error) { } args = append(args, "exec", "-T", "lgtm", "sh", "-c", "cat /tmp/grafana-sa-token 2>/dev/null || true") cmd := exec.Command("docker", args...) - cmd.Env = append(cmd.Environ(), plan.Fixture.Env...) + cmd.Env = append(cmd.Environ(), compose.Env...) out, err := cmd.Output() if err != nil { return "", err diff --git a/fixture/fixture.go b/fixture/fixture.go index 96a16ca7..f9eb16d0 100644 --- a/fixture/fixture.go +++ b/fixture/fixture.go @@ -31,14 +31,15 @@ var ( if sourceDir == "" { sourceDir = "." } + k3d := plan.Fixture.K3D model := &kubernetes.Kubernetes{ - Dir: plan.Fixture.K8sDir, - AppService: plan.Fixture.AppService, - AppDockerFile: plan.Fixture.AppDockerFile, - AppDockerContext: plan.Fixture.AppDockerContext, - AppDockerTag: plan.Fixture.AppDockerTag, - AppDockerPort: plan.Fixture.AppPort, - ImportImages: plan.Fixture.ImportImages, + Dir: k3d.K8sDir, + AppService: k3d.AppService, + AppDockerFile: k3d.AppDockerFile, + AppDockerContext: k3d.AppDockerContext, + AppDockerTag: k3d.AppDockerTag, + AppDockerPort: k3d.AppPort, + ImportImages: k3d.ImportImages, } return kubernetes.NewEndpoint("localhost", model, ports, plan.Suite.Name, sourceDir) } @@ -76,7 +77,7 @@ type startableHandle interface { // teardown plus the resolved Runtime. Remote/empty fixtures need no boot and // return a nil Handle. func Start(ctx context.Context, plan discovery.Plan) (Handle, Runtime, error) { - switch plan.Fixture.Type { + switch plan.Fixture.Kind() { case "", "remote": return nil, Runtime{ParallelSafe: true}, nil case "compose": @@ -84,14 +85,14 @@ func Start(ctx context.Context, plan discovery.Plan) (Handle, Runtime, error) { case "k3d": return startK3D(ctx, plan) default: - return nil, Runtime{}, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) + return nil, Runtime{}, fmt.Errorf("fixture kind %q is not supported in oats", plan.Fixture.Kind()) } } // WaitForReady blocks until the fixture's Grafana and OTLP endpoints answer, or // the per-endpoint timeout elapses. Remote/empty fixtures are assumed ready. func WaitForReady(plan discovery.Plan, rt Runtime) error { - switch plan.Fixture.Type { + switch plan.Fixture.Kind() { case "compose", "k3d": if err := waitForHTTP(rt.GrafanaURL+"/api/health", 2*time.Minute); err != nil { return fmt.Errorf("wait for grafana: %w", err) @@ -106,11 +107,11 @@ func WaitForReady(plan discovery.Plan, rt Runtime) error { // SupportsParallel reports whether a suite on this fixture can run alongside // other suites, and if not, a human-readable reason. func SupportsParallel(plan discovery.Plan) (bool, string) { - switch plan.Fixture.Type { + switch plan.Fixture.Kind() { case "", "remote": return true, "" case "compose": - if plan.Fixture.Template != "lgtm" { + if plan.Fixture.Compose.Template != "lgtm" { return false, "compose fixtures are only parallel-safe when OATS owns the LGTM ports via template=lgtm" } for _, c := range plan.Cases { @@ -119,7 +120,7 @@ func SupportsParallel(plan discovery.Plan) (bool, string) { // fixture.app_service (+ app_port) so the published port can be // discovered. Without it the app falls back to the fixed --app-port // and parallel suites would collide. - if c.Seed.Type == "app" && (plan.Fixture.AppService == "" || plan.Fixture.AppPort == 0) { + if c.Seed.Type == "app" && !plan.Fixture.HasManagedApp() { return false, "compose app-seed suites need fixture.app_service and app_port so OATS can publish an ephemeral app port; otherwise they share a fixed app port" } } @@ -244,7 +245,7 @@ func waitForGrafanaTokenImpl(plan discovery.Plan) (string, error) { for time.Now().Before(deadline) { var token string var err error - switch plan.Fixture.Type { + switch plan.Fixture.Kind() { case "compose": token, err = readComposeGrafanaToken(plan) case "k3d": diff --git a/fixture/fixture_test.go b/fixture/fixture_test.go index f449970f..0f76bb15 100644 --- a/fixture/fixture_test.go +++ b/fixture/fixture_test.go @@ -15,29 +15,29 @@ import ( ) func TestResolveComposeFiles(t *testing.T) { - got, cleanup, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFile: "stack/compose.yml"}) + got, cleanup, err := resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{File: "stack/compose.yml"}) if err != nil { - t.Fatalf("resolveComposeFiles compose_file: %v", err) + t.Fatalf("resolveComposeFiles file: %v", err) } if cleanup != nil { - t.Fatalf("unexpected cleanup for compose_file fixture") + t.Fatalf("unexpected cleanup for file fixture") } if want := []string{"/tmp/work/stack/compose.yml"}; len(got) != 1 || got[0] != want[0] { t.Fatalf("got %q want %q", got, want) } - got, cleanup, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", ComposeFiles: []string{"a.yml", "b.yml"}}) + got, cleanup, err = resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{Files: []string{"a.yml", "b.yml"}}) if err != nil { - t.Fatalf("resolveComposeFiles compose_files: %v", err) + t.Fatalf("resolveComposeFiles files: %v", err) } if cleanup != nil { - t.Fatalf("unexpected cleanup for compose_files fixture") + t.Fatalf("unexpected cleanup for files fixture") } if len(got) != 2 || got[0] != "/tmp/work/a.yml" || got[1] != "/tmp/work/b.yml" { - t.Fatalf("unexpected compose_files resolution: %v", got) + t.Fatalf("unexpected files resolution: %v", got) } - got, cleanup, err = resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "lgtm"}) + got, cleanup, err = resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{Template: "lgtm"}) if err != nil { t.Fatalf("resolveComposeFiles template=lgtm: %v", err) } @@ -78,10 +78,11 @@ func TestStart_ComposeLifecycle(t *testing.T) { fix, _, err := Start(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, - Fixture: discovery.FixtureConfig{ - Type: "compose", - ComposeFiles: []string{"a.yml", "b.yml"}, - Env: []string{"FOO=bar"}, + Fixture: casefile.FixtureConfig{ + Compose: &casefile.ComposeFixture{ + Files: []string{"a.yml", "b.yml"}, + Env: []string{"FOO=bar"}, + }, }, FixtureSourceDir: "/tmp/work", }) @@ -115,7 +116,7 @@ func TestStart_ComposeStartFailure(t *testing.T) { _, _, err := Start(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, - Fixture: discovery.FixtureConfig{Type: "compose", ComposeFile: "compose.yml"}, + Fixture: casefile.FixtureConfig{Compose: &casefile.ComposeFixture{File: "compose.yml"}}, FixtureSourceDir: "/tmp/work", }) if err == nil || !strings.Contains(err.Error(), "boom") { @@ -142,27 +143,31 @@ func TestStart_K3DLifecycle(t *testing.T) { }, nil) } - fix, _, err := Start(context.Background(), discovery.Plan{ + fix, rt, err := Start(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, - Fixture: discovery.FixtureConfig{ - Type: "k3d", - K8sDir: "k8s", - AppService: "dice", - AppDockerFile: "Dockerfile", - AppDockerContext: ".", - AppDockerTag: "dice:test", - AppPort: 18080, - ImportImages: []string{"busybox:latest"}, + Fixture: casefile.FixtureConfig{ + K3D: &casefile.K3DFixture{ + K8sDir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerContext: ".", + AppDockerTag: "dice:test", + AppPort: 18080, + ImportImages: []string{"busybox:latest"}, + }, }, FixtureSourceDir: "/tmp/work", }) if err != nil { t.Fatalf("Start k3d: %v", err) } + if rt.AppHostPort != 18080 { + t.Fatalf("expected Runtime.AppHostPort=18080 from fixture config, got %d", rt.AppHostPort) + } if starts != 1 { t.Fatalf("expected one endpoint start, got %d", starts) } - if capturedPlan.FixtureSourceDir != "/tmp/work" || capturedPlan.Suite.Name != "cluster-smoke" || capturedPlan.Fixture.AppPort != 18080 { + if capturedPlan.FixtureSourceDir != "/tmp/work" || capturedPlan.Suite.Name != "cluster-smoke" || capturedPlan.Fixture.K3D.AppPort != 18080 { t.Fatalf("unexpected endpoint factory args: plan=%+v", capturedPlan) } if capturedPorts.GrafanaHTTPPort == 0 || capturedPorts.OTLPHTTPPort == 0 || capturedPorts.LokiHttpPort == 0 || capturedPorts.PrometheusHTTPPort == 0 || capturedPorts.TempoHTTPPort == 0 || capturedPorts.PyroscopeHttpPort == 0 { @@ -190,14 +195,15 @@ func TestStart_K3DStartFailure(t *testing.T) { _, _, err := Start(context.Background(), discovery.Plan{ Suite: discovery.SuiteConfig{Name: "cluster-smoke", Fixture: "cluster"}, - Fixture: discovery.FixtureConfig{ - Type: "k3d", - K8sDir: "k8s", - AppService: "dice", - AppDockerFile: "Dockerfile", - AppDockerContext: ".", - AppDockerTag: "dice:test", - AppPort: 18080, + Fixture: casefile.FixtureConfig{ + K3D: &casefile.K3DFixture{ + K8sDir: "k8s", + AppService: "dice", + AppDockerFile: "Dockerfile", + AppDockerContext: ".", + AppDockerTag: "dice:test", + AppPort: 18080, + }, }, FixtureSourceDir: "/tmp/work", }) @@ -225,15 +231,15 @@ func TestK3DCheckEnv_UsesConfiguredPorts(t *testing.T) { } func TestResolveComposeFiles_UnsupportedTemplate(t *testing.T) { - _, _, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose", Template: "weird"}) + _, _, err := resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{Template: "weird"}) if err == nil || !strings.Contains(err.Error(), `unsupported compose fixture template "weird"`) { t.Fatalf("expected unsupported template error, got %v", err) } } func TestResolveComposeFiles_MissingConfig(t *testing.T) { - _, _, err := resolveComposeFiles("/tmp/work", discovery.FixtureConfig{Type: "compose"}) - if err == nil || !strings.Contains(err.Error(), "compose fixture requires compose_file, compose_files, or supported template") { + _, _, err := resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{}) + if err == nil || !strings.Contains(err.Error(), "compose fixture requires file, files, or supported template") { t.Fatalf("expected missing compose config error, got %v", err) } } @@ -281,10 +287,9 @@ name = "parallel-safe" cases = ["cases/*.yaml"] fixture = "stack" -[fixture.stack] -type = "compose" +[fixture.stack.compose] template = "lgtm" -compose_file = "docker-compose.oats.yml" +file = "docker-compose.oats.yml" `) writeFile(t, dir, "docker-compose.oats.yml", `services: app: @@ -327,7 +332,7 @@ func TestSupportsParallel_AppSeedRequiresAppService(t *testing.T) { // app seed, no app_service → not parallel-safe. noService := discovery.Plan{ - Fixture: discovery.FixtureConfig{Type: "compose", Template: "lgtm"}, + Fixture: casefile.FixtureConfig{Compose: &casefile.ComposeFixture{Template: "lgtm"}}, Cases: []*casefile.Case{appCase}, } if safe, reason := SupportsParallel(noService); safe { @@ -336,7 +341,7 @@ func TestSupportsParallel_AppSeedRequiresAppService(t *testing.T) { // app seed with app_service + app_port → parallel-safe. withService := discovery.Plan{ - Fixture: discovery.FixtureConfig{Type: "compose", Template: "lgtm", AppService: "app", AppPort: 8080}, + Fixture: casefile.FixtureConfig{Compose: &casefile.ComposeFixture{Template: "lgtm", AppService: "app", AppPort: 8080}}, Cases: []*casefile.Case{appCase}, } if safe, reason := SupportsParallel(withService); !safe { @@ -349,7 +354,7 @@ func TestSupportsParallel_AppSeedRequiresAppService(t *testing.T) { // empty token and no error. func TestWaitForGrafanaToken_DefaultFixtureReturnsEmpty(t *testing.T) { token, err := waitForGrafanaToken(discovery.Plan{ - Fixture: discovery.FixtureConfig{Type: "remote"}, + Fixture: casefile.FixtureConfig{Remote: &casefile.RemoteFixture{}}, }) if err != nil { t.Fatalf("waitForGrafanaToken: %v", err) diff --git a/fixture/k3d.go b/fixture/k3d.go index f3310b43..bda05331 100644 --- a/fixture/k3d.go +++ b/fixture/k3d.go @@ -19,11 +19,13 @@ func startK3D(ctx context.Context, plan discovery.Plan) (Handle, Runtime, error) if err := ep.Start(ctx); err != nil { return nil, Runtime{}, err } + appPort := plan.Fixture.K3D.AppPort rt := Runtime{ GrafanaURL: fmt.Sprintf("http://localhost:%d", ports.GrafanaHTTPPort), OTLPHTTP: fmt.Sprintf("http://localhost:%d", ports.OTLPHTTPPort), PyroscopeURL: fmt.Sprintf("http://localhost:%d", ports.PyroscopeHttpPort), - CustomCheckEnv: k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: plan.Fixture.AppPort}, ports), + AppHostPort: appPort, + CustomCheckEnv: k3dCheckEnv(runner.Endpoint{AppHost: "localhost", AppPort: appPort}, ports), ParallelSafe: false, ParallelDisabled: "k3d fixtures currently use shared clusters and kubectl port-forwards", } diff --git a/internal/cli/integration_test.go b/internal/cli/integration_test.go index 88bb6350..cd909a56 100644 --- a/internal/cli/integration_test.go +++ b/internal/cli/integration_test.go @@ -14,6 +14,7 @@ import ( "testing" "time" + "github.com/grafana/oats/casefile" "github.com/grafana/oats/discovery" "github.com/grafana/oats/engine" "github.com/grafana/oats/fixture" @@ -25,7 +26,7 @@ import ( func TestResolveEndpoint_ComposeDefaults(t *testing.T) { ep, err := resolveEndpoint(discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, - Fixture: discovery.FixtureConfig{Type: "compose", Template: "lgtm"}, + Fixture: casefile.FixtureConfig{Compose: &casefile.ComposeFixture{Template: "lgtm"}}, }, fixture.Runtime{GCXConfig: "/tmp/gcx.yaml", OTLPHTTP: "http://127.0.0.1:4318"}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) @@ -35,11 +36,14 @@ func TestResolveEndpoint_ComposeDefaults(t *testing.T) { } } -func TestResolveEndpoint_K3DUsesFixtureAppPort(t *testing.T) { +func TestResolveEndpoint_UsesRuntimeAppHostPort(t *testing.T) { + // The resolved app host port is surfaced uniformly through Runtime (compose + // discovers an ephemeral port, k3d copies its configured one), and + // resolveEndpoint applies it regardless of fixture type. ep, err := resolveEndpoint(discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "cluster"}, - Fixture: discovery.FixtureConfig{Type: "k3d", AppPort: 18080}, - }, fixture.Runtime{GCXConfig: "/tmp/gcx.yaml", OTLPHTTP: "http://127.0.0.1:4318"}, "", "localhost", 8080, "http://localhost:4318") + Fixture: casefile.FixtureConfig{K3D: &casefile.K3DFixture{AppPort: 18080}}, + }, fixture.Runtime{GCXConfig: "/tmp/gcx.yaml", OTLPHTTP: "http://127.0.0.1:4318", AppHostPort: 18080}, "", "localhost", 8080, "http://localhost:4318") if err != nil { t.Fatalf("resolveEndpoint: %v", err) } @@ -53,7 +57,7 @@ func TestCloseFixture_EmitsTeardownEvent(t *testing.T) { fix := &fakeSuiteFixture{} plan := discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, - Fixture: discovery.FixtureConfig{Type: "compose"}, + Fixture: casefile.FixtureConfig{Compose: &casefile.ComposeFixture{}}, } if err := closeFixture(rep, plan, fix); err != nil { t.Fatalf("closeFixture: %v", err) @@ -74,7 +78,7 @@ func TestCloseFixture_RemoteDoesNotEmitTeardownEvent(t *testing.T) { fix := &fakeSuiteFixture{} plan := discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "remote-lgtm"}, - Fixture: discovery.FixtureConfig{Type: "remote"}, + Fixture: casefile.FixtureConfig{Remote: &casefile.RemoteFixture{}}, } if err := closeFixture(rep, plan, fix); err != nil { t.Fatalf("closeFixture: %v", err) @@ -91,7 +95,7 @@ func TestEmitFixtureStartAndReady(t *testing.T) { rep := &recordingReporter{} plan := discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, - Fixture: discovery.FixtureConfig{Type: "compose"}, + Fixture: casefile.FixtureConfig{Compose: &casefile.ComposeFixture{}}, } start := emitFixtureStart(rep, plan) if start.IsZero() { @@ -117,7 +121,7 @@ func TestEmitFixtureStartAndReady_NoOpForRemote(t *testing.T) { rep := &recordingReporter{} plan := discovery.Plan{ Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "remote-lgtm"}, - Fixture: discovery.FixtureConfig{Type: "remote"}, + Fixture: casefile.FixtureConfig{Remote: &casefile.RemoteFixture{}}, } start := emitFixtureStart(rep, plan) if start.IsZero() { @@ -150,8 +154,7 @@ name = "smoke" cases = ["cases/*.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "REPLACED_AT_RUNTIME" `) writeFile(t, dir, "cases/inline.yaml", `oats-schema-version: 3 @@ -273,8 +276,7 @@ name = "smoke" cases = ["cases/*.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "http://localhost:4318" `) writeFile(t, dir, "cases/app.yaml", `oats-schema-version: 3 @@ -360,8 +362,7 @@ name = "profiles" cases = ["cases/*.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "http://localhost:4318" `) writeFile(t, dir, "cases/profile.yaml", `oats-schema-version: 3 @@ -449,8 +450,7 @@ name = "migrated-profile" cases = ["cases/*.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) @@ -537,8 +537,7 @@ name = "migrated" cases = ["cases/*.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) @@ -616,8 +615,7 @@ name = "migrated-custom" cases = ["cases/*.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) @@ -690,8 +688,7 @@ name = "migrated-custom-path" cases = ["cases/*.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) @@ -786,8 +783,7 @@ name = "migrated-matrix" cases = ["cases/*.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) diff --git a/internal/cli/main.go b/internal/cli/main.go index f176e823..9b032a96 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -347,13 +347,13 @@ func verbosityFromInt(n int) report.Verbosity { // concrete endpoint the runner needs. func resolveEndpoint(plan discovery.Plan, rt fixture.Runtime, gcxContextOverride, appHost string, appPort int, otlpHTTP string) (runner.Endpoint, error) { ep := runner.Endpoint{AppHost: appHost, AppPort: appPort, OTLPHTTP: otlpHTTP} - switch plan.Fixture.Type { + switch plan.Fixture.Kind() { case "remote": // For a remote fixture, the gcx context is configured externally // (e.g. `gcx login` already ran). We pass the fixture name as a // best-effort default; --gcx-context overrides. ep.GCXContext = plan.Suite.Fixture - ep.OTLPHTTP = plan.Fixture.Endpoint + ep.OTLPHTTP = plan.Fixture.Remote.Endpoint ep.CustomCheckEnv = append(ep.CustomCheckEnv, "OATS_FIXTURE_TYPE=remote", "OATS_GRAFANA_URL="+grafanaURL(), @@ -366,9 +366,6 @@ func resolveEndpoint(plan discovery.Plan, rt fixture.Runtime, gcxContextOverride if rt.OTLPHTTP != "" { ep.OTLPHTTP = rt.OTLPHTTP } - if rt.AppHostPort > 0 { - ep.AppPort = rt.AppHostPort - } ep.CustomCheckEnv = append(ep.CustomCheckEnv, rt.CustomCheckEnv...) case "k3d": if rt.GCXConfig != "" { @@ -377,9 +374,6 @@ func resolveEndpoint(plan discovery.Plan, rt fixture.Runtime, gcxContextOverride if rt.OTLPHTTP != "" { ep.OTLPHTTP = rt.OTLPHTTP } - if plan.Fixture.AppPort > 0 { - ep.AppPort = plan.Fixture.AppPort - } ep.CustomCheckEnv = append(ep.CustomCheckEnv, rt.CustomCheckEnv...) case "": // No fixture configured — caller (or --gcx-context) must supply @@ -388,7 +382,14 @@ func resolveEndpoint(plan discovery.Plan, rt fixture.Runtime, gcxContextOverride "OATS_APP_URL="+fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), ) default: - return ep, fmt.Errorf("fixture type %q is not supported in oats", plan.Fixture.Type) + return ep, fmt.Errorf("fixture kind %q is not supported in oats", plan.Fixture.Kind()) + } + // A fixture that resolved a concrete app host port drives the app there — + // compose discovers the published ephemeral port, k3d uses the configured + // one. Surfaced uniformly through Runtime so every fixture type reads it the + // same way. + if rt.AppHostPort > 0 { + ep.AppPort = rt.AppHostPort } if gcxContextOverride != "" { ep.GCXContext = gcxContextOverride @@ -542,7 +543,7 @@ func runPlan(ctx context.Context, rep report.Reporter, plan discovery.Plan, opts Type: report.EventSuiteStart, Suite: plan.Suite.Name, Fixture: plan.Suite.Fixture, - FixtureType: plan.Fixture.Type, + FixtureType: plan.Fixture.Kind(), CaseCount: len(plan.Cases), }) @@ -599,12 +600,12 @@ func runPlan(ctx context.Context, rep report.Reporter, plan discovery.Plan, opts func emitFixtureStart(rep report.Reporter, plan discovery.Plan) time.Time { start := time.Now() - if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { + if plan.Fixture.Kind() != "" && plan.Fixture.Kind() != "remote" { rep.Emit(report.Event{ Type: report.EventFixtureStart, Suite: plan.Suite.Name, Fixture: plan.Suite.Fixture, - FixtureType: plan.Fixture.Type, + FixtureType: plan.Fixture.Kind(), Ts: start, }) } @@ -612,12 +613,12 @@ func emitFixtureStart(rep report.Reporter, plan discovery.Plan) time.Time { } func emitFixtureReady(rep report.Reporter, plan discovery.Plan, start time.Time) { - if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { + if plan.Fixture.Kind() != "" && plan.Fixture.Kind() != "remote" { rep.Emit(report.Event{ Type: report.EventFixtureReady, Suite: plan.Suite.Name, Fixture: plan.Suite.Fixture, - FixtureType: plan.Fixture.Type, + FixtureType: plan.Fixture.Kind(), DurationMs: time.Since(start).Milliseconds(), }) } @@ -628,12 +629,12 @@ func closeFixture(rep report.Reporter, plan discovery.Plan, fix fixture.Handle) if err := fix.Close(); err != nil { return err } - if plan.Fixture.Type != "" && plan.Fixture.Type != "remote" { + if plan.Fixture.Kind() != "" && plan.Fixture.Kind() != "remote" { rep.Emit(report.Event{ Type: report.EventFixtureTeardown, Suite: plan.Suite.Name, Fixture: plan.Suite.Fixture, - FixtureType: plan.Fixture.Type, + FixtureType: plan.Fixture.Kind(), DurationMs: time.Since(start).Milliseconds(), }) } diff --git a/migrate/migrate.go b/migrate/migrate.go index 26d66f6d..b841f94b 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -266,12 +266,11 @@ func composeFixtureHint(name string, def model.TestCaseDefinition) string { fixtureName := slug(name) var b strings.Builder fmt.Fprintf(&b, "suggested oats.toml fixture snippet for %q:\n", name) - fmt.Fprintf(&b, "[fixture.%s]\n", fixtureName) - fmt.Fprintf(&b, "type = \"compose\"\n") + fmt.Fprintf(&b, "[fixture.%s.compose]\n", fixtureName) if len(def.DockerCompose.Files) == 1 { - fmt.Fprintf(&b, "compose_file = %q\n", def.DockerCompose.Files[0]) + fmt.Fprintf(&b, "file = %q\n", def.DockerCompose.Files[0]) } else if len(def.DockerCompose.Files) > 1 { - fmt.Fprintf(&b, "compose_files = [%s]\n", quotedList(def.DockerCompose.Files)) + fmt.Fprintf(&b, "files = [%s]\n", quotedList(def.DockerCompose.Files)) } if len(def.DockerCompose.Environment) > 0 { fmt.Fprintf(&b, "env = [%s]\n", quotedList(def.DockerCompose.Environment)) @@ -284,8 +283,7 @@ func kubernetesFixtureHint(name string, def model.TestCaseDefinition) string { k := def.Kubernetes var b strings.Builder fmt.Fprintf(&b, "suggested oats.toml fixture snippet for %q:\n", name) - fmt.Fprintf(&b, "[fixture.%s]\n", fixtureName) - fmt.Fprintf(&b, "type = \"k3d\"\n") + fmt.Fprintf(&b, "[fixture.%s.k3d]\n", fixtureName) fmt.Fprintf(&b, "k8s_dir = %q\n", k.Dir) fmt.Fprintf(&b, "app_service = %q\n", k.AppService) fmt.Fprintf(&b, "app_docker_file = %q\n", k.AppDockerFile) diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 9a081837..d50e0377 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -126,8 +126,8 @@ func TestConvertFile_MatrixSampleIsParseable(t *testing.T) { for _, want := range []string{ `matrix definitions are not migrated automatically when multiple entries exist`, `suggested matrix expansion for "matrix test.oats"`, - `[fixture.matrix-test-oats-docker]`, - `[fixture.matrix-test-oats-k8s]`, + `[fixture.matrix-test-oats-docker.compose]`, + `[fixture.matrix-test-oats-k8s.k3d]`, } { if !strings.Contains(joined, want) { fatalf(t, "expected matrix warning to contain %q:\n%s", want, joined) @@ -212,8 +212,7 @@ func TestConvertDefinition_KubernetesProducesFixtureHint(t *testing.T) { } joined := strings.Join(warnings, "\n") for _, want := range []string{ - `[fixture.k8s-case]`, - `type = "k3d"`, + `[fixture.k8s-case.k3d]`, `k8s_dir = "k8s"`, `app_service = "dice"`, `app_docker_file = "Dockerfile"`, @@ -279,8 +278,8 @@ func TestConvertDefinition_FlattensSingleMatrixEntry(t *testing.T) { joined := strings.Join(warnings, "\n") for _, want := range []string{ `flattened single matrix entry "docker"`, - `[fixture.matrix-case-docker]`, - `compose_file = "docker-compose.yml"`, + `[fixture.matrix-case-docker.compose]`, + `file = "docker-compose.yml"`, } { if !strings.Contains(joined, want) { fatalf(t, "expected warning to contain %q:\n%s", want, joined) @@ -327,10 +326,9 @@ func TestConvertDefinition_MultiMatrixEmitsExpansionHint(t *testing.T) { for _, want := range []string{ `suggested matrix expansion for "matrix case"`, `- docker`, - `[fixture.matrix-case-docker]`, + `[fixture.matrix-case-docker.compose]`, `- k8s`, - `[fixture.matrix-case-k8s]`, - `type = "k3d"`, + `[fixture.matrix-case-k8s.k3d]`, } { if !strings.Contains(joined, want) { fatalf(t, "expected warning to contain %q:\n%s", want, joined) diff --git a/tests/e2e/cases/assert/absent-fail/files/oats.yaml b/tests/e2e/cases/assert/absent-fail/files/oats.yaml index 754ac696..6ac71bc2 100644 --- a/tests/e2e/cases/assert/absent-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/absent-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: absent-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/absent/files/oats.yaml b/tests/e2e/cases/assert/absent/files/oats.yaml index 4eb62d73..260ec7b3 100644 --- a/tests/e2e/cases/assert/absent/files/oats.yaml +++ b/tests/e2e/cases/assert/absent/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: absent fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/contains-fail/files/oats.yaml b/tests/e2e/cases/assert/contains-fail/files/oats.yaml index 3efd76b8..7617dbc4 100644 --- a/tests/e2e/cases/assert/contains-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/contains-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: contains-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/contains/files/oats.yaml b/tests/e2e/cases/assert/contains/files/oats.yaml index 600e1615..816e489f 100644 --- a/tests/e2e/cases/assert/contains/files/oats.yaml +++ b/tests/e2e/cases/assert/contains/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: contains fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/count-fail/files/oats.yaml b/tests/e2e/cases/assert/count-fail/files/oats.yaml index a4f66222..c4539ba1 100644 --- a/tests/e2e/cases/assert/count-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/count-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: count-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/count/files/oats.yaml b/tests/e2e/cases/assert/count/files/oats.yaml index 0bcf6370..74d45554 100644 --- a/tests/e2e/cases/assert/count/files/oats.yaml +++ b/tests/e2e/cases/assert/count/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: count fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/match-fail/files/oats.yaml b/tests/e2e/cases/assert/match-fail/files/oats.yaml index a82c71c7..69be12b1 100644 --- a/tests/e2e/cases/assert/match-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/match-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: match-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml b/tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml index 3f0980ca..272dfb1d 100644 --- a/tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: match-regexp-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/match-regexp/files/oats.yaml b/tests/e2e/cases/assert/match-regexp/files/oats.yaml index 462b2b6a..b8a1a733 100644 --- a/tests/e2e/cases/assert/match-regexp/files/oats.yaml +++ b/tests/e2e/cases/assert/match-regexp/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: match-regexp fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml b/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml index 221ab8a3..7632a522 100644 --- a/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: match-spans-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp traces: diff --git a/tests/e2e/cases/assert/not-contains-fail/files/oats.yaml b/tests/e2e/cases/assert/not-contains-fail/files/oats.yaml index ec982de9..7ecd8fa9 100644 --- a/tests/e2e/cases/assert/not-contains-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/not-contains-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: not-contains-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/not-contains/files/oats.yaml b/tests/e2e/cases/assert/not-contains/files/oats.yaml index 514b00e3..702752cc 100644 --- a/tests/e2e/cases/assert/not-contains/files/oats.yaml +++ b/tests/e2e/cases/assert/not-contains/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: not-contains fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/profile-fail/files/oats.yaml b/tests/e2e/cases/assert/profile-fail/files/oats.yaml index 135051f3..2ea140e9 100644 --- a/tests/e2e/cases/assert/profile-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/profile-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: profile-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: app expected: diff --git a/tests/e2e/cases/assert/profile/files/oats.yaml b/tests/e2e/cases/assert/profile/files/oats.yaml index 6d6eb421..7a0e7c14 100644 --- a/tests/e2e/cases/assert/profile/files/oats.yaml +++ b/tests/e2e/cases/assert/profile/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: profile fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: app expected: diff --git a/tests/e2e/cases/assert/regex-fail/files/oats.yaml b/tests/e2e/cases/assert/regex-fail/files/oats.yaml index b43ed2ac..6e18f146 100644 --- a/tests/e2e/cases/assert/regex-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/regex-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: regex-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/regex/files/oats.yaml b/tests/e2e/cases/assert/regex/files/oats.yaml index fb055975..5fdafe64 100644 --- a/tests/e2e/cases/assert/regex/files/oats.yaml +++ b/tests/e2e/cases/assert/regex/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: regex fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/assert/value-fail/files/oats.yaml b/tests/e2e/cases/assert/value-fail/files/oats.yaml index ad43f26b..75201588 100644 --- a/tests/e2e/cases/assert/value-fail/files/oats.yaml +++ b/tests/e2e/cases/assert/value-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: value-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp metrics: diff --git a/tests/e2e/cases/assert/value/files/oats.yaml b/tests/e2e/cases/assert/value/files/oats.yaml index c82b29c5..c70e6a81 100644 --- a/tests/e2e/cases/assert/value/files/oats.yaml +++ b/tests/e2e/cases/assert/value/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: value fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp metrics: diff --git a/tests/e2e/cases/cli/cache-clear/files/oats.toml b/tests/e2e/cases/cli/cache-clear/files/oats.toml index 4eddddc1..7b99a7d4 100644 --- a/tests/e2e/cases/cli/cache-clear/files/oats.toml +++ b/tests/e2e/cases/cli/cache-clear/files/oats.toml @@ -6,6 +6,5 @@ name = "cache-clear" cases = ["case.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/cache-skip/files/oats.toml b/tests/e2e/cases/cli/cache-skip/files/oats.toml index a208e9e1..43526a06 100644 --- a/tests/e2e/cases/cli/cache-skip/files/oats.toml +++ b/tests/e2e/cases/cli/cache-skip/files/oats.toml @@ -6,6 +6,5 @@ name = "cache" cases = ["case.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/fail-fast/files/oats.toml b/tests/e2e/cases/cli/fail-fast/files/oats.toml index cde32081..6cb2ec20 100644 --- a/tests/e2e/cases/cli/fail-fast/files/oats.toml +++ b/tests/e2e/cases/cli/fail-fast/files/oats.toml @@ -6,6 +6,5 @@ name = "failfast" cases = ["01-fail.yaml", "02-pass.yaml"] fixture = "remote-lgtm" -[fixture.remote-lgtm] -type = "remote" +[fixture.remote-lgtm.remote] endpoint = "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml b/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml index 25ebc4d5..429c3a5e 100644 --- a/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml +++ b/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: custom-check-fail fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/custom/custom-check/files/oats.yaml b/tests/e2e/cases/custom/custom-check/files/oats.yaml index 5656a8cc..897697ae 100644 --- a/tests/e2e/cases/custom/custom-check/files/oats.yaml +++ b/tests/e2e/cases/custom/custom-check/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: custom-check fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml b/tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml index 7abb0d9a..caef6555 100644 --- a/tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml +++ b/tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml @@ -1,9 +1,9 @@ oats-schema-version: 3 name: compose-app-seed fixture: - type: compose - template: lgtm - compose_file: docker-compose.oats.yml + compose: + template: lgtm + file: docker-compose.oats.yml seed: type: app expected: diff --git a/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml b/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml index d17a4e1f..f5fcd78b 100644 --- a/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml +++ b/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml @@ -1,9 +1,9 @@ oats-schema-version: 3 name: compose-logs-fail fixture: - type: compose - template: lgtm - compose_file: docker-compose.oats.yml + compose: + template: lgtm + file: docker-compose.oats.yml seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/fixture/compose-logs/files/oats.yaml b/tests/e2e/cases/fixture/compose-logs/files/oats.yaml index e9bc4e7a..4233f713 100644 --- a/tests/e2e/cases/fixture/compose-logs/files/oats.yaml +++ b/tests/e2e/cases/fixture/compose-logs/files/oats.yaml @@ -1,9 +1,9 @@ oats-schema-version: 3 name: compose-logs fixture: - type: compose - template: lgtm - compose_file: docker-compose.oats.yml + compose: + template: lgtm + file: docker-compose.oats.yml seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml b/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml index 6796f219..710f9ab3 100644 --- a/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml +++ b/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml @@ -1,13 +1,13 @@ oats-schema-version: 3 name: k3d-fail fixture: - type: k3d - k8s_dir: k8s - app_service: dice - app_docker_file: Dockerfile - app_docker_context: . - app_docker_tag: dice:test - app_port: 18080 + k3d: + k8s_dir: k8s + app_service: dice + app_docker_file: Dockerfile + app_docker_context: . + app_docker_tag: dice:test + app_port: 18080 seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml index 424b1e25..eb185e09 100644 --- a/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml +++ b/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml @@ -1,13 +1,13 @@ oats-schema-version: 3 name: k3d-smoke fixture: - type: k3d - k8s_dir: k8s - app_service: dice - app_docker_file: Dockerfile - app_docker_context: . - app_docker_tag: dice:test - app_port: 18080 + k3d: + k8s_dir: k8s + app_service: dice + app_docker_file: Dockerfile + app_docker_context: . + app_docker_tag: dice:test + app_port: 18080 seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/fixture/parallel-compose/files/oats.toml b/tests/e2e/cases/fixture/parallel-compose/files/oats.toml index c52ee17d..ea8ee4bc 100644 --- a/tests/e2e/cases/fixture/parallel-compose/files/oats.toml +++ b/tests/e2e/cases/fixture/parallel-compose/files/oats.toml @@ -11,7 +11,6 @@ name = "beta" cases = ["beta.yaml"] fixture = "shared" -[fixture.shared] -type = "compose" +[fixture.shared.compose] template = "lgtm" -compose_file = "docker-compose.oats.yml" +file = "docker-compose.oats.yml" diff --git a/tests/e2e/cases/fixture/remote-basic/files/oats.yaml b/tests/e2e/cases/fixture/remote-basic/files/oats.yaml index f4486137..1df08e59 100644 --- a/tests/e2e/cases/fixture/remote-basic/files/oats.yaml +++ b/tests/e2e/cases/fixture/remote-basic/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: remote-basic fixture: - type: remote - endpoint: http://localhost:4319 + remote: + endpoint: http://localhost:4319 seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/fixture/remote-basic/test.yaml b/tests/e2e/cases/fixture/remote-basic/test.yaml index 6362d876..bb6aab0a 100644 --- a/tests/e2e/cases/fixture/remote-basic/test.yaml +++ b/tests/e2e/cases/fixture/remote-basic/test.yaml @@ -37,8 +37,8 @@ run: oats-schema-version: 3 name: remote-basic fixture: - type: remote - endpoint: http://127.0.0.1:${otlp_port} + remote: + endpoint: http://127.0.0.1:${otlp_port} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/format/ndjson/files/oats.yaml b/tests/e2e/cases/format/ndjson/files/oats.yaml index 3e419773..e7c07213 100644 --- a/tests/e2e/cases/format/ndjson/files/oats.yaml +++ b/tests/e2e/cases/format/ndjson/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: ndjson fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp logs: diff --git a/tests/e2e/cases/seed/inline-otlp/files/oats.yaml b/tests/e2e/cases/seed/inline-otlp/files/oats.yaml index 7062ef3c..65d5451c 100644 --- a/tests/e2e/cases/seed/inline-otlp/files/oats.yaml +++ b/tests/e2e/cases/seed/inline-otlp/files/oats.yaml @@ -1,8 +1,8 @@ oats-schema-version: 3 name: inline-otlp fixture: - type: remote - endpoint: {{REMOTE_OTLP_HTTP}} + remote: + endpoint: {{REMOTE_OTLP_HTTP}} seed: type: inline-otlp traces: From 3e8571f66d5d5f1156b21c3b529502638d7bc6ba Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 13:32:11 +0000 Subject: [PATCH 112/124] refactor(config): YAML config format, oats-config.yaml + oats-case.yaml Drop TOML for the project config and standardize on YAML everywhere (the case files were already YAML). discovery.Load now parses YAML with strict unknown-key rejection; removed the BurntSushi/toml dependency. The `[[suite]]`/`[fixture.x]` tables become `suites:`/`fixture:` maps. Rename the two file kinds for clarity, both prefixed so they're obvious in a directory of yaml: - project config: oats.toml -> oats-config.yaml (--config default) - lone/default case: -> oats-case.yaml Migrated examples, the e2e case suite + harness, migrate hint output, and docs (README/case-reference/ci/UPGRADING/AGENTS). Regenerated renovate-tracked-deps. Signed-off-by: Gregor Zeitlinger --- .github/renovate-tracked-deps.json | 1 - AGENTS.md | 6 +- README.md | 8 +- UPGRADING.md | 30 +-- casefile/case.go | 46 ++-- discovery/discovery.go | 55 +++-- discovery/discovery_test.go | 133 ++++++------ docs/case-reference.md | 20 +- docs/ci.md | 4 +- examples/fixtures/oats-config.yaml | 25 +++ examples/fixtures/oats.toml | 27 --- examples/smoke/oats-config.yaml | 11 + examples/smoke/oats.toml | 11 - fixture/fixture_test.go | 26 +-- go.mod | 1 - go.sum | 2 - internal/cli/integration_test.go | 200 +++++++++--------- internal/cli/main.go | 8 +- migrate/migrate.go | 40 ++-- migrate/migrate_test.go | 29 +-- tests/e2e/README.md | 8 +- .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../count/files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../regex/files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../value/files/{oats.yaml => oats-case.yaml} | 0 .../files/{case.yaml => oats-case.yaml} | 0 .../cli/cache-clear/files/oats-config.yaml | 10 + .../e2e/cases/cli/cache-clear/files/oats.toml | 10 - tests/e2e/cases/cli/cache-clear/test.yaml | 4 +- .../files/{case.yaml => oats-case.yaml} | 0 .../cli/cache-skip/files/oats-config.yaml | 10 + .../e2e/cases/cli/cache-skip/files/oats.toml | 10 - tests/e2e/cases/cli/cache-skip/test.yaml | 4 +- .../cli/fail-fast/files/oats-config.yaml | 10 + tests/e2e/cases/cli/fail-fast/files/oats.toml | 10 - .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../parallel-compose/files/oats-config.yaml | 14 ++ .../fixture/parallel-compose/files/oats.toml | 16 -- .../files/{oats.yaml => oats-case.yaml} | 0 .../e2e/cases/fixture/remote-basic/test.yaml | 4 +- .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 .../files/{oats.yaml => oats-case.yaml} | 0 tests/e2e/e2e_test.go | 8 +- 64 files changed, 393 insertions(+), 408 deletions(-) create mode 100644 examples/fixtures/oats-config.yaml delete mode 100644 examples/fixtures/oats.toml create mode 100644 examples/smoke/oats-config.yaml delete mode 100644 examples/smoke/oats.toml rename tests/e2e/cases/assert/absent-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/absent/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/contains-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/contains/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/count-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/count/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/match-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/match-regexp-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/match-regexp/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/match-spans-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/not-contains-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/not-contains/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/profile-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/profile/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/regex-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/regex/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/value-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/assert/value/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/cli/cache-clear/files/{case.yaml => oats-case.yaml} (100%) create mode 100644 tests/e2e/cases/cli/cache-clear/files/oats-config.yaml delete mode 100644 tests/e2e/cases/cli/cache-clear/files/oats.toml rename tests/e2e/cases/cli/cache-skip/files/{case.yaml => oats-case.yaml} (100%) create mode 100644 tests/e2e/cases/cli/cache-skip/files/oats-config.yaml delete mode 100644 tests/e2e/cases/cli/cache-skip/files/oats.toml create mode 100644 tests/e2e/cases/cli/fail-fast/files/oats-config.yaml delete mode 100644 tests/e2e/cases/cli/fail-fast/files/oats.toml rename tests/e2e/cases/custom/custom-check-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/custom/custom-check/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/fixture/compose-app-seed/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/fixture/compose-logs-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/fixture/compose-logs/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/fixture/k3d-fail/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/fixture/k3d-smoke/files/{oats.yaml => oats-case.yaml} (100%) create mode 100644 tests/e2e/cases/fixture/parallel-compose/files/oats-config.yaml delete mode 100644 tests/e2e/cases/fixture/parallel-compose/files/oats.toml rename tests/e2e/cases/fixture/remote-basic/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/format/ndjson/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/seed/inline-otlp-invalid/files/{oats.yaml => oats-case.yaml} (100%) rename tests/e2e/cases/seed/inline-otlp/files/{oats.yaml => oats-case.yaml} (100%) diff --git a/.github/renovate-tracked-deps.json b/.github/renovate-tracked-deps.json index 3bd350bc..0ce21129 100644 --- a/.github/renovate-tracked-deps.json +++ b/.github/renovate-tracked-deps.json @@ -28,7 +28,6 @@ }, "go.mod": { "gomod": [ - "github.com/BurntSushi/toml", "github.com/cenkalti/backoff/v5", "github.com/cespare/xxhash/v2", "github.com/davecgh/go-spew", diff --git a/AGENTS.md b/AGENTS.md index ba307a50..1004c923 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -61,16 +61,16 @@ EditorConfig rules live in `.editorconfig`. Current user-facing syntax is documented in `README.md`. Legacy yaml parsing still exists in-package for migration support, but the repo's CLI surface is -the current `oats.toml` + case-yaml flow. +the current `oats-config.yaml` + case-yaml flow. ## CLI Usage ```bash # Print a plan -oats --config oats.toml --list +oats --config oats-config.yaml --list # With flags -oats --config oats.toml --timeout 1m +oats --config oats-config.yaml --timeout 1m ``` Key flags: `--config`, `--suite`, `--tags`, `--timeout`, `--interval`, diff --git a/README.md b/README.md index 0b29910b..c888ed6e 100644 --- a/README.md +++ b/README.md @@ -56,10 +56,10 @@ bootstrap gcx itself, but pinning it explicitly keeps runs reproducible. ```sh # Print the run plan without executing -oats list --config examples/smoke/oats.toml +oats list --config examples/smoke/oats-config.yaml # Run it -oats --config examples/smoke/oats.toml +oats --config examples/smoke/oats-config.yaml ``` To hack on OATS itself (builds local `oats` + `gcx` into `./bin`): @@ -74,7 +74,7 @@ bin/oats version ```sh oats [flags] # run the suites (implicit; same as `oats run`) oats run [flags] # run the suites -oats list --config oats.toml # print the run plan and exit +oats list --config oats-config.yaml # print the run plan and exit oats migrate legacy.yaml # convert one legacy yaml to the v3 shape oats cache clear # delete all cached results oats version # print the version @@ -84,7 +84,7 @@ Common flags: | Flag | Default | Meaning | |------|---------|---------| -| `--config` | `oats.toml` | path to the config file | +| `--config` | `oats-config.yaml` | path to the config file | | `--suite` | all | comma-separated suite names to run | | `--tags` | all | comma-separated tags; a case runs if it matches any | | `--timeout` | `30s` | per-assertion timeout — each assertion is retried until it passes or this elapses | diff --git a/UPGRADING.md b/UPGRADING.md index bcbc7acc..43ca6e15 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -13,7 +13,7 @@ The root `oats` binary now runs the gcx-driven current CLI only. That means upgrades now require migrating to the current: -- `oats.toml` suite/discovery file +- `oats-config.yaml` suite/discovery file - current case yaml shape documented in [docs/case-reference.md](docs/case-reference.md) The legacy “pass one or more old yaml files directly to `oats`” runner has been @@ -122,21 +122,21 @@ expected: contains: ["rolling the dice"] ``` -Run `oats migrate ./oats.yaml`. Split the result into an `oats.toml` and a v3 -case (the migrator prints both the case yaml and a suggested `[fixture]` block): +Run `oats migrate ./oats.yaml`. Split the result into an `oats-config.yaml` and a v3 +case (the migrator prints both the case yaml and a suggested `fixture:` block): -```toml -# oats.toml -[meta] -version = 2 - -[[suite]] -name = "rolldice" -cases = ["cases/*.yaml"] -fixture = "compose-lgtm" - -[fixture.compose-lgtm.compose] -file = "docker-compose.yaml" +```yaml +# oats-config.yaml +meta: + version: 2 +suites: + - name: rolldice + cases: ["cases/*.yaml"] + fixture: compose-lgtm +fixture: + compose-lgtm: + compose: + file: docker-compose.yaml ``` ```yaml diff --git a/casefile/case.go b/casefile/case.go index 0c30d078..6c5e7de9 100644 --- a/casefile/case.go +++ b/casefile/case.go @@ -27,7 +27,7 @@ import ( const SchemaVersion = 3 // Case is one entry point yaml file. Cases are independently runnable; -// suites group them via oats.toml. +// suites group them via oats-config.yaml. type Case struct { OatsSchemaVersion int `yaml:"oats-schema-version"` Name string `yaml:"name"` @@ -59,12 +59,12 @@ type Seed struct { // FixtureConfig declares how a suite stands up the backends its cases run // against. Exactly one of the type-specific blocks is set; the block that is -// present selects the fixture kind. Dual toml/yaml tags let the same struct -// load from oats.toml (BurntSushi/toml) and from a case yaml (yaml.v3). +// present selects the fixture kind. The same struct loads from oats-config.yaml and +// from a case yaml (both via yaml.v3). type FixtureConfig struct { - Compose *ComposeFixture `toml:"compose,omitempty" yaml:"compose,omitempty"` - K3D *K3DFixture `toml:"k3d,omitempty" yaml:"k3d,omitempty"` - Remote *RemoteFixture `toml:"remote,omitempty" yaml:"remote,omitempty"` + Compose *ComposeFixture `yaml:"compose,omitempty"` + K3D *K3DFixture `yaml:"k3d,omitempty"` + Remote *RemoteFixture `yaml:"remote,omitempty"` } // ComposeFixture boots a docker-compose stack. template selects a built-in @@ -72,29 +72,29 @@ type FixtureConfig struct { // source dir. app_service + app_port let OATS publish and discover an // ephemeral host port for the application under test. type ComposeFixture struct { - Template string `toml:"template,omitempty" yaml:"template,omitempty"` - File string `toml:"file,omitempty" yaml:"file,omitempty"` - Files []string `toml:"files,omitempty" yaml:"files,omitempty"` - Env []string `toml:"env,omitempty" yaml:"env,omitempty"` - AppService string `toml:"app_service,omitempty" yaml:"app_service,omitempty"` - AppPort int `toml:"app_port,omitempty" yaml:"app_port,omitempty"` + Template string `yaml:"template,omitempty"` + File string `yaml:"file,omitempty"` + Files []string `yaml:"files,omitempty"` + Env []string `yaml:"env,omitempty"` + AppService string `yaml:"app_service,omitempty"` + AppPort int `yaml:"app_port,omitempty"` } // K3DFixture boots a k3d cluster and builds/imports the application image. type K3DFixture struct { - K8sDir string `toml:"k8s_dir,omitempty" yaml:"k8s_dir,omitempty"` - AppService string `toml:"app_service,omitempty" yaml:"app_service,omitempty"` - AppDockerFile string `toml:"app_docker_file,omitempty" yaml:"app_docker_file,omitempty"` - AppDockerContext string `toml:"app_docker_context,omitempty" yaml:"app_docker_context,omitempty"` - AppDockerTag string `toml:"app_docker_tag,omitempty" yaml:"app_docker_tag,omitempty"` - AppPort int `toml:"app_port,omitempty" yaml:"app_port,omitempty"` - ImportImages []string `toml:"import_images,omitempty" yaml:"import_images,omitempty"` - PoolSize int `toml:"pool_size,omitempty" yaml:"pool_size,omitempty"` + K8sDir string `yaml:"k8s_dir,omitempty"` + AppService string `yaml:"app_service,omitempty"` + AppDockerFile string `yaml:"app_docker_file,omitempty"` + AppDockerContext string `yaml:"app_docker_context,omitempty"` + AppDockerTag string `yaml:"app_docker_tag,omitempty"` + AppPort int `yaml:"app_port,omitempty"` + ImportImages []string `yaml:"import_images,omitempty"` + PoolSize int `yaml:"pool_size,omitempty"` } // RemoteFixture points at an already-running stack; OATS boots nothing. type RemoteFixture struct { - Endpoint string `toml:"endpoint,omitempty" yaml:"endpoint,omitempty"` + Endpoint string `yaml:"endpoint,omitempty"` } // Kind returns "compose"/"k3d"/"remote", or "" when no block is set. Exactly @@ -389,7 +389,7 @@ func (c *Case) Validate() error { switch c.Seed.Type { case "app": // App-backed cases are normally booted by the suite fixture declared in - // oats.toml. seed.compose remains accepted as a legacy/migration + // oats-config.yaml. seed.compose remains accepted as a legacy/migration // shorthand, but is not required for validation or execution here. case "inline-otlp": if len(c.Seed.Traces)+len(c.Seed.Logs)+len(c.Seed.Metrics) == 0 { @@ -460,7 +460,7 @@ func (c *Case) Validate() error { // Validate enforces that exactly one fixture block is set and that the set // block carries the fields its kind requires. label names the fixture in -// error messages (the fixture name from oats.toml, or "fixture" for a +// error messages (the fixture name from oats-config.yaml, or "fixture" for a // case-level block). func (f FixtureConfig) Validate(label string) error { set := 0 diff --git a/discovery/discovery.go b/discovery/discovery.go index ada00c3f..5a57e586 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -1,14 +1,15 @@ -// Package discovery turns an oats.toml + a collection of case yamls into a +// Package discovery turns an oats-config.yaml + a collection of case yamls into a // concrete run plan. // // In OATS v1, the runner walked the file system for any yaml carrying // "oats-schema-version" and ran whatever it found. The current format -// declares the plan up front: oats.toml lists suites, each suite lists cases +// declares the plan up front: oats-config.yaml lists suites, each suite lists cases // (path globs) and the fixture they share. "oats list" prints the plan // before "oats run" executes it. package discovery import ( + "bytes" "fmt" "os" "path/filepath" @@ -16,57 +17,55 @@ import ( "sort" "strings" - "github.com/BurntSushi/toml" "github.com/grafana/oats/casefile" + "go.yaml.in/yaml/v3" ) -// RootConfig is the parsed oats.toml file. Field names mirror the TOML keys +// RootConfig is the parsed oats-config.yaml file. Field names mirror the YAML keys // directly so a misnamed key surfaces as a "field not defined" error from -// the toml decoder rather than a silent miss. +// the yaml decoder rather than a silent miss. type RootConfig struct { - Meta Meta `toml:"meta"` - Cases []string `toml:"cases"` - Suites []SuiteConfig `toml:"suite"` - Fixture map[string]casefile.FixtureConfig `toml:"fixture"` - Cache CacheConfig `toml:"cache,omitempty"` + Meta Meta `yaml:"meta"` + Cases []string `yaml:"cases,omitempty"` + Suites []SuiteConfig `yaml:"suites,omitempty"` + Fixture map[string]casefile.FixtureConfig `yaml:"fixture,omitempty"` + Cache CacheConfig `yaml:"cache,omitempty"` - // SourceDir is the directory of the loaded oats.toml. Case glob + // SourceDir is the directory of the loaded oats-config.yaml. Case glob // expressions resolve relative to it. - SourceDir string `toml:"-"` + SourceDir string `yaml:"-"` } type Meta struct { - Version int `toml:"version"` + Version int `yaml:"version"` } type SuiteConfig struct { - Name string `toml:"name"` - Cases []string `toml:"cases"` // path globs, relative to oats.toml dir - Fixture string `toml:"fixture,omitempty"` - Tags []string `toml:"tags,omitempty"` + Name string `yaml:"name"` + Cases []string `yaml:"cases"` // path globs, relative to oats-config.yaml dir + Fixture string `yaml:"fixture,omitempty"` + Tags []string `yaml:"tags,omitempty"` } type CacheConfig struct { - TTLDays int `toml:"ttl_days,omitempty"` // zero → use runtime default + TTLDays int `yaml:"ttl_days,omitempty"` // zero → use runtime default } -// SupportedVersion is the value of [meta].version that this binary parses. +// SupportedVersion is the value of meta.version that this binary parses. const SupportedVersion = 2 -// Load reads an oats.toml from disk. +// Load reads an oats-config.yaml from disk. func Load(path string) (*RootConfig, error) { data, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("discovery load %s: %w", path, err) } + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) // reject unknown keys var cfg RootConfig - md, err := toml.Decode(string(data), &cfg) - if err != nil { + if err := dec.Decode(&cfg); err != nil { return nil, fmt.Errorf("discovery parse %s: %w", path, err) } - if undecoded := md.Undecoded(); len(undecoded) > 0 { - return nil, fmt.Errorf("discovery: %s contains unknown keys: %v", path, undecoded) - } cfg.SourceDir = filepath.Dir(path) if err := cfg.Validate(); err != nil { return nil, err @@ -81,10 +80,10 @@ func (c *RootConfig) Validate() error { return fmt.Errorf("meta.version: expected %d, got %d", SupportedVersion, c.Meta.Version) } if len(c.Cases) > 0 && len(c.Suites) > 0 { - return fmt.Errorf("use top-level cases or [[suite]], not both") + return fmt.Errorf("use top-level cases or suites, not both") } if len(c.Cases) == 0 && len(c.Suites) == 0 { - return fmt.Errorf("at least one top-level case or [[suite]] required") + return fmt.Errorf("at least one top-level case or suites entry required") } if len(c.Cases) > 0 { for i, path := range c.Cases { @@ -139,7 +138,7 @@ func (c *RootConfig) effectiveSuites() []SuiteConfig { } // PlanRun expands globs and applies the filter against the loaded config. -// Returns plans in oats.toml order; cases within a plan are sorted by +// Returns plans in oats-config.yaml order; cases within a plan are sorted by // SourcePath for stable test ordering. func (c *RootConfig) PlanRun(f Filter) ([]Plan, error) { wantSuite := func(name string) bool { diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index 6a1d93bf..a52fc67c 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -33,23 +33,23 @@ expected: func TestLoad_ValidConfig(t *testing.T) { dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "lgtm" -cases = ["cases/*.yaml"] -fixture = "lgtm-shared" -tags = ["traces"] - -[fixture.lgtm-shared.compose] -template = "lgtm" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: lgtm + cases: ["cases/*.yaml"] + fixture: lgtm-shared + tags: ["traces"] +fixture: + lgtm-shared: + compose: + template: lgtm `) writeFile(t, dir, "cases/a.yaml", strings.Replace(validCaseYAML, "%s", "case-a", 1)) writeFile(t, dir, "cases/b.yaml", strings.Replace(validCaseYAML, "%s", "case-b", 1)) - cfg, err := Load(filepath.Join(dir, "oats.toml")) + cfg, err := Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -74,40 +74,37 @@ template = "lgtm" func TestLoad_RejectsUnknownKey(t *testing.T) { dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 -typooo = "boom" -[[suite]] -name = "x" -cases = ["a.yaml"] + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +typooo: boom +suites: + - name: x + cases: ["a.yaml"] `) - _, err := Load(filepath.Join(dir, "oats.toml")) - if err == nil || !strings.Contains(err.Error(), "unknown keys") { - t.Errorf("expected unknown-keys error, got %v", err) + _, err := Load(filepath.Join(dir, "oats-config.yaml")) + if err == nil || !strings.Contains(err.Error(), "typooo") { + t.Errorf("expected unknown-key error, got %v", err) } } func TestPlanRun_FilterByTag(t *testing.T) { dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "alpha" -cases = ["a.yaml"] -tags = ["traces"] - -[[suite]] -name = "beta" -cases = ["b.yaml"] -tags = ["logs"] + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: alpha + cases: ["a.yaml"] + tags: ["traces"] + - name: beta + cases: ["b.yaml"] + tags: ["logs"] `) writeFile(t, dir, "a.yaml", strings.Replace(validCaseYAML, "%s", "a", 1)) writeFile(t, dir, "b.yaml", strings.Replace(validCaseYAML, "%s", "b", 1)) - cfg, err := Load(filepath.Join(dir, "oats.toml")) + cfg, err := Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -122,15 +119,14 @@ tags = ["logs"] func TestSummary_TopLevelCases(t *testing.T) { dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -cases = ["cases/*.yaml"] - -[meta] -version = 2 + writeFile(t, dir, "oats-config.yaml", ` +cases: ["cases/*.yaml"] +meta: + version: 2 `) writeFile(t, dir, "cases/a.yaml", strings.Replace(validCaseYAML, "%s", "case-a", 1)) - cfg, err := Load(filepath.Join(dir, "oats.toml")) + cfg, err := Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -146,22 +142,19 @@ version = 2 func TestPlanRun_FilterBySuiteName(t *testing.T) { dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "alpha" -cases = ["a.yaml"] - -[[suite]] -name = "beta" -cases = ["b.yaml"] + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: alpha + cases: ["a.yaml"] + - name: beta + cases: ["b.yaml"] `) writeFile(t, dir, "a.yaml", strings.Replace(validCaseYAML, "%s", "a", 1)) writeFile(t, dir, "b.yaml", strings.Replace(validCaseYAML, "%s", "b", 1)) - cfg, err := Load(filepath.Join(dir, "oats.toml")) + cfg, err := Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -266,15 +259,14 @@ func TestValidate_K3DRequiresFields(t *testing.T) { func TestPlanRun_EmptyGlobIsAnError(t *testing.T) { dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "s" -cases = ["nope/*.yaml"] + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: s + cases: ["nope/*.yaml"] `) - cfg, err := Load(filepath.Join(dir, "oats.toml")) + cfg, err := Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -297,7 +289,7 @@ func TestSummary(t *testing.T) { } func TestExampleV2SmokeConfigLoads(t *testing.T) { - cfg, err := Load(filepath.Join("..", "examples", "smoke", "oats.toml")) + cfg, err := Load(filepath.Join("..", "examples", "smoke", "oats-config.yaml")) if err != nil { t.Fatalf("Load example config: %v", err) } @@ -318,7 +310,7 @@ func TestExampleV2SmokeConfigLoads(t *testing.T) { } func TestExampleV2FixturesConfigLoads(t *testing.T) { - cfg, err := Load(filepath.Join("..", "examples", "fixtures", "oats.toml")) + cfg, err := Load(filepath.Join("..", "examples", "fixtures", "oats-config.yaml")) if err != nil { t.Fatalf("Load fixture example config: %v", err) } @@ -350,11 +342,10 @@ func planNames(p []Plan) []string { func TestLoadTopLevelCases(t *testing.T) { dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -cases = ["cases/a.yaml", "cases/b.yaml"] - -[meta] -version = 2 + writeFile(t, dir, "oats-config.yaml", ` +cases: ["cases/a.yaml", "cases/b.yaml"] +meta: + version: 2 `) writeFile(t, dir, "cases/a.yaml", ` oats-schema-version: 3 @@ -374,7 +365,7 @@ expected: - traceql: "{}" absent: true `) - cfg, err := Load(filepath.Join(dir, "oats.toml")) + cfg, err := Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatal(err) } diff --git a/docs/case-reference.md b/docs/case-reference.md index 58261044..c3f6536f 100644 --- a/docs/case-reference.md +++ b/docs/case-reference.md @@ -1,21 +1,19 @@ # Case reference -The full shape of an OATS project: the `oats.toml` config, the case yaml, seed +The full shape of an OATS project: the `oats-config.yaml` config, the case yaml, seed modes, the assertion vocabulary, and custom checks. For a quick start and the CLI summary see the [README](../README.md); for running in CI see [ci.md](ci.md). -## oats.toml +## oats-config.yaml -```toml -[meta] -version = 2 # oats.toml schema version (distinct from a case's oats-schema-version) - -[[suite]] -cases = ["examples/smoke/cases/*.yaml"] - -[cache] -ttl_days = 7 # skip-when-unchanged TTL; 0 → default (7 days) +```yaml +meta: + version: 2 # oats-config.yaml schema version (distinct from a case's oats-schema-version) +suites: + - cases: ["examples/smoke/cases/*.yaml"] +cache: + ttl_days: 7 # skip-when-unchanged TTL; 0 → default (7 days) ``` A suite may pin its own fixture, or let each case carry a case-local `fixture:` diff --git a/docs/ci.md b/docs/ci.md index 45f3c1e4..dd5f665c 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -68,7 +68,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: jdx/mise-action@v2 # installs pinned oats + gcx from mise.toml - - run: oats --config oats.toml + - run: oats --config oats-config.yaml ``` The `paths` filter is the cheapest and most important optimization: if a PR @@ -89,7 +89,7 @@ To persist it across CI runs: with: path: ~/.cache/oats key: oats-${{ hashFiles('mise.toml') }} - - run: oats --config oats.toml --cache-dir ~/.cache/oats + - run: oats --config oats-config.yaml --cache-dir ~/.cache/oats ``` Keying on `hashFiles('mise.toml')` scopes the cache to a gcx/oats version pair, diff --git a/examples/fixtures/oats-config.yaml b/examples/fixtures/oats-config.yaml new file mode 100644 index 00000000..1d5f0f23 --- /dev/null +++ b/examples/fixtures/oats-config.yaml @@ -0,0 +1,25 @@ +meta: + version: 2 +suites: + - name: compose-app + cases: ["cases/compose/*.yaml"] + fixture: compose-dev + tags: [compose, app] + - name: k3d-app + cases: ["cases/k3d/*.yaml"] + fixture: k3d-dev + tags: [k3d, app] +fixture: + compose-dev: + compose: + files: ["docker-compose.yml", "docker-compose.override.yml"] + env: ["OTEL_EXPORTER_OTLP_ENDPOINT=http://lgtm:4318"] + k3d-dev: + k3d: + k8s_dir: k8s + app_service: rolldice + app_docker_file: Dockerfile + app_docker_context: . + app_docker_tag: rolldice:test + app_port: 18080 + import_images: ["grafana/otel-lgtm:latest"] diff --git a/examples/fixtures/oats.toml b/examples/fixtures/oats.toml deleted file mode 100644 index e3ab4a85..00000000 --- a/examples/fixtures/oats.toml +++ /dev/null @@ -1,27 +0,0 @@ -[meta] -version = 2 - -[[suite]] -name = "compose-app" -cases = ["cases/compose/*.yaml"] -fixture = "compose-dev" -tags = ["compose", "app"] - -[[suite]] -name = "k3d-app" -cases = ["cases/k3d/*.yaml"] -fixture = "k3d-dev" -tags = ["k3d", "app"] - -[fixture.compose-dev.compose] -files = ["docker-compose.yml", "docker-compose.override.yml"] -env = ["OTEL_EXPORTER_OTLP_ENDPOINT=http://lgtm:4318"] - -[fixture.k3d-dev.k3d] -k8s_dir = "k8s" -app_service = "rolldice" -app_docker_file = "Dockerfile" -app_docker_context = "." -app_docker_tag = "rolldice:test" -app_port = 18080 -import_images = ["grafana/otel-lgtm:latest"] diff --git a/examples/smoke/oats-config.yaml b/examples/smoke/oats-config.yaml new file mode 100644 index 00000000..407a15fd --- /dev/null +++ b/examples/smoke/oats-config.yaml @@ -0,0 +1,11 @@ +meta: + version: 2 +suites: + - name: smoke + cases: ["cases/*.yaml"] + fixture: local-lgtm + tags: [traces, logs, metrics] +fixture: + local-lgtm: + remote: + endpoint: http://localhost:4318 diff --git a/examples/smoke/oats.toml b/examples/smoke/oats.toml deleted file mode 100644 index 4c8b105e..00000000 --- a/examples/smoke/oats.toml +++ /dev/null @@ -1,11 +0,0 @@ -[meta] -version = 2 - -[[suite]] -name = "smoke" -cases = ["cases/*.yaml"] -fixture = "local-lgtm" -tags = ["traces", "logs", "metrics"] - -[fixture.local-lgtm.remote] -endpoint = "http://localhost:4318" diff --git a/fixture/fixture_test.go b/fixture/fixture_test.go index 0f76bb15..511acda7 100644 --- a/fixture/fixture_test.go +++ b/fixture/fixture_test.go @@ -278,18 +278,18 @@ func TestComposeFilePublishesFixedHostPorts(t *testing.T) { func TestSupportsParallel_ComposeTemplateLGTM(t *testing.T) { dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "parallel-safe" -cases = ["cases/*.yaml"] -fixture = "stack" - -[fixture.stack.compose] -template = "lgtm" -file = "docker-compose.oats.yml" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: parallel-safe + cases: ["cases/*.yaml"] + fixture: stack +fixture: + stack: + compose: + template: lgtm + file: docker-compose.oats.yml `) writeFile(t, dir, "docker-compose.oats.yml", `services: app: @@ -309,7 +309,7 @@ expected: contains: line `) - cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + cfg, err := discovery.Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } diff --git a/go.mod b/go.mod index 509b98ed..2c9db0ac 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/grafana/oats go 1.26.4 require ( - github.com/BurntSushi/toml v1.6.0 github.com/onsi/gomega v1.42.1 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.9 diff --git a/go.sum b/go.sum index f7986659..773222e6 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= -github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= diff --git a/internal/cli/integration_test.go b/internal/cli/integration_test.go index cd909a56..53d9274e 100644 --- a/internal/cli/integration_test.go +++ b/internal/cli/integration_test.go @@ -143,19 +143,19 @@ func TestIntegration_FullPipelineWithFakeGCX(t *testing.T) { t.Skip("fake-gcx is a POSIX shell script") } - // Build the temp workspace: an oats.toml plus one inline-otlp case. + // Build the temp workspace: an oats-config.yaml plus one inline-otlp case. dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "smoke" -cases = ["cases/*.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "REPLACED_AT_RUNTIME" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: smoke + cases: ["cases/*.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "REPLACED_AT_RUNTIME" `) writeFile(t, dir, "cases/inline.yaml", `oats-schema-version: 3 name: inline seed end-to-end @@ -204,11 +204,11 @@ expected: })) defer stub.Close() - // Patch the endpoint into oats.toml after the stub URL is known. - tomlPath := filepath.Join(dir, "oats.toml") - rewrite(t, tomlPath, "REPLACED_AT_RUNTIME", stub.URL) + // Patch the endpoint into oats-config.yaml after the stub URL is known. + configPath := filepath.Join(dir, "oats-config.yaml") + rewrite(t, configPath, "REPLACED_AT_RUNTIME", stub.URL) - cfg, err := discovery.Load(tomlPath) + cfg, err := discovery.Load(configPath) if err != nil { t.Fatalf("Load: %v", err) } @@ -267,17 +267,17 @@ func TestIntegration_AppSeedWithRemoteFixtureAndInput(t *testing.T) { appHost, appPort := splitHostPort(t, app.Listener.Addr().String()) dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "smoke" -cases = ["cases/*.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "http://localhost:4318" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: smoke + cases: ["cases/*.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "http://localhost:4318" `) writeFile(t, dir, "cases/app.yaml", `oats-schema-version: 3 name: app seed end-to-end @@ -302,7 +302,7 @@ expected: service_name: gcx-e2e-seed `) - cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + cfg, err := discovery.Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -353,17 +353,17 @@ func TestIntegration_ProfileQueryWithFakeGCX(t *testing.T) { } dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "profiles" -cases = ["cases/*.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "http://localhost:4318" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: profiles + cases: ["cases/*.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "http://localhost:4318" `) writeFile(t, dir, "cases/profile.yaml", `oats-schema-version: 3 name: profile query end-to-end @@ -377,7 +377,7 @@ expected: name: 'main|worker' `) - cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + cfg, err := discovery.Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -441,21 +441,21 @@ expected: } dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "migrated-profile" -cases = ["cases/*.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "http://localhost:4318" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: migrated-profile + cases: ["cases/*.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) - cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + cfg, err := discovery.Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -528,21 +528,21 @@ expected: } dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "migrated" -cases = ["cases/*.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "http://localhost:4318" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: migrated + cases: ["cases/*.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) - cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + cfg, err := discovery.Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -606,21 +606,21 @@ expected: } dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "migrated-custom" -cases = ["cases/*.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "http://localhost:4318" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: migrated-custom + cases: ["cases/*.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) - cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + cfg, err := discovery.Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -679,17 +679,17 @@ expected: } dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "migrated-custom-path" -cases = ["cases/*.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "http://localhost:4318" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: migrated-custom-path + cases: ["cases/*.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) writeFile(t, dir, "cases/scripts/verify.sh", "#!/bin/sh\nexit 0\n") @@ -697,7 +697,7 @@ endpoint = "http://localhost:4318" t.Fatalf("Chmod: %v", err) } - cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + cfg, err := discovery.Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } @@ -774,21 +774,21 @@ expected: } dir := t.TempDir() - writeFile(t, dir, "oats.toml", ` -[meta] -version = 2 - -[[suite]] -name = "migrated-matrix" -cases = ["cases/*.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "http://localhost:4318" + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 2 +suites: + - name: migrated-matrix + cases: ["cases/*.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "http://localhost:4318" `) writeFile(t, dir, "cases/migrated.yaml", string(migrated)) - cfg, err := discovery.Load(filepath.Join(dir, "oats.toml")) + cfg, err := discovery.Load(filepath.Join(dir, "oats-config.yaml")) if err != nil { t.Fatalf("Load: %v", err) } diff --git a/internal/cli/main.go b/internal/cli/main.go index 9b032a96..f5963fb3 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -14,7 +14,7 @@ // // Run flags (subset): // -// --config Path to oats.toml (default ./oats.toml) +// --config Path to oats-config.yaml (default ./oats-config.yaml) // --gcx Path to gcx binary (default "gcx" on PATH) // --format Output format: "text" (default) or "ndjson" // --suite Comma-separated suite names to include @@ -105,7 +105,7 @@ func newRootCmd(exit *int) *cobra.Command { // addRunFlags registers the flags shared by the implicit-run root and the // explicit `run` subcommand. func addRunFlags(fs *pflag.FlagSet) { - fs.String("config", "oats.toml", "path to oats.toml") + fs.String("config", "oats-config.yaml", "path to oats-config.yaml") fs.String("gcx", "gcx", "path to gcx binary (PATH-resolved if a bare name)") fs.String("format", "text", "output format: text | ndjson") fs.String("suite", "", "comma-separated suite names") @@ -134,7 +134,7 @@ func addRunFlags(fs *pflag.FlagSet) { func newRunCmd(verbose, exit *int) *cobra.Command { cmd := &cobra.Command{ Use: "run", - Short: "Run the suites in oats.toml (default when no subcommand is given)", + Short: "Run the suites in oats-config.yaml (default when no subcommand is given)", SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, _ []string) error { @@ -155,7 +155,7 @@ func newListCmd() *cobra.Command { return listAction(cmd) }, } - cmd.Flags().String("config", "oats.toml", "path to oats.toml") + cmd.Flags().String("config", "oats-config.yaml", "path to oats-config.yaml") return cmd } diff --git a/migrate/migrate.go b/migrate/migrate.go index b841f94b..91ffa054 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -47,7 +47,7 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Cas if def.Kubernetes != nil { c.Seed.Type = "app" - warnings = append(warnings, "legacy kubernetes fixture migrated as an app-backed case; paste the suggested [fixture] block below into oats.toml") + warnings = append(warnings, "legacy kubernetes fixture migrated as an app-backed case; paste the suggested fixture: block below into oats-config.yaml") warnings = append(warnings, kubernetesFixtureHint(name, def)) } if len(def.Matrix) > 0 { @@ -61,12 +61,12 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Cas } c.Seed.Type = "app" c.Seed.Compose = selectedMatrix.DockerCompose.Files[0] - warnings = append(warnings, "single matrix docker-compose fixture selected; paste the suggested [fixture] block below into oats.toml") + warnings = append(warnings, "single matrix docker-compose fixture selected; paste the suggested fixture: block below into oats-config.yaml") warnings = append(warnings, matrixFixtureHint(c.Name, *selectedMatrix)) } if selectedMatrix.Kubernetes != nil { c.Seed.Type = "app" - warnings = append(warnings, "single matrix kubernetes fixture selected; paste the suggested [fixture] block below into oats.toml") + warnings = append(warnings, "single matrix kubernetes fixture selected; paste the suggested fixture: block below into oats-config.yaml") warnings = append(warnings, matrixFixtureHint(c.Name, *selectedMatrix)) } } else { @@ -88,7 +88,7 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Cas } c.Seed.Compose = def.DockerCompose.Files[0] if len(def.DockerCompose.Files) > 1 || len(def.DockerCompose.Environment) > 0 { - warnings = append(warnings, "legacy docker-compose fixture uses suite-level config richer than case-level seed.compose; paste the suggested [fixture] block below into oats.toml") + warnings = append(warnings, "legacy docker-compose fixture uses suite-level config richer than case-level seed.compose; paste the suggested fixture: block below into oats-config.yaml") warnings = append(warnings, composeFixtureHint(name, def)) } } else if def.Kubernetes == nil && selectedMatrix == nil { @@ -265,15 +265,17 @@ func keepForMatrix(matrixCondition string, selected *model.Matrix) bool { func composeFixtureHint(name string, def model.TestCaseDefinition) string { fixtureName := slug(name) var b strings.Builder - fmt.Fprintf(&b, "suggested oats.toml fixture snippet for %q:\n", name) - fmt.Fprintf(&b, "[fixture.%s.compose]\n", fixtureName) + fmt.Fprintf(&b, "suggested oats-config.yaml fixture snippet for %q:\n", name) + fmt.Fprintf(&b, "fixture:\n") + fmt.Fprintf(&b, " %s:\n", fixtureName) + fmt.Fprintf(&b, " compose:\n") if len(def.DockerCompose.Files) == 1 { - fmt.Fprintf(&b, "file = %q\n", def.DockerCompose.Files[0]) + fmt.Fprintf(&b, " file: %s\n", def.DockerCompose.Files[0]) } else if len(def.DockerCompose.Files) > 1 { - fmt.Fprintf(&b, "files = [%s]\n", quotedList(def.DockerCompose.Files)) + fmt.Fprintf(&b, " files: [%s]\n", quotedList(def.DockerCompose.Files)) } if len(def.DockerCompose.Environment) > 0 { - fmt.Fprintf(&b, "env = [%s]\n", quotedList(def.DockerCompose.Environment)) + fmt.Fprintf(&b, " env: [%s]\n", quotedList(def.DockerCompose.Environment)) } return strings.TrimRight(b.String(), "\n") } @@ -282,18 +284,20 @@ func kubernetesFixtureHint(name string, def model.TestCaseDefinition) string { fixtureName := slug(name) k := def.Kubernetes var b strings.Builder - fmt.Fprintf(&b, "suggested oats.toml fixture snippet for %q:\n", name) - fmt.Fprintf(&b, "[fixture.%s.k3d]\n", fixtureName) - fmt.Fprintf(&b, "k8s_dir = %q\n", k.Dir) - fmt.Fprintf(&b, "app_service = %q\n", k.AppService) - fmt.Fprintf(&b, "app_docker_file = %q\n", k.AppDockerFile) + fmt.Fprintf(&b, "suggested oats-config.yaml fixture snippet for %q:\n", name) + fmt.Fprintf(&b, "fixture:\n") + fmt.Fprintf(&b, " %s:\n", fixtureName) + fmt.Fprintf(&b, " k3d:\n") + fmt.Fprintf(&b, " k8s_dir: %s\n", k.Dir) + fmt.Fprintf(&b, " app_service: %s\n", k.AppService) + fmt.Fprintf(&b, " app_docker_file: %s\n", k.AppDockerFile) if k.AppDockerContext != "" { - fmt.Fprintf(&b, "app_docker_context = %q\n", k.AppDockerContext) + fmt.Fprintf(&b, " app_docker_context: %s\n", k.AppDockerContext) } - fmt.Fprintf(&b, "app_docker_tag = %q\n", k.AppDockerTag) - fmt.Fprintf(&b, "app_port = %d\n", k.AppDockerPort) + fmt.Fprintf(&b, " app_docker_tag: %s\n", k.AppDockerTag) + fmt.Fprintf(&b, " app_port: %d\n", k.AppDockerPort) if len(k.ImportImages) > 0 { - fmt.Fprintf(&b, "import_images = [%s]\n", quotedList(k.ImportImages)) + fmt.Fprintf(&b, " import_images: [%s]\n", quotedList(k.ImportImages)) } return strings.TrimRight(b.String(), "\n") } diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index d50e0377..f6457d1f 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -126,8 +126,8 @@ func TestConvertFile_MatrixSampleIsParseable(t *testing.T) { for _, want := range []string{ `matrix definitions are not migrated automatically when multiple entries exist`, `suggested matrix expansion for "matrix test.oats"`, - `[fixture.matrix-test-oats-docker.compose]`, - `[fixture.matrix-test-oats-k8s.k3d]`, + `matrix-test-oats-docker:`, + `matrix-test-oats-k8s:`, } { if !strings.Contains(joined, want) { fatalf(t, "expected matrix warning to contain %q:\n%s", want, joined) @@ -212,14 +212,15 @@ func TestConvertDefinition_KubernetesProducesFixtureHint(t *testing.T) { } joined := strings.Join(warnings, "\n") for _, want := range []string{ - `[fixture.k8s-case.k3d]`, - `k8s_dir = "k8s"`, - `app_service = "dice"`, - `app_docker_file = "Dockerfile"`, - `app_docker_context = ".."`, - `app_docker_tag = "dice:test"`, - `app_port = 8080`, - `import_images = ["busybox:latest"]`, + `k8s-case:`, + `k3d:`, + `k8s_dir: k8s`, + `app_service: dice`, + `app_docker_file: Dockerfile`, + `app_docker_context: ..`, + `app_docker_tag: dice:test`, + `app_port: 8080`, + `import_images: ["busybox:latest"]`, } { if !strings.Contains(joined, want) { fatalf(t, "expected kubernetes migration warning to contain %q:\n%s", want, joined) @@ -278,8 +279,8 @@ func TestConvertDefinition_FlattensSingleMatrixEntry(t *testing.T) { joined := strings.Join(warnings, "\n") for _, want := range []string{ `flattened single matrix entry "docker"`, - `[fixture.matrix-case-docker.compose]`, - `file = "docker-compose.yml"`, + `matrix-case-docker:`, + `file: docker-compose.yml`, } { if !strings.Contains(joined, want) { fatalf(t, "expected warning to contain %q:\n%s", want, joined) @@ -326,9 +327,9 @@ func TestConvertDefinition_MultiMatrixEmitsExpansionHint(t *testing.T) { for _, want := range []string{ `suggested matrix expansion for "matrix case"`, `- docker`, - `[fixture.matrix-case-docker.compose]`, + `matrix-case-docker:`, `- k8s`, - `[fixture.matrix-case-k8s.k3d]`, + `matrix-case-k8s:`, } { if !strings.Contains(joined, want) { fatalf(t, "expected warning to contain %q:\n%s", want, joined) diff --git a/tests/e2e/README.md b/tests/e2e/README.md index b9c04cd3..4b5cdb2a 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -11,14 +11,14 @@ tests/e2e/cases/// ## Defaults -- `files/oats.yaml` is the default OATS input. -- When `files/oats.toml` is absent, the test runner synthesizes one that points - at `files/oats.yaml`. +- `files/oats-config.yaml` is the default OATS config (suite/case list). +- When `files/oats-config.yaml` is absent, the test runner synthesizes one that points + at `files/oats-case.yaml`. - The harness boots a shared local LGTM stack plus real `oats` and real `gcx`. - The default run command is: ```bash - --config .generated-oats.toml --gcx --gcx-context local --no-cache --timeout 10s --interval 1s + --config .generated-oats.yaml --gcx --gcx-context local --no-cache --timeout 10s --interval 1s ``` - Expected exit code defaults to `0`. diff --git a/tests/e2e/cases/assert/absent-fail/files/oats.yaml b/tests/e2e/cases/assert/absent-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/absent-fail/files/oats.yaml rename to tests/e2e/cases/assert/absent-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/absent/files/oats.yaml b/tests/e2e/cases/assert/absent/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/absent/files/oats.yaml rename to tests/e2e/cases/assert/absent/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/contains-fail/files/oats.yaml b/tests/e2e/cases/assert/contains-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/contains-fail/files/oats.yaml rename to tests/e2e/cases/assert/contains-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/contains/files/oats.yaml b/tests/e2e/cases/assert/contains/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/contains/files/oats.yaml rename to tests/e2e/cases/assert/contains/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/count-fail/files/oats.yaml b/tests/e2e/cases/assert/count-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/count-fail/files/oats.yaml rename to tests/e2e/cases/assert/count-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/count/files/oats.yaml b/tests/e2e/cases/assert/count/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/count/files/oats.yaml rename to tests/e2e/cases/assert/count/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/match-fail/files/oats.yaml b/tests/e2e/cases/assert/match-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/match-fail/files/oats.yaml rename to tests/e2e/cases/assert/match-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml b/tests/e2e/cases/assert/match-regexp-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/match-regexp-fail/files/oats.yaml rename to tests/e2e/cases/assert/match-regexp-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/match-regexp/files/oats.yaml b/tests/e2e/cases/assert/match-regexp/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/match-regexp/files/oats.yaml rename to tests/e2e/cases/assert/match-regexp/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/match-spans-fail/files/oats.yaml b/tests/e2e/cases/assert/match-spans-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/match-spans-fail/files/oats.yaml rename to tests/e2e/cases/assert/match-spans-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/not-contains-fail/files/oats.yaml b/tests/e2e/cases/assert/not-contains-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/not-contains-fail/files/oats.yaml rename to tests/e2e/cases/assert/not-contains-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/not-contains/files/oats.yaml b/tests/e2e/cases/assert/not-contains/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/not-contains/files/oats.yaml rename to tests/e2e/cases/assert/not-contains/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/profile-fail/files/oats.yaml b/tests/e2e/cases/assert/profile-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/profile-fail/files/oats.yaml rename to tests/e2e/cases/assert/profile-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/profile/files/oats.yaml b/tests/e2e/cases/assert/profile/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/profile/files/oats.yaml rename to tests/e2e/cases/assert/profile/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/regex-fail/files/oats.yaml b/tests/e2e/cases/assert/regex-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/regex-fail/files/oats.yaml rename to tests/e2e/cases/assert/regex-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/regex/files/oats.yaml b/tests/e2e/cases/assert/regex/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/regex/files/oats.yaml rename to tests/e2e/cases/assert/regex/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/value-fail/files/oats.yaml b/tests/e2e/cases/assert/value-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/value-fail/files/oats.yaml rename to tests/e2e/cases/assert/value-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/assert/value/files/oats.yaml b/tests/e2e/cases/assert/value/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/assert/value/files/oats.yaml rename to tests/e2e/cases/assert/value/files/oats-case.yaml diff --git a/tests/e2e/cases/cli/cache-clear/files/case.yaml b/tests/e2e/cases/cli/cache-clear/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/cli/cache-clear/files/case.yaml rename to tests/e2e/cases/cli/cache-clear/files/oats-case.yaml diff --git a/tests/e2e/cases/cli/cache-clear/files/oats-config.yaml b/tests/e2e/cases/cli/cache-clear/files/oats-config.yaml new file mode 100644 index 00000000..47f1de30 --- /dev/null +++ b/tests/e2e/cases/cli/cache-clear/files/oats-config.yaml @@ -0,0 +1,10 @@ +meta: + version: 2 +suites: + - name: cache-clear + cases: ["oats-case.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/cache-clear/files/oats.toml b/tests/e2e/cases/cli/cache-clear/files/oats.toml deleted file mode 100644 index 7b99a7d4..00000000 --- a/tests/e2e/cases/cli/cache-clear/files/oats.toml +++ /dev/null @@ -1,10 +0,0 @@ -[meta] -version = 2 - -[[suite]] -name = "cache-clear" -cases = ["case.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/cache-clear/test.yaml b/tests/e2e/cases/cli/cache-clear/test.yaml index 2215260e..2431287a 100644 --- a/tests/e2e/cases/cli/cache-clear/test.yaml +++ b/tests/e2e/cases/cli/cache-clear/test.yaml @@ -8,10 +8,10 @@ tags: [cli, cache] run: shell: | set -e - {{OATS}} --config files/oats.toml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v + {{OATS}} --config files/oats-config.yaml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v {{OATS}} cache clear --cache-dir "$PWD/cache" echo "=== rerun after clear ===" - {{OATS}} --config files/oats.toml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v + {{OATS}} --config files/oats-config.yaml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v expect: stdout_contains: diff --git a/tests/e2e/cases/cli/cache-skip/files/case.yaml b/tests/e2e/cases/cli/cache-skip/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/cli/cache-skip/files/case.yaml rename to tests/e2e/cases/cli/cache-skip/files/oats-case.yaml diff --git a/tests/e2e/cases/cli/cache-skip/files/oats-config.yaml b/tests/e2e/cases/cli/cache-skip/files/oats-config.yaml new file mode 100644 index 00000000..99788d5e --- /dev/null +++ b/tests/e2e/cases/cli/cache-skip/files/oats-config.yaml @@ -0,0 +1,10 @@ +meta: + version: 2 +suites: + - name: cache + cases: ["oats-case.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/cache-skip/files/oats.toml b/tests/e2e/cases/cli/cache-skip/files/oats.toml deleted file mode 100644 index 43526a06..00000000 --- a/tests/e2e/cases/cli/cache-skip/files/oats.toml +++ /dev/null @@ -1,10 +0,0 @@ -[meta] -version = 2 - -[[suite]] -name = "cache" -cases = ["case.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/cache-skip/test.yaml b/tests/e2e/cases/cli/cache-skip/test.yaml index eec7bcb6..31740eb5 100644 --- a/tests/e2e/cases/cli/cache-skip/test.yaml +++ b/tests/e2e/cases/cli/cache-skip/test.yaml @@ -6,9 +6,9 @@ tags: [cli, cache] run: shell: | set -e - {{OATS}} --config files/oats.toml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v + {{OATS}} --config files/oats-config.yaml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v echo "=== second run ===" - {{OATS}} --config files/oats.toml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v + {{OATS}} --config files/oats-config.yaml --gcx {{GCX}} --gcx-context local --cache-dir "$PWD/cache" --timeout 30s --interval 2s -v expect: stdout_contains: diff --git a/tests/e2e/cases/cli/fail-fast/files/oats-config.yaml b/tests/e2e/cases/cli/fail-fast/files/oats-config.yaml new file mode 100644 index 00000000..3b24079b --- /dev/null +++ b/tests/e2e/cases/cli/fail-fast/files/oats-config.yaml @@ -0,0 +1,10 @@ +meta: + version: 2 +suites: + - name: failfast + cases: ["01-fail.yaml", "02-pass.yaml"] + fixture: remote-lgtm +fixture: + remote-lgtm: + remote: + endpoint: "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/cli/fail-fast/files/oats.toml b/tests/e2e/cases/cli/fail-fast/files/oats.toml deleted file mode 100644 index 6cb2ec20..00000000 --- a/tests/e2e/cases/cli/fail-fast/files/oats.toml +++ /dev/null @@ -1,10 +0,0 @@ -[meta] -version = 2 - -[[suite]] -name = "failfast" -cases = ["01-fail.yaml", "02-pass.yaml"] -fixture = "remote-lgtm" - -[fixture.remote-lgtm.remote] -endpoint = "{{REMOTE_OTLP_HTTP}}" diff --git a/tests/e2e/cases/custom/custom-check-fail/files/oats.yaml b/tests/e2e/cases/custom/custom-check-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/custom/custom-check-fail/files/oats.yaml rename to tests/e2e/cases/custom/custom-check-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/custom/custom-check/files/oats.yaml b/tests/e2e/cases/custom/custom-check/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/custom/custom-check/files/oats.yaml rename to tests/e2e/cases/custom/custom-check/files/oats-case.yaml diff --git a/tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml b/tests/e2e/cases/fixture/compose-app-seed/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/fixture/compose-app-seed/files/oats.yaml rename to tests/e2e/cases/fixture/compose-app-seed/files/oats-case.yaml diff --git a/tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml b/tests/e2e/cases/fixture/compose-logs-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/fixture/compose-logs-fail/files/oats.yaml rename to tests/e2e/cases/fixture/compose-logs-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/fixture/compose-logs/files/oats.yaml b/tests/e2e/cases/fixture/compose-logs/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/fixture/compose-logs/files/oats.yaml rename to tests/e2e/cases/fixture/compose-logs/files/oats-case.yaml diff --git a/tests/e2e/cases/fixture/k3d-fail/files/oats.yaml b/tests/e2e/cases/fixture/k3d-fail/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/fixture/k3d-fail/files/oats.yaml rename to tests/e2e/cases/fixture/k3d-fail/files/oats-case.yaml diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/fixture/k3d-smoke/files/oats.yaml rename to tests/e2e/cases/fixture/k3d-smoke/files/oats-case.yaml diff --git a/tests/e2e/cases/fixture/parallel-compose/files/oats-config.yaml b/tests/e2e/cases/fixture/parallel-compose/files/oats-config.yaml new file mode 100644 index 00000000..3b6f0f96 --- /dev/null +++ b/tests/e2e/cases/fixture/parallel-compose/files/oats-config.yaml @@ -0,0 +1,14 @@ +meta: + version: 2 +suites: + - name: alpha + cases: ["alpha.yaml"] + fixture: shared + - name: beta + cases: ["beta.yaml"] + fixture: shared +fixture: + shared: + compose: + template: lgtm + file: docker-compose.oats.yml diff --git a/tests/e2e/cases/fixture/parallel-compose/files/oats.toml b/tests/e2e/cases/fixture/parallel-compose/files/oats.toml deleted file mode 100644 index ea8ee4bc..00000000 --- a/tests/e2e/cases/fixture/parallel-compose/files/oats.toml +++ /dev/null @@ -1,16 +0,0 @@ -[meta] -version = 2 - -[[suite]] -name = "alpha" -cases = ["alpha.yaml"] -fixture = "shared" - -[[suite]] -name = "beta" -cases = ["beta.yaml"] -fixture = "shared" - -[fixture.shared.compose] -template = "lgtm" -file = "docker-compose.oats.yml" diff --git a/tests/e2e/cases/fixture/remote-basic/files/oats.yaml b/tests/e2e/cases/fixture/remote-basic/files/oats-case.yaml similarity index 100% rename from tests/e2e/cases/fixture/remote-basic/files/oats.yaml rename to tests/e2e/cases/fixture/remote-basic/files/oats-case.yaml diff --git a/tests/e2e/cases/fixture/remote-basic/test.yaml b/tests/e2e/cases/fixture/remote-basic/test.yaml index bb6aab0a..9d758d55 100644 --- a/tests/e2e/cases/fixture/remote-basic/test.yaml +++ b/tests/e2e/cases/fixture/remote-basic/test.yaml @@ -33,7 +33,7 @@ run: tempo: tempo pyroscope: pyroscope CFG - cat > "$workdir/files/oats.yaml" < "$workdir/files/oats-case.yaml" < Date: Wed, 8 Jul 2026 14:10:59 +0000 Subject: [PATCH 113/124] feat(migrate): directory mode that generates a full v3 project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `oats migrate ` now walks a legacy tree (reusing the v1 collector — honors .oatsignore, skips templates), rewrites each case in place as v3, and generates an oats-config.yaml listing them explicitly. `oats migrate ` still prints a single case to stdout. Migrated cases now carry a case-local fixture: block built from the legacy docker-compose/kubernetes config, replacing the old "paste this snippet" text hints — output is self-contained. YAML is emitted with 2-space indent to match the repo convention. Signed-off-by: Gregor Zeitlinger --- internal/cli/main.go | 34 ++++- migrate/migrate.go | 302 ++++++++++++++++++++++------------------ migrate/migrate_test.go | 161 ++++++++++++++------- 3 files changed, 307 insertions(+), 190 deletions(-) diff --git a/internal/cli/main.go b/internal/cli/main.go index f5963fb3..da10f317 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -161,8 +161,19 @@ func newListCmd() *cobra.Command { func newMigrateCmd() *cobra.Command { cmd := &cobra.Command{ - Use: "migrate ", - Short: "Convert one legacy OATS yaml file and print the result to stdout", + Use: "migrate ", + Short: "Convert legacy OATS yaml to the v3 shape", + Long: `Convert legacy OATS test cases to the v3 shape. + +File mode (argument is a file): + Convert one legacy yaml and print the self-contained v3 case (including its + fixture: block) to stdout. Warnings go to stderr. + +Directory mode (argument is a directory): + Migrate every legacy case found under the directory in place (each file is + overwritten with its v3 equivalent) and write an oats-config.yaml listing + them explicitly. A human summary and per-file warnings go to stderr; nothing + is written to stdout.`, Args: cobra.ExactArgs(1), SilenceUsage: true, SilenceErrors: true, @@ -223,6 +234,25 @@ func listAction(cmd *cobra.Command) error { } func migrateAction(path string) error { + info, err := os.Stat(path) + if err != nil { + return err + } + if info.IsDir() { + res, err := migrate.ConvertTree(path) + if err != nil { + return err + } + for _, w := range res.Warnings { + fmt.Fprintln(os.Stderr, "migrate warning:", w) + } + for _, f := range res.Written { + fmt.Fprintln(os.Stderr, "migrated:", f) + } + fmt.Fprintf(os.Stderr, "wrote %d case(s); config: %s\n", len(res.Written), res.Config) + return nil + } + out, warnings, err := migrate.ConvertFile(path) if err != nil { return err diff --git a/migrate/migrate.go b/migrate/migrate.go index 91ffa054..7cda8dd4 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -1,20 +1,41 @@ package migrate import ( + "bytes" "fmt" + "os" "path/filepath" "regexp" "sort" "strings" "github.com/grafana/oats/casefile" + "github.com/grafana/oats/discovery" "github.com/grafana/oats/model" legacyyaml "github.com/grafana/oats/yaml" goyaml "go.yaml.in/yaml/v3" ) +// marshalYAML renders v as YAML with 2-space indentation, matching the repo's +// YAML convention and the shipped examples (goyaml.Marshal defaults to 4). +func marshalYAML(v any) ([]byte, error) { + var b bytes.Buffer + enc := goyaml.NewEncoder(&b) + enc.SetIndent(2) + if err := enc.Encode(v); err != nil { + _ = enc.Close() + return nil, err + } + if err := enc.Close(); err != nil { + return nil, err + } + return b.Bytes(), nil +} + // ConvertFile reads one legacy OATS yaml file and returns a best-effort // current-format case yaml plus any warnings about dropped or lossy fields. +// The migrated case carries its own case-local fixture: block, so the output +// is self-contained. func ConvertFile(path string) ([]byte, []string, error) { def, err := legacyyaml.LoadTestCaseDefinition(path) if err != nil { @@ -27,15 +48,100 @@ func ConvertFile(path string) ([]byte, []string, error) { if err != nil { return nil, warnings, err } - out, err := goyaml.Marshal(c) + out, err := marshalYAML(c) if err != nil { return nil, warnings, err } return out, warnings, nil } +// TreeResult reports the outcome of a directory ("project") migration. +type TreeResult struct { + // Written lists the absolute paths of the case files rewritten in place. + Written []string + // Config is the absolute path of the generated oats-config.yaml. + Config string + // Warnings aggregates every case's warnings, each prefixed with the + // relative path of the case it came from. + Warnings []string +} + +// ConvertTree migrates every legacy OATS case found under dir in place and +// writes an oats-config.yaml listing them explicitly. Each legacy file is +// overwritten with its v3 equivalent (same path/filename); the generated +// config references each written file by its dir-relative path. +func ConvertTree(dir string) (*TreeResult, error) { + cases, err := legacyyaml.ReadTestCases([]string{dir}, true) + if err != nil { + return nil, err + } + absDir, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + + result := &TreeResult{} + seen := map[string]bool{} + var relpaths []string + for _, tc := range cases { + // ReadTestCases expands matrix files into one entry per matrix row, + // all sharing a Path. Convert each source file once; ConvertDefinition + // handles the matrix internally. + if seen[tc.Path] { + continue + } + seen[tc.Path] = true + + c, warnings, err := ConvertDefinition(tc.Definition, deriveName(tc.Path)) + if err != nil { + return nil, fmt.Errorf("%s: %w", tc.Path, err) + } + out, err := marshalYAML(c) + if err != nil { + return nil, fmt.Errorf("%s: %w", tc.Path, err) + } + if err := os.WriteFile(tc.Path, out, 0o644); err != nil { + return nil, fmt.Errorf("write %s: %w", tc.Path, err) + } + result.Written = append(result.Written, tc.Path) + + rel, err := filepath.Rel(absDir, tc.Path) + if err != nil { + return nil, err + } + rel = filepath.ToSlash(rel) + relpaths = append(relpaths, rel) + for _, w := range warnings { + result.Warnings = append(result.Warnings, fmt.Sprintf("%s: %s", rel, w)) + } + } + + sort.Strings(result.Written) + sort.Strings(relpaths) + + configPath := filepath.Join(absDir, "oats-config.yaml") + if _, err := os.Stat(configPath); err == nil { + result.Warnings = append(result.Warnings, "oats-config.yaml already existed and was overwritten as a migration artifact") + } + cfg := discovery.RootConfig{ + Meta: discovery.Meta{Version: discovery.SupportedVersion}, + Cases: relpaths, + } + cfgOut, err := marshalYAML(cfg) + if err != nil { + return nil, err + } + if err := os.WriteFile(configPath, cfgOut, 0o644); err != nil { + return nil, fmt.Errorf("write %s: %w", configPath, err) + } + result.Config = configPath + return result, nil +} + // ConvertDefinition maps a legacy v1/v1.5-style OATS definition into the -// current case yaml shape. Unsupported fields are dropped with warnings. +// current case yaml shape. Unsupported fields are dropped with warnings. The +// legacy docker-compose/kubernetes fixture is carried over as a case-local +// fixture: block. func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Case, []string, error) { var warnings []string c := &casefile.Case{ @@ -45,33 +151,13 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Cas } selectedMatrix := (*model.Matrix)(nil) - if def.Kubernetes != nil { - c.Seed.Type = "app" - warnings = append(warnings, "legacy kubernetes fixture migrated as an app-backed case; paste the suggested fixture: block below into oats-config.yaml") - warnings = append(warnings, kubernetesFixtureHint(name, def)) - } if len(def.Matrix) > 0 { if len(def.Matrix) == 1 { selectedMatrix = &def.Matrix[0] c.Name = fmt.Sprintf("%s [%s]", name, selectedMatrix.Name) warnings = append(warnings, fmt.Sprintf("flattened single matrix entry %q into the migrated case", selectedMatrix.Name)) - if selectedMatrix.DockerCompose != nil { - if len(selectedMatrix.DockerCompose.Files) == 0 { - return nil, warnings, fmt.Errorf("matrix docker-compose present but no files declared") - } - c.Seed.Type = "app" - c.Seed.Compose = selectedMatrix.DockerCompose.Files[0] - warnings = append(warnings, "single matrix docker-compose fixture selected; paste the suggested fixture: block below into oats-config.yaml") - warnings = append(warnings, matrixFixtureHint(c.Name, *selectedMatrix)) - } - if selectedMatrix.Kubernetes != nil { - c.Seed.Type = "app" - warnings = append(warnings, "single matrix kubernetes fixture selected; paste the suggested fixture: block below into oats-config.yaml") - warnings = append(warnings, matrixFixtureHint(c.Name, *selectedMatrix)) - } } else { warnings = append(warnings, "matrix definitions are not migrated automatically when multiple entries exist; convert expanded matrix cases manually") - warnings = append(warnings, matrixExpansionHint(name, def.Matrix)) } } if len(def.Include) > 0 { @@ -81,17 +167,24 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Cas warnings = append(warnings, "compose-logs assertions are not migrated") } - if def.DockerCompose != nil { + // Guard against a docker-compose block with no files before building the + // fixture, so the error message stays specific to its source. + if selectedMatrix != nil && selectedMatrix.DockerCompose != nil && len(selectedMatrix.DockerCompose.Files) == 0 { + return nil, warnings, fmt.Errorf("matrix docker-compose present but no files declared") + } + if def.DockerCompose != nil && len(def.DockerCompose.Files) == 0 && + (selectedMatrix == nil || selectedMatrix.DockerCompose == nil) { + return nil, warnings, fmt.Errorf("docker-compose present but no files declared") + } + + c.Fixture = fixtureFor(def, selectedMatrix) + switch { + case c.Fixture != nil: c.Seed.Type = "app" - if len(def.DockerCompose.Files) == 0 { - return nil, warnings, fmt.Errorf("docker-compose present but no files declared") + if c.Fixture.Compose != nil { + warnings = append(warnings, "for parallel-safe app-seed runs, set fixture.compose.app_service + app_port (not derivable from the legacy file)") } - c.Seed.Compose = def.DockerCompose.Files[0] - if len(def.DockerCompose.Files) > 1 || len(def.DockerCompose.Environment) > 0 { - warnings = append(warnings, "legacy docker-compose fixture uses suite-level config richer than case-level seed.compose; paste the suggested fixture: block below into oats-config.yaml") - warnings = append(warnings, composeFixtureHint(name, def)) - } - } else if def.Kubernetes == nil && selectedMatrix == nil { + default: warnings = append(warnings, "no docker-compose fixture found; defaulting seed.type to inline-otlp placeholder") c.Seed.Type = "inline-otlp" c.Seed.Traces = []casefile.SeedTrace{{Service: "migrated-service", Spans: []casefile.SeedSpan{{Name: "replace-me"}}}} @@ -169,6 +262,45 @@ func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Cas return c, warnings, nil } +// fixtureFor builds the case-local fixture: block from a legacy definition. A +// single-matrix override (selectedMatrix) takes precedence over the suite-level +// blocks. Returns nil when the legacy definition declares no fixture. +func fixtureFor(def model.TestCaseDefinition, selectedMatrix *model.Matrix) *casefile.FixtureConfig { + dc := def.DockerCompose + k8s := def.Kubernetes + if selectedMatrix != nil { + if selectedMatrix.DockerCompose != nil { + dc = selectedMatrix.DockerCompose + } + if selectedMatrix.Kubernetes != nil { + k8s = selectedMatrix.Kubernetes + } + } + + if dc != nil { + // Legacy files carry no app_service/app_port; a warning covers it. + compose := &casefile.ComposeFixture{Env: dc.Environment} + if len(dc.Files) == 1 { + compose.File = dc.Files[0] + } else { + compose.Files = dc.Files + } + return &casefile.FixtureConfig{Compose: compose} + } + if k8s != nil { + return &casefile.FixtureConfig{K3D: &casefile.K3DFixture{ + K8sDir: k8s.Dir, + AppService: k8s.AppService, + AppDockerFile: k8s.AppDockerFile, + AppDockerContext: k8s.AppDockerContext, + AppDockerTag: k8s.AppDockerTag, + AppPort: k8s.AppDockerPort, + ImportImages: k8s.ImportImages, + }} + } + return nil +} + func convertSignal(label string, s model.ExpectedSignal) (casefile.AssertionCommon, []string) { var warnings []string out := casefile.AssertionCommon{} @@ -262,112 +394,4 @@ func keepForMatrix(matrixCondition string, selected *model.Matrix) bool { return re.MatchString(selected.Name) } -func composeFixtureHint(name string, def model.TestCaseDefinition) string { - fixtureName := slug(name) - var b strings.Builder - fmt.Fprintf(&b, "suggested oats-config.yaml fixture snippet for %q:\n", name) - fmt.Fprintf(&b, "fixture:\n") - fmt.Fprintf(&b, " %s:\n", fixtureName) - fmt.Fprintf(&b, " compose:\n") - if len(def.DockerCompose.Files) == 1 { - fmt.Fprintf(&b, " file: %s\n", def.DockerCompose.Files[0]) - } else if len(def.DockerCompose.Files) > 1 { - fmt.Fprintf(&b, " files: [%s]\n", quotedList(def.DockerCompose.Files)) - } - if len(def.DockerCompose.Environment) > 0 { - fmt.Fprintf(&b, " env: [%s]\n", quotedList(def.DockerCompose.Environment)) - } - return strings.TrimRight(b.String(), "\n") -} - -func kubernetesFixtureHint(name string, def model.TestCaseDefinition) string { - fixtureName := slug(name) - k := def.Kubernetes - var b strings.Builder - fmt.Fprintf(&b, "suggested oats-config.yaml fixture snippet for %q:\n", name) - fmt.Fprintf(&b, "fixture:\n") - fmt.Fprintf(&b, " %s:\n", fixtureName) - fmt.Fprintf(&b, " k3d:\n") - fmt.Fprintf(&b, " k8s_dir: %s\n", k.Dir) - fmt.Fprintf(&b, " app_service: %s\n", k.AppService) - fmt.Fprintf(&b, " app_docker_file: %s\n", k.AppDockerFile) - if k.AppDockerContext != "" { - fmt.Fprintf(&b, " app_docker_context: %s\n", k.AppDockerContext) - } - fmt.Fprintf(&b, " app_docker_tag: %s\n", k.AppDockerTag) - fmt.Fprintf(&b, " app_port: %d\n", k.AppDockerPort) - if len(k.ImportImages) > 0 { - fmt.Fprintf(&b, " import_images: [%s]\n", quotedList(k.ImportImages)) - } - return strings.TrimRight(b.String(), "\n") -} - -func matrixFixtureHint(name string, m model.Matrix) string { - def := model.TestCaseDefinition{ - DockerCompose: m.DockerCompose, - Kubernetes: m.Kubernetes, - } - if m.DockerCompose != nil { - return composeFixtureHint(name, def) - } - if m.Kubernetes != nil { - return kubernetesFixtureHint(name, def) - } - return fmt.Sprintf("matrix %q has no fixture override", m.Name) -} - -func matrixExpansionHint(name string, matrix []model.Matrix) string { - var b strings.Builder - fmt.Fprintf(&b, "suggested matrix expansion for %q:\n", name) - for _, m := range matrix { - fmt.Fprintf(&b, "- %s\n", m.Name) - if m.DockerCompose != nil || m.Kubernetes != nil { - fixture := indentLines(matrixFixtureHint(fmt.Sprintf("%s [%s]", name, m.Name), m), " ") - fmt.Fprintf(&b, "%s\n", fixture) - } - } - return strings.TrimRight(b.String(), "\n") -} - -func indentLines(s, prefix string) string { - lines := strings.Split(s, "\n") - for i := range lines { - lines[i] = prefix + lines[i] - } - return strings.Join(lines, "\n") -} - -func quotedList(items []string) string { - quoted := make([]string, 0, len(items)) - for _, item := range items { - quoted = append(quoted, fmt.Sprintf("%q", item)) - } - return strings.Join(quoted, ", ") -} - -func slug(s string) string { - s = strings.ToLower(s) - s = strings.ReplaceAll(s, "_", "-") - s = strings.ReplaceAll(s, " ", "-") - var b strings.Builder - lastDash := false - for _, r := range s { - keep := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') - if keep { - b.WriteRune(r) - lastDash = false - continue - } - if !lastDash { - b.WriteByte('-') - lastDash = true - } - } - out := strings.Trim(b.String(), "-") - if out == "" { - return "migrated" - } - return out -} - func strPtr(s string) *string { return &s } diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index f6457d1f..71b1b372 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/grafana/oats/casefile" + "github.com/grafana/oats/discovery" "github.com/grafana/oats/model" "github.com/grafana/oats/testhelpers/kubernetes" ) @@ -50,9 +51,12 @@ func TestConvertDefinition_MapsSignalsToMatchSchema(t *testing.T) { if err != nil { fatalf(t, "ConvertDefinition: %v", err) } - if c.Seed.Type != "app" || c.Seed.Compose != "docker-compose.yml" { + if c.Seed.Type != "app" { fatalf(t, "unexpected seed mapping: %+v", c.Seed) } + if c.Fixture == nil || c.Fixture.Compose == nil || c.Fixture.Compose.File != "docker-compose.yml" { + fatalf(t, "expected case-local compose fixture, got %+v", c.Fixture) + } if c.Interval != 500*time.Millisecond { fatalf(t, "expected interval to carry over, got %v", c.Interval) } @@ -123,15 +127,8 @@ func TestConvertFile_MatrixSampleIsParseable(t *testing.T) { fatalf(t, "ConvertFile matrix sample: %v", err) } joined := strings.Join(warnings, "\n") - for _, want := range []string{ - `matrix definitions are not migrated automatically when multiple entries exist`, - `suggested matrix expansion for "matrix test.oats"`, - `matrix-test-oats-docker:`, - `matrix-test-oats-k8s:`, - } { - if !strings.Contains(joined, want) { - fatalf(t, "expected matrix warning to contain %q:\n%s", want, joined) - } + if !strings.Contains(joined, "matrix definitions are not migrated automatically when multiple entries exist") { + fatalf(t, "expected multi-matrix warning:\n%s", joined) } c, err := casefile.Parse(out) if err != nil { @@ -167,8 +164,10 @@ expected: if err != nil { fatalf(t, "ConvertFile custom checks: %v", err) } - if len(warnings) != 0 { - fatalf(t, "expected no warnings for straightforward custom checks, got %v", warnings) + // A compose fixture always warns that app_service/app_port must be set by + // hand; that is the only expected warning here. + if len(warnings) != 1 || !strings.Contains(warnings[0], "app_service + app_port") { + fatalf(t, "expected only the app_service warning for straightforward custom checks, got %v", warnings) } c, err := casefile.Parse(out) if err != nil { @@ -182,7 +181,7 @@ expected: } } -func TestConvertDefinition_KubernetesProducesFixtureHint(t *testing.T) { +func TestConvertDefinition_KubernetesProducesFixtureBlock(t *testing.T) { def := model.TestCaseDefinition{ Kubernetes: &kubernetes.Kubernetes{ Dir: "k8s", @@ -203,28 +202,23 @@ func TestConvertDefinition_KubernetesProducesFixtureHint(t *testing.T) { }, } - c, warnings, err := ConvertDefinition(def, "k8s case") + c, _, err := ConvertDefinition(def, "k8s case") if err != nil { fatalf(t, "ConvertDefinition kubernetes: %v", err) } if c.Seed.Type != "app" { fatalf(t, "expected app seed for kubernetes migration, got %+v", c.Seed) } - joined := strings.Join(warnings, "\n") - for _, want := range []string{ - `k8s-case:`, - `k3d:`, - `k8s_dir: k8s`, - `app_service: dice`, - `app_docker_file: Dockerfile`, - `app_docker_context: ..`, - `app_docker_tag: dice:test`, - `app_port: 8080`, - `import_images: ["busybox:latest"]`, - } { - if !strings.Contains(joined, want) { - fatalf(t, "expected kubernetes migration warning to contain %q:\n%s", want, joined) - } + if c.Fixture == nil || c.Fixture.K3D == nil { + fatalf(t, "expected case-local k3d fixture, got %+v", c.Fixture) + } + k := c.Fixture.K3D + if k.K8sDir != "k8s" || k.AppService != "dice" || k.AppDockerFile != "Dockerfile" || + k.AppDockerContext != ".." || k.AppDockerTag != "dice:test" || k.AppPort != 8080 { + fatalf(t, "unexpected k3d fixture: %+v", k) + } + if len(k.ImportImages) != 1 || k.ImportImages[0] != "busybox:latest" { + fatalf(t, "unexpected import images: %+v", k.ImportImages) } } @@ -276,19 +270,16 @@ func TestConvertDefinition_FlattensSingleMatrixEntry(t *testing.T) { if got := len(c.Expected.Logs); got != 2 { fatalf(t, "expected 2 kept log assertions, got %d (%+v)", got, c.Expected.Logs) } + if c.Fixture == nil || c.Fixture.Compose == nil || c.Fixture.Compose.File != "docker-compose.yml" { + fatalf(t, "expected single-matrix compose fixture, got %+v", c.Fixture) + } joined := strings.Join(warnings, "\n") - for _, want := range []string{ - `flattened single matrix entry "docker"`, - `matrix-case-docker:`, - `file: docker-compose.yml`, - } { - if !strings.Contains(joined, want) { - fatalf(t, "expected warning to contain %q:\n%s", want, joined) - } + if !strings.Contains(joined, `flattened single matrix entry "docker"`) { + fatalf(t, "expected flatten warning:\n%s", joined) } } -func TestConvertDefinition_MultiMatrixEmitsExpansionHint(t *testing.T) { +func TestConvertDefinition_MultiMatrixWarnsAndFallsBack(t *testing.T) { def := model.TestCaseDefinition{ Matrix: []model.Matrix{ { @@ -319,21 +310,21 @@ func TestConvertDefinition_MultiMatrixEmitsExpansionHint(t *testing.T) { }, } - _, warnings, err := ConvertDefinition(def, "matrix case") + c, warnings, err := ConvertDefinition(def, "matrix case") if err != nil { fatalf(t, "ConvertDefinition multi matrix: %v", err) } + // Multiple matrix entries are not migrated automatically: no fixture is + // selected and the case falls back to the inline-otlp placeholder. + if c.Fixture != nil { + fatalf(t, "expected no fixture for multi-matrix case, got %+v", c.Fixture) + } + if c.Seed.Type != "inline-otlp" { + fatalf(t, "expected inline-otlp fallback, got %+v", c.Seed) + } joined := strings.Join(warnings, "\n") - for _, want := range []string{ - `suggested matrix expansion for "matrix case"`, - `- docker`, - `matrix-case-docker:`, - `- k8s`, - `matrix-case-k8s:`, - } { - if !strings.Contains(joined, want) { - fatalf(t, "expected warning to contain %q:\n%s", want, joined) - } + if !strings.Contains(joined, "matrix definitions are not migrated automatically when multiple entries exist") { + fatalf(t, "expected multi-matrix warning:\n%s", joined) } } @@ -359,6 +350,78 @@ func TestConvertDefinition_MigratesCustomChecks(t *testing.T) { } } +func TestConvertTree_RewritesCasesAndWritesConfig(t *testing.T) { + dir := t.TempDir() + composeCase := filepath.Join(dir, "compose.oats.yaml") + if err := os.WriteFile(composeCase, []byte(` +oats-schema-version: 2 +docker-compose: + files: + - docker-compose.yml +expected: + metrics: + - promql: up + value: ">= 1" +`), 0o644); err != nil { + fatalf(t, "WriteFile compose: %v", err) + } + nested := filepath.Join(dir, "nested") + if err := os.MkdirAll(nested, 0o755); err != nil { + fatalf(t, "MkdirAll: %v", err) + } + minimalCase := filepath.Join(nested, "minimal.oats.yaml") + if err := os.WriteFile(minimalCase, []byte(` +oats-schema-version: 2 +expected: + custom-checks: + - script: ./verify.sh +`), 0o644); err != nil { + fatalf(t, "WriteFile minimal: %v", err) + } + + res, err := ConvertTree(dir) + if err != nil { + fatalf(t, "ConvertTree: %v", err) + } + if len(res.Written) != 2 { + fatalf(t, "expected 2 written cases, got %v", res.Written) + } + + // Both legacy files must now be valid v3 cases parseable in place. + compose, err := casefile.Load(composeCase) + if err != nil { + fatalf(t, "compose case should be v3 after migration: %v", err) + } + if compose.OatsSchemaVersion != casefile.SchemaVersion { + fatalf(t, "expected schema version %d, got %d", casefile.SchemaVersion, compose.OatsSchemaVersion) + } + if compose.Fixture == nil || compose.Fixture.Compose == nil || compose.Fixture.Compose.File != "docker-compose.yml" { + fatalf(t, "expected migrated compose fixture, got %+v", compose.Fixture) + } + if _, err := casefile.Load(minimalCase); err != nil { + fatalf(t, "minimal case should be v3 after migration: %v", err) + } + + // The generated config must list both cases as explicit relative paths. + configPath := filepath.Join(dir, "oats-config.yaml") + if res.Config != configPath { + fatalf(t, "unexpected config path: %q", res.Config) + } + cfg, err := discovery.Load(configPath) + if err != nil { + fatalf(t, "discovery.Load should accept generated config: %v", err) + } + want := []string{"compose.oats.yaml", "nested/minimal.oats.yaml"} + if len(cfg.Cases) != len(want) { + fatalf(t, "expected cases %v, got %v", want, cfg.Cases) + } + for i, w := range want { + if cfg.Cases[i] != w { + fatalf(t, "expected sorted explicit case %q at %d, got %v", w, i, cfg.Cases) + } + } +} + func fatalf(t *testing.T, format string, args ...any) { t.Helper() t.Fatalf(format, args...) From 38d8ceb306ce2f542e291d473417fe915eb40e56 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 14:37:01 +0000 Subject: [PATCH 114/124] refactor(schema): drop per-case oats-schema-version, single config version Cases in v3 are only loaded through the explicit oats-config.yaml, so a per-case version marker is redundant with the config's meta.version. Remove oats-schema-version from the case schema (casefile drops the field, const, and check) and make meta.version the single version, bumped 2 -> 3 to match "OATS v3" and end the confusing 2-vs-3 split. Stripped the tag from all v3 example/e2e cases, bumped configs to version 3, and updated migrate output + docs. Legacy input files (read by `oats migrate`) keep their own oats-schema-version: 2 in the untouched legacy `yaml/` parser. Signed-off-by: Gregor Zeitlinger --- AGENTS.md | 8 ++-- README.md | 1 - UPGRADING.md | 16 ++++---- casefile/case.go | 22 ++++------- casefile/case_test.go | 37 ++++--------------- discovery/discovery.go | 6 ++- discovery/discovery_test.go | 31 +++++++--------- docs/case-reference.md | 3 +- examples/fixtures/cases/compose/rolldice.yaml | 1 - examples/fixtures/cases/k3d/rolldice.yaml | 1 - examples/fixtures/oats-config.yaml | 2 +- examples/smoke/cases/custom-check.yaml | 1 - examples/smoke/cases/inline-seed.yaml | 1 - examples/smoke/cases/profile.yaml | 1 - examples/smoke/cases/rolldice.yaml | 1 - examples/smoke/oats-config.yaml | 2 +- fixture/fixture_test.go | 5 +-- internal/cli/integration_test.go | 25 ++++++------- migrate/migrate.go | 5 +-- migrate/migrate_test.go | 5 +-- runner/cache_integration_test.go | 3 +- runner/runner_test.go | 13 ------- .../assert/absent-fail/files/oats-case.yaml | 1 - .../cases/assert/absent/files/oats-case.yaml | 1 - .../assert/contains-fail/files/oats-case.yaml | 1 - .../assert/contains/files/oats-case.yaml | 1 - .../assert/count-fail/files/oats-case.yaml | 1 - .../cases/assert/count/files/oats-case.yaml | 1 - .../assert/match-fail/files/oats-case.yaml | 1 - .../match-regexp-fail/files/oats-case.yaml | 1 - .../assert/match-regexp/files/oats-case.yaml | 1 - .../match-spans-fail/files/oats-case.yaml | 1 - .../not-contains-fail/files/oats-case.yaml | 1 - .../assert/not-contains/files/oats-case.yaml | 1 - .../assert/profile-fail/files/oats-case.yaml | 1 - .../cases/assert/profile/files/oats-case.yaml | 1 - .../assert/regex-fail/files/oats-case.yaml | 1 - .../cases/assert/regex/files/oats-case.yaml | 1 - .../assert/value-fail/files/oats-case.yaml | 1 - .../cases/assert/value/files/oats-case.yaml | 1 - .../cli/cache-clear/files/oats-case.yaml | 1 - .../cli/cache-clear/files/oats-config.yaml | 2 +- .../cases/cli/cache-skip/files/oats-case.yaml | 1 - .../cli/cache-skip/files/oats-config.yaml | 2 +- .../cases/cli/fail-fast/files/01-fail.yaml | 1 - .../cases/cli/fail-fast/files/02-pass.yaml | 1 - .../cli/fail-fast/files/oats-config.yaml | 2 +- tests/e2e/cases/cli/migrate/test.yaml | 4 +- .../custom-check-fail/files/oats-case.yaml | 1 - .../custom/custom-check/files/oats-case.yaml | 1 - .../compose-app-seed/files/oats-case.yaml | 1 - .../compose-logs-fail/files/oats-case.yaml | 1 - .../fixture/compose-logs/files/oats-case.yaml | 1 - .../fixture/k3d-fail/files/oats-case.yaml | 1 - .../fixture/k3d-smoke/files/oats-case.yaml | 1 - .../fixture/parallel-compose/files/alpha.yaml | 1 - .../fixture/parallel-compose/files/beta.yaml | 1 - .../parallel-compose/files/oats-config.yaml | 2 +- .../fixture/remote-basic/files/oats-case.yaml | 1 - .../e2e/cases/fixture/remote-basic/test.yaml | 1 - .../cases/format/ndjson/files/oats-case.yaml | 1 - .../inline-otlp-invalid/files/oats-case.yaml | 1 - .../seed/inline-otlp/files/oats-case.yaml | 1 - tests/e2e/e2e_test.go | 2 +- 64 files changed, 75 insertions(+), 165 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1004c923..3d02da4f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,9 +9,11 @@ OpenTelemetry written in Go. It enables full round-trip testing from instrumented applications to the observability stack (LGTM: Loki, Grafana, Tempo, Prometheus, Mimir, OpenTelemetry Collector). -Test cases are defined in YAML files with `oats-schema-version: 3` and can -validate traces (TraceQL), logs (LogQL), metrics (PromQL), and profiles -(Pyroscope queries) using Docker Compose or Kubernetes backends. +Test cases are defined in YAML files, discovered through an `oats-config.yaml` +whose `meta.version: 3` is the single schema version (cases carry no version +field of their own). They can validate traces (TraceQL), logs (LogQL), metrics +(PromQL), and profiles (Pyroscope queries) using Docker Compose or Kubernetes +backends. ## Build Commands diff --git a/README.md b/README.md index c888ed6e..5e66911a 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,6 @@ Pyroscope) via [`gcx`](https://github.com/grafana/gcx). A case reads like the outcome you care about: ```yaml -oats-schema-version: 3 name: rolldice traces have a route attribute fixture: diff --git a/UPGRADING.md b/UPGRADING.md index 43ca6e15..6c68e5f7 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -23,12 +23,15 @@ removed. For one-off help migrating old cases, use: oats migrate path/to/legacy.yaml ``` -### Case schema: `oats-schema-version: 2` → `3` +### Case schema: version 2 → 3 -The case-yaml assertion shape changed with the gcx-driven runner. Bump the tag -to `3` and update assertions. The `0.5.0` / `0.6.0` notes further down describe -the **version-2** shape (`equals`, `attribute-regexp`, `flamebearers`, -`regexp`); the mappings below take you from that shape to version 3. +The case-yaml assertion shape changed with the gcx-driven runner, and the schema +version moved. A v3 case no longer carries an `oats-schema-version` field: cases +are only ever loaded through `oats-config.yaml`, whose `meta.version: 3` is the +single schema version. Drop the per-case `oats-schema-version` tag and update +assertions. The `0.5.0` / `0.6.0` notes further down describe the **version-2** +shape (`equals`, `attribute-regexp`, `flamebearers`, `regexp`); the mappings +below take you from that shape to version 3. **Logs — regex.** Version 2's `regexp:` becomes `regex:` (scalar or list). `contains` / `not_contains` are also first-class again: @@ -128,7 +131,7 @@ case (the migrator prints both the case yaml and a suggested `fixture:` block): ```yaml # oats-config.yaml meta: - version: 2 + version: 3 suites: - name: rolldice cases: ["cases/*.yaml"] @@ -141,7 +144,6 @@ fixture: ```yaml # cases/rolldice.yaml (v3) -oats-schema-version: 3 name: rolldice seed: type: app diff --git a/casefile/case.go b/casefile/case.go index 6c5e7de9..2f790e38 100644 --- a/casefile/case.go +++ b/casefile/case.go @@ -21,19 +21,15 @@ import ( "go.yaml.in/yaml/v3" ) -// SchemaVersion is the value of the top-level `oats-schema-version` field -// that this loader understands. Cases with any other value are rejected at -// parse time. -const SchemaVersion = 3 - // Case is one entry point yaml file. Cases are independently runnable; -// suites group them via oats-config.yaml. +// suites group them via oats-config.yaml. A case carries no version field of +// its own: cases are only ever loaded through oats-config.yaml, whose +// meta.version is the single schema version (see discovery.SupportedVersion). type Case struct { - OatsSchemaVersion int `yaml:"oats-schema-version"` - Name string `yaml:"name"` - Tags []string `yaml:"tags,omitempty"` - Interval time.Duration `yaml:"interval,omitempty"` - Fixture *FixtureConfig `yaml:"fixture,omitempty"` + Name string `yaml:"name"` + Tags []string `yaml:"tags,omitempty"` + Interval time.Duration `yaml:"interval,omitempty"` + Fixture *FixtureConfig `yaml:"fixture,omitempty"` Seed Seed `yaml:"seed"` Input []Input `yaml:"input,omitempty"` @@ -371,10 +367,6 @@ func Parse(data []byte) (*Case, error) { // Called automatically by Parse; exported for tests that construct Cases // programmatically. func (c *Case) Validate() error { - if c.OatsSchemaVersion != SchemaVersion { - return fmt.Errorf("unsupported oats-schema-version '%d' required version is '%d'", - c.OatsSchemaVersion, SchemaVersion) - } if c.Name == "" { return fmt.Errorf("name: required, non-empty") } diff --git a/casefile/case_test.go b/casefile/case_test.go index c4dcc6ab..e985ac0f 100644 --- a/casefile/case_test.go +++ b/casefile/case_test.go @@ -8,7 +8,6 @@ import ( func TestParse_AppSeed(t *testing.T) { src := []byte(` -oats-schema-version: 3 name: rolldice traces have route attribute interval: 250ms seed: @@ -50,7 +49,6 @@ expected: func TestParse_InlineOTLPSeed(t *testing.T) { src := []byte(` -oats-schema-version: 3 name: gcx returns seeded trace seed: type: inline-otlp @@ -77,7 +75,6 @@ expected: func TestParse_MatchAssertions(t *testing.T) { src := []byte(` -oats-schema-version: 3 name: structured match seed: type: app @@ -116,7 +113,6 @@ expected: func TestParse_StringListScalarOrList(t *testing.T) { src := []byte(` -oats-schema-version: 3 name: text assertions seed: type: app @@ -146,7 +142,6 @@ expected: func TestParse_LegacyAttributeMapStillAccepted(t *testing.T) { src := []byte(` -oats-schema-version: 3 name: legacy attrs seed: type: app @@ -177,7 +172,6 @@ expected: func TestParse_CustomChecks(t *testing.T) { src := []byte(` -oats-schema-version: 3 name: custom checks seed: type: app @@ -196,7 +190,6 @@ expected: func TestParse_RejectsUnknownFields(t *testing.T) { src := []byte(` -oats-schema-version: 3 name: typo seed: type: app @@ -213,16 +206,8 @@ expected: } } -func TestValidate_MissingOats(t *testing.T) { - c := &Case{Name: "x", Seed: Seed{Type: "app", Compose: "y"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} - err := c.Validate() - if err == nil || !strings.Contains(err.Error(), "oats-schema-version") { - t.Errorf("expected oats version error, got %v", err) - } -} - func TestValidate_MissingName(t *testing.T) { - c := &Case{OatsSchemaVersion: 3, Seed: Seed{Type: "app", Compose: "y"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + c := &Case{Seed: Seed{Type: "app", Compose: "y"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() if err == nil || !strings.Contains(err.Error(), "name:") { t.Errorf("expected name error, got %v", err) @@ -230,7 +215,7 @@ func TestValidate_MissingName(t *testing.T) { } func TestValidate_UnknownSeedType(t *testing.T) { - c := &Case{OatsSchemaVersion: 3, Name: "x", Seed: Seed{Type: "weird"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + c := &Case{Name: "x", Seed: Seed{Type: "weird"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() if err == nil || !strings.Contains(err.Error(), "seed.type") { t.Errorf("expected seed type error, got %v", err) @@ -238,7 +223,7 @@ func TestValidate_UnknownSeedType(t *testing.T) { } func TestValidate_AppSeedAllowsFixtureManagedBoot(t *testing.T) { - c := &Case{OatsSchemaVersion: 3, Name: "x", Seed: Seed{Type: "app"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + c := &Case{Name: "x", Seed: Seed{Type: "app"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() if err != nil { t.Errorf("expected app seed without compose to validate, got %v", err) @@ -246,7 +231,7 @@ func TestValidate_AppSeedAllowsFixtureManagedBoot(t *testing.T) { } func TestValidate_InlineOTLPNeedsPayload(t *testing.T) { - c := &Case{OatsSchemaVersion: 3, Name: "x", Seed: Seed{Type: "inline-otlp"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} + c := &Case{Name: "x", Seed: Seed{Type: "inline-otlp"}, Expected: Expected{Traces: []TraceAssertion{{TraceQL: "{}"}}}} err := c.Validate() if err == nil || !strings.Contains(err.Error(), "at least one trace") { t.Errorf("expected payload error, got %v", err) @@ -254,7 +239,7 @@ func TestValidate_InlineOTLPNeedsPayload(t *testing.T) { } func TestValidate_NoExpectations(t *testing.T) { - c := &Case{OatsSchemaVersion: 3, Name: "x", Seed: Seed{Type: "app", Compose: "y"}} + c := &Case{Name: "x", Seed: Seed{Type: "app", Compose: "y"}} err := c.Validate() if err == nil || !strings.Contains(err.Error(), "expected:") { t.Errorf("expected 'no expectations' error, got %v", err) @@ -263,10 +248,9 @@ func TestValidate_NoExpectations(t *testing.T) { func TestValidate_CustomCheckOnlyCase(t *testing.T) { c := &Case{ - OatsSchemaVersion: 3, - Name: "x", - Seed: Seed{Type: "app"}, - Expected: Expected{Custom: []CustomCheck{{Script: "./verify.sh"}}}, + Name: "x", + Seed: Seed{Type: "app"}, + Expected: Expected{Custom: []CustomCheck{{Script: "./verify.sh"}}}, } if err := c.Validate(); err != nil { t.Fatalf("expected custom-check-only case to validate, got %v", err) @@ -275,7 +259,6 @@ func TestValidate_CustomCheckOnlyCase(t *testing.T) { func TestValidate_RejectsEmptyCustomCheck(t *testing.T) { _, err := Parse([]byte(` -oats-schema-version: 3 name: bad custom seed: type: app @@ -290,7 +273,6 @@ expected: func TestValidate_RejectsInputWithoutPath(t *testing.T) { _, err := Parse([]byte(` -oats-schema-version: 3 name: bad input seed: type: app @@ -309,7 +291,6 @@ expected: func TestValidate_RejectsUnknownMatchType(t *testing.T) { _, err := Parse([]byte(` -oats-schema-version: 3 name: bad match type seed: type: app @@ -328,7 +309,6 @@ expected: func TestValidate_RejectsDuplicateAttributeKeys(t *testing.T) { _, err := Parse([]byte(` -oats-schema-version: 3 name: duplicate attribute keys seed: type: app @@ -350,7 +330,6 @@ expected: func TestValidate_RejectsInvalidMatchRegexp(t *testing.T) { _, err := Parse([]byte(` -oats-schema-version: 3 name: bad regexp seed: type: app diff --git a/discovery/discovery.go b/discovery/discovery.go index 5a57e586..dc042ad9 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -51,8 +51,10 @@ type CacheConfig struct { TTLDays int `yaml:"ttl_days,omitempty"` // zero → use runtime default } -// SupportedVersion is the value of meta.version that this binary parses. -const SupportedVersion = 2 +// SupportedVersion is the value of meta.version that this binary parses. It is +// the single schema version for a v3 project: cases carry no version field of +// their own. +const SupportedVersion = 3 // Load reads an oats-config.yaml from disk. func Load(path string) (*RootConfig, error) { diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index a52fc67c..f84b6dee 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -20,8 +20,7 @@ func writeFile(t *testing.T, dir, rel, body string) { } } -const validCaseYAML = `oats-schema-version: 3 -name: %s +const validCaseYAML = `name: %s seed: type: app compose: docker-compose.app.yml @@ -35,7 +34,7 @@ func TestLoad_ValidConfig(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: lgtm cases: ["cases/*.yaml"] @@ -76,7 +75,7 @@ func TestLoad_RejectsUnknownKey(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 typooo: boom suites: - name: x @@ -92,7 +91,7 @@ func TestPlanRun_FilterByTag(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: alpha cases: ["a.yaml"] @@ -122,7 +121,7 @@ func TestSummary_TopLevelCases(t *testing.T) { writeFile(t, dir, "oats-config.yaml", ` cases: ["cases/*.yaml"] meta: - version: 2 + version: 3 `) writeFile(t, dir, "cases/a.yaml", strings.Replace(validCaseYAML, "%s", "case-a", 1)) @@ -144,7 +143,7 @@ func TestPlanRun_FilterBySuiteName(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: alpha cases: ["a.yaml"] @@ -169,7 +168,7 @@ suites: func TestValidate_EmptyFixtureRejected(t *testing.T) { cfg := &RootConfig{ - Meta: Meta{Version: 2}, + Meta: Meta{Version: 3}, Suites: []SuiteConfig{{ Name: "s", Cases: []string{"a.yaml"}, Fixture: "x", }}, @@ -183,7 +182,7 @@ func TestValidate_EmptyFixtureRejected(t *testing.T) { func TestValidate_MultipleFixtureBlocksRejected(t *testing.T) { cfg := &RootConfig{ - Meta: Meta{Version: 2}, + Meta: Meta{Version: 3}, Suites: []SuiteConfig{{ Name: "s", Cases: []string{"a.yaml"}, Fixture: "x", }}, @@ -200,7 +199,7 @@ func TestValidate_MultipleFixtureBlocksRejected(t *testing.T) { func TestValidate_FixtureRefNotDefined(t *testing.T) { cfg := &RootConfig{ - Meta: Meta{Version: 2}, + Meta: Meta{Version: 3}, Suites: []SuiteConfig{{ Name: "s", Cases: []string{"a.yaml"}, Fixture: "missing", }}, @@ -213,7 +212,7 @@ func TestValidate_FixtureRefNotDefined(t *testing.T) { func TestValidate_RemoteRequiresEndpoint(t *testing.T) { cfg := &RootConfig{ - Meta: Meta{Version: 2}, + Meta: Meta{Version: 3}, Suites: []SuiteConfig{{Name: "s", Cases: []string{"a.yaml"}}}, Fixture: map[string]casefile.FixtureConfig{"r": {Remote: &casefile.RemoteFixture{}}}, } @@ -224,7 +223,7 @@ func TestValidate_RemoteRequiresEndpoint(t *testing.T) { func TestValidate_ComposeFilesConflict(t *testing.T) { cfg := &RootConfig{ - Meta: Meta{Version: 2}, + Meta: Meta{Version: 3}, Suites: []SuiteConfig{{ Name: "s", Cases: []string{"a.yaml"}, Fixture: "c", }}, @@ -244,7 +243,7 @@ func TestValidate_ComposeFilesConflict(t *testing.T) { func TestValidate_K3DRequiresFields(t *testing.T) { cfg := &RootConfig{ - Meta: Meta{Version: 2}, + Meta: Meta{Version: 3}, Suites: []SuiteConfig{{ Name: "s", Cases: []string{"a.yaml"}, Fixture: "k", }}, @@ -261,7 +260,7 @@ func TestPlanRun_EmptyGlobIsAnError(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: s cases: ["nope/*.yaml"] @@ -345,10 +344,9 @@ func TestLoadTopLevelCases(t *testing.T) { writeFile(t, dir, "oats-config.yaml", ` cases: ["cases/a.yaml", "cases/b.yaml"] meta: - version: 2 + version: 3 `) writeFile(t, dir, "cases/a.yaml", ` -oats-schema-version: 3 name: a seed: { type: app } expected: @@ -357,7 +355,6 @@ expected: absent: true `) writeFile(t, dir, "cases/b.yaml", ` -oats-schema-version: 3 name: b seed: { type: app } expected: diff --git a/docs/case-reference.md b/docs/case-reference.md index c3f6536f..3536b676 100644 --- a/docs/case-reference.md +++ b/docs/case-reference.md @@ -9,7 +9,7 @@ CLI summary see the [README](../README.md); for running in CI see ```yaml meta: - version: 2 # oats-config.yaml schema version (distinct from a case's oats-schema-version) + version: 3 # the single OATS schema version; cases carry no version field of their own suites: - cases: ["examples/smoke/cases/*.yaml"] cache: @@ -22,7 +22,6 @@ block (the common one-case-per-suite shape). ## Case yaml ```yaml -oats-schema-version: 3 name: rolldice traces have route attribute fixture: diff --git a/examples/fixtures/cases/compose/rolldice.yaml b/examples/fixtures/cases/compose/rolldice.yaml index 8f9337fc..b05545f5 100644 --- a/examples/fixtures/cases/compose/rolldice.yaml +++ b/examples/fixtures/cases/compose/rolldice.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: compose fixture app smoke seed: type: app diff --git a/examples/fixtures/cases/k3d/rolldice.yaml b/examples/fixtures/cases/k3d/rolldice.yaml index 7aea5d75..71f649ea 100644 --- a/examples/fixtures/cases/k3d/rolldice.yaml +++ b/examples/fixtures/cases/k3d/rolldice.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: k3d fixture app smoke seed: type: app diff --git a/examples/fixtures/oats-config.yaml b/examples/fixtures/oats-config.yaml index 1d5f0f23..dcebca20 100644 --- a/examples/fixtures/oats-config.yaml +++ b/examples/fixtures/oats-config.yaml @@ -1,5 +1,5 @@ meta: - version: 2 + version: 3 suites: - name: compose-app cases: ["cases/compose/*.yaml"] diff --git a/examples/smoke/cases/custom-check.yaml b/examples/smoke/cases/custom-check.yaml index 6b3ba892..9790ab95 100644 --- a/examples/smoke/cases/custom-check.yaml +++ b/examples/smoke/cases/custom-check.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: custom check smoke seed: type: app diff --git a/examples/smoke/cases/inline-seed.yaml b/examples/smoke/cases/inline-seed.yaml index 0f147065..5690fee4 100644 --- a/examples/smoke/cases/inline-seed.yaml +++ b/examples/smoke/cases/inline-seed.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: inline seed smoke seed: type: inline-otlp diff --git a/examples/smoke/cases/profile.yaml b/examples/smoke/cases/profile.yaml index 2f613447..862cd30e 100644 --- a/examples/smoke/cases/profile.yaml +++ b/examples/smoke/cases/profile.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: profile smoke seed: type: app diff --git a/examples/smoke/cases/rolldice.yaml b/examples/smoke/cases/rolldice.yaml index d03d6598..ea2d1285 100644 --- a/examples/smoke/cases/rolldice.yaml +++ b/examples/smoke/cases/rolldice.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: rolldice smoke input: - path: /rolldice?rolls=5 diff --git a/examples/smoke/oats-config.yaml b/examples/smoke/oats-config.yaml index 407a15fd..f5cd10a0 100644 --- a/examples/smoke/oats-config.yaml +++ b/examples/smoke/oats-config.yaml @@ -1,5 +1,5 @@ meta: - version: 2 + version: 3 suites: - name: smoke cases: ["cases/*.yaml"] diff --git a/fixture/fixture_test.go b/fixture/fixture_test.go index 511acda7..49608951 100644 --- a/fixture/fixture_test.go +++ b/fixture/fixture_test.go @@ -280,7 +280,7 @@ func TestSupportsParallel_ComposeTemplateLGTM(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: parallel-safe cases: ["cases/*.yaml"] @@ -296,8 +296,7 @@ fixture: image: alpine command: ["sh", "-c", "sleep 1"] `) - writeFile(t, dir, "cases/a.yaml", `oats-schema-version: 3 -name: a + writeFile(t, dir, "cases/a.yaml", `name: a seed: type: inline-otlp logs: diff --git a/internal/cli/integration_test.go b/internal/cli/integration_test.go index 53d9274e..596ab00e 100644 --- a/internal/cli/integration_test.go +++ b/internal/cli/integration_test.go @@ -147,7 +147,7 @@ func TestIntegration_FullPipelineWithFakeGCX(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: smoke cases: ["cases/*.yaml"] @@ -157,8 +157,7 @@ fixture: remote: endpoint: "REPLACED_AT_RUNTIME" `) - writeFile(t, dir, "cases/inline.yaml", `oats-schema-version: 3 -name: inline seed end-to-end + writeFile(t, dir, "cases/inline.yaml", `name: inline seed end-to-end seed: type: inline-otlp traces: @@ -269,7 +268,7 @@ func TestIntegration_AppSeedWithRemoteFixtureAndInput(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: smoke cases: ["cases/*.yaml"] @@ -279,8 +278,7 @@ fixture: remote: endpoint: "http://localhost:4318" `) - writeFile(t, dir, "cases/app.yaml", `oats-schema-version: 3 -name: app seed end-to-end + writeFile(t, dir, "cases/app.yaml", `name: app seed end-to-end seed: type: app input: @@ -355,7 +353,7 @@ func TestIntegration_ProfileQueryWithFakeGCX(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: profiles cases: ["cases/*.yaml"] @@ -365,8 +363,7 @@ fixture: remote: endpoint: "http://localhost:4318" `) - writeFile(t, dir, "cases/profile.yaml", `oats-schema-version: 3 -name: profile query end-to-end + writeFile(t, dir, "cases/profile.yaml", `name: profile query end-to-end seed: type: app expected: @@ -443,7 +440,7 @@ expected: dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: migrated-profile cases: ["cases/*.yaml"] @@ -530,7 +527,7 @@ expected: dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: migrated cases: ["cases/*.yaml"] @@ -608,7 +605,7 @@ expected: dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: migrated-custom cases: ["cases/*.yaml"] @@ -681,7 +678,7 @@ expected: dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: migrated-custom-path cases: ["cases/*.yaml"] @@ -776,7 +773,7 @@ expected: dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` meta: - version: 2 + version: 3 suites: - name: migrated-matrix cases: ["cases/*.yaml"] diff --git a/migrate/migrate.go b/migrate/migrate.go index 7cda8dd4..31ee73c3 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -145,9 +145,8 @@ func ConvertTree(dir string) (*TreeResult, error) { func ConvertDefinition(def model.TestCaseDefinition, name string) (*casefile.Case, []string, error) { var warnings []string c := &casefile.Case{ - OatsSchemaVersion: casefile.SchemaVersion, - Name: name, - Interval: def.Interval, + Name: name, + Interval: def.Interval, } selectedMatrix := (*model.Matrix)(nil) diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index 71b1b372..e199a175 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -113,7 +113,7 @@ func TestConvertFile_RendersYAML(t *testing.T) { fatalf(t, "expected at least one warning for flattened include or fixture migration") } text := string(out) - for _, want := range []string{"oats-schema-version: 3", "seed:", "input:", "path: /stock", "match_spans:", "match_type: regexp", "key: db.system", "value: h2", "promql: foo"} { + for _, want := range []string{"seed:", "input:", "path: /stock", "match_spans:", "match_type: regexp", "key: db.system", "value: h2", "promql: foo"} { if !strings.Contains(text, want) { fatalf(t, "expected migrated yaml to contain %q:\n%s", want, text) } @@ -392,9 +392,6 @@ expected: if err != nil { fatalf(t, "compose case should be v3 after migration: %v", err) } - if compose.OatsSchemaVersion != casefile.SchemaVersion { - fatalf(t, "expected schema version %d, got %d", casefile.SchemaVersion, compose.OatsSchemaVersion) - } if compose.Fixture == nil || compose.Fixture.Compose == nil || compose.Fixture.Compose.File != "docker-compose.yml" { fatalf(t, "expected migrated compose fixture, got %+v", compose.Fixture) } diff --git a/runner/cache_integration_test.go b/runner/cache_integration_test.go index b0849b50..a14fd011 100644 --- a/runner/cache_integration_test.go +++ b/runner/cache_integration_test.go @@ -16,8 +16,7 @@ import ( func cachedRunnerCase(t *testing.T) (*casefile.Case, []byte) { t.Helper() - src := []byte(`oats-schema-version: 3 -name: cached + src := []byte(`name: cached seed: type: app compose: x.yml diff --git a/runner/runner_test.go b/runner/runner_test.go index 1cb88796..f265009c 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -59,7 +59,6 @@ func mustParse(t *testing.T, src string) *casefile.Case { } const tracesCase = ` -oats-schema-version: 3 name: traces pass seed: type: app @@ -94,7 +93,6 @@ func TestRunCase_TracesPass(t *testing.T) { } const containsMissingCase = ` -oats-schema-version: 3 name: traces fail seed: type: app @@ -125,7 +123,6 @@ func TestRunCase_TracesFail_ContainsMissing(t *testing.T) { } const metricsValueCase = ` -oats-schema-version: 3 name: metric value seed: type: app @@ -172,7 +169,6 @@ func TestRunCase_LogsStructuredMatchPass(t *testing.T) { r, buf := newRunner(t, exec, Options{Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) c := mustParse(t, ` -oats-schema-version: 3 name: logs structured match seed: type: app @@ -205,7 +201,6 @@ func TestRunCase_MetricsStructuredMatchPass(t *testing.T) { r, _ := newRunner(t, exec, Options{Timeout: 100 * time.Millisecond, Interval: 5 * time.Millisecond, SeedSettleDelay: 1}) c := mustParse(t, ` -oats-schema-version: 3 name: metrics structured match seed: type: app @@ -245,7 +240,6 @@ func TestRunCase_DrivesInputRequests(t *testing.T) { r.endpoint.AppPort = port c := mustParse(t, ` -oats-schema-version: 3 name: traces pass with input seed: type: app @@ -273,7 +267,6 @@ expected: func TestRunCase_InlineOTLPSeedRequiresEndpoint(t *testing.T) { c := mustParse(t, ` -oats-schema-version: 3 name: inline seed seed: type: inline-otlp @@ -312,7 +305,6 @@ func TestRunCase_CustomCheckScriptPath(t *testing.T) { } c := mustParse(t, ` -oats-schema-version: 3 name: custom check path seed: type: app @@ -337,7 +329,6 @@ expected: func TestRunCase_CustomCheckInlineScript(t *testing.T) { c := mustParse(t, ` -oats-schema-version: 3 name: custom check inline seed: type: app @@ -363,7 +354,6 @@ expected: func TestRunCase_CustomCheckFailureSurfaced(t *testing.T) { c := mustParse(t, ` -oats-schema-version: 3 name: custom check fail seed: type: app @@ -404,7 +394,6 @@ func TestRunCase_ComposeLogsPass(t *testing.T) { t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) c := mustParse(t, ` -oats-schema-version: 3 name: compose logs pass seed: type: app @@ -448,7 +437,6 @@ func TestRunCase_ComposeLogsMissingSurfaced(t *testing.T) { t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) c := mustParse(t, ` -oats-schema-version: 3 name: compose logs missing seed: type: app @@ -484,7 +472,6 @@ expected: func TestRunCase_ComposeLogsRequireComposeFixture(t *testing.T) { c := mustParse(t, ` -oats-schema-version: 3 name: compose logs wrong fixture seed: type: app diff --git a/tests/e2e/cases/assert/absent-fail/files/oats-case.yaml b/tests/e2e/cases/assert/absent-fail/files/oats-case.yaml index 6ac71bc2..86849356 100644 --- a/tests/e2e/cases/assert/absent-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/absent-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: absent-fail fixture: remote: diff --git a/tests/e2e/cases/assert/absent/files/oats-case.yaml b/tests/e2e/cases/assert/absent/files/oats-case.yaml index 260ec7b3..4030d7c3 100644 --- a/tests/e2e/cases/assert/absent/files/oats-case.yaml +++ b/tests/e2e/cases/assert/absent/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: absent fixture: remote: diff --git a/tests/e2e/cases/assert/contains-fail/files/oats-case.yaml b/tests/e2e/cases/assert/contains-fail/files/oats-case.yaml index 7617dbc4..cff5a15f 100644 --- a/tests/e2e/cases/assert/contains-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/contains-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: contains-fail fixture: remote: diff --git a/tests/e2e/cases/assert/contains/files/oats-case.yaml b/tests/e2e/cases/assert/contains/files/oats-case.yaml index 816e489f..142a16c2 100644 --- a/tests/e2e/cases/assert/contains/files/oats-case.yaml +++ b/tests/e2e/cases/assert/contains/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: contains fixture: remote: diff --git a/tests/e2e/cases/assert/count-fail/files/oats-case.yaml b/tests/e2e/cases/assert/count-fail/files/oats-case.yaml index c4539ba1..8c478e59 100644 --- a/tests/e2e/cases/assert/count-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/count-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: count-fail fixture: remote: diff --git a/tests/e2e/cases/assert/count/files/oats-case.yaml b/tests/e2e/cases/assert/count/files/oats-case.yaml index 74d45554..11980e24 100644 --- a/tests/e2e/cases/assert/count/files/oats-case.yaml +++ b/tests/e2e/cases/assert/count/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: count fixture: remote: diff --git a/tests/e2e/cases/assert/match-fail/files/oats-case.yaml b/tests/e2e/cases/assert/match-fail/files/oats-case.yaml index 69be12b1..4e9cdb12 100644 --- a/tests/e2e/cases/assert/match-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/match-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: match-fail fixture: remote: diff --git a/tests/e2e/cases/assert/match-regexp-fail/files/oats-case.yaml b/tests/e2e/cases/assert/match-regexp-fail/files/oats-case.yaml index 272dfb1d..9b51cc99 100644 --- a/tests/e2e/cases/assert/match-regexp-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/match-regexp-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: match-regexp-fail fixture: remote: diff --git a/tests/e2e/cases/assert/match-regexp/files/oats-case.yaml b/tests/e2e/cases/assert/match-regexp/files/oats-case.yaml index b8a1a733..aefcdfb5 100644 --- a/tests/e2e/cases/assert/match-regexp/files/oats-case.yaml +++ b/tests/e2e/cases/assert/match-regexp/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: match-regexp fixture: remote: diff --git a/tests/e2e/cases/assert/match-spans-fail/files/oats-case.yaml b/tests/e2e/cases/assert/match-spans-fail/files/oats-case.yaml index 7632a522..6b146064 100644 --- a/tests/e2e/cases/assert/match-spans-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/match-spans-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: match-spans-fail fixture: remote: diff --git a/tests/e2e/cases/assert/not-contains-fail/files/oats-case.yaml b/tests/e2e/cases/assert/not-contains-fail/files/oats-case.yaml index 7ecd8fa9..c359e2b4 100644 --- a/tests/e2e/cases/assert/not-contains-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/not-contains-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: not-contains-fail fixture: remote: diff --git a/tests/e2e/cases/assert/not-contains/files/oats-case.yaml b/tests/e2e/cases/assert/not-contains/files/oats-case.yaml index 702752cc..dc6ee7a8 100644 --- a/tests/e2e/cases/assert/not-contains/files/oats-case.yaml +++ b/tests/e2e/cases/assert/not-contains/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: not-contains fixture: remote: diff --git a/tests/e2e/cases/assert/profile-fail/files/oats-case.yaml b/tests/e2e/cases/assert/profile-fail/files/oats-case.yaml index 2ea140e9..8d13ba38 100644 --- a/tests/e2e/cases/assert/profile-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/profile-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: profile-fail fixture: remote: diff --git a/tests/e2e/cases/assert/profile/files/oats-case.yaml b/tests/e2e/cases/assert/profile/files/oats-case.yaml index 7a0e7c14..73bd9694 100644 --- a/tests/e2e/cases/assert/profile/files/oats-case.yaml +++ b/tests/e2e/cases/assert/profile/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: profile fixture: remote: diff --git a/tests/e2e/cases/assert/regex-fail/files/oats-case.yaml b/tests/e2e/cases/assert/regex-fail/files/oats-case.yaml index 6e18f146..95f13387 100644 --- a/tests/e2e/cases/assert/regex-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/regex-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: regex-fail fixture: remote: diff --git a/tests/e2e/cases/assert/regex/files/oats-case.yaml b/tests/e2e/cases/assert/regex/files/oats-case.yaml index 5fdafe64..0e9bed5a 100644 --- a/tests/e2e/cases/assert/regex/files/oats-case.yaml +++ b/tests/e2e/cases/assert/regex/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: regex fixture: remote: diff --git a/tests/e2e/cases/assert/value-fail/files/oats-case.yaml b/tests/e2e/cases/assert/value-fail/files/oats-case.yaml index 75201588..25da77e6 100644 --- a/tests/e2e/cases/assert/value-fail/files/oats-case.yaml +++ b/tests/e2e/cases/assert/value-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: value-fail fixture: remote: diff --git a/tests/e2e/cases/assert/value/files/oats-case.yaml b/tests/e2e/cases/assert/value/files/oats-case.yaml index c70e6a81..0d67bab8 100644 --- a/tests/e2e/cases/assert/value/files/oats-case.yaml +++ b/tests/e2e/cases/assert/value/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: value fixture: remote: diff --git a/tests/e2e/cases/cli/cache-clear/files/oats-case.yaml b/tests/e2e/cases/cli/cache-clear/files/oats-case.yaml index 4d4ff123..a29ad368 100644 --- a/tests/e2e/cases/cli/cache-clear/files/oats-case.yaml +++ b/tests/e2e/cases/cli/cache-clear/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: cache-clear-case seed: type: inline-otlp diff --git a/tests/e2e/cases/cli/cache-clear/files/oats-config.yaml b/tests/e2e/cases/cli/cache-clear/files/oats-config.yaml index 47f1de30..2851ec15 100644 --- a/tests/e2e/cases/cli/cache-clear/files/oats-config.yaml +++ b/tests/e2e/cases/cli/cache-clear/files/oats-config.yaml @@ -1,5 +1,5 @@ meta: - version: 2 + version: 3 suites: - name: cache-clear cases: ["oats-case.yaml"] diff --git a/tests/e2e/cases/cli/cache-skip/files/oats-case.yaml b/tests/e2e/cases/cli/cache-skip/files/oats-case.yaml index 653c409f..c72918f5 100644 --- a/tests/e2e/cases/cli/cache-skip/files/oats-case.yaml +++ b/tests/e2e/cases/cli/cache-skip/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: cache-case seed: type: inline-otlp diff --git a/tests/e2e/cases/cli/cache-skip/files/oats-config.yaml b/tests/e2e/cases/cli/cache-skip/files/oats-config.yaml index 99788d5e..a772dbdc 100644 --- a/tests/e2e/cases/cli/cache-skip/files/oats-config.yaml +++ b/tests/e2e/cases/cli/cache-skip/files/oats-config.yaml @@ -1,5 +1,5 @@ meta: - version: 2 + version: 3 suites: - name: cache cases: ["oats-case.yaml"] diff --git a/tests/e2e/cases/cli/fail-fast/files/01-fail.yaml b/tests/e2e/cases/cli/fail-fast/files/01-fail.yaml index 824a4b09..3d7865ad 100644 --- a/tests/e2e/cases/cli/fail-fast/files/01-fail.yaml +++ b/tests/e2e/cases/cli/fail-fast/files/01-fail.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: ff-first-fails seed: type: inline-otlp diff --git a/tests/e2e/cases/cli/fail-fast/files/02-pass.yaml b/tests/e2e/cases/cli/fail-fast/files/02-pass.yaml index 15a296e8..02d72d06 100644 --- a/tests/e2e/cases/cli/fail-fast/files/02-pass.yaml +++ b/tests/e2e/cases/cli/fail-fast/files/02-pass.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: ff-second-should-not-run seed: type: inline-otlp diff --git a/tests/e2e/cases/cli/fail-fast/files/oats-config.yaml b/tests/e2e/cases/cli/fail-fast/files/oats-config.yaml index 3b24079b..d8bd6768 100644 --- a/tests/e2e/cases/cli/fail-fast/files/oats-config.yaml +++ b/tests/e2e/cases/cli/fail-fast/files/oats-config.yaml @@ -1,5 +1,5 @@ meta: - version: 2 + version: 3 suites: - name: failfast cases: ["01-fail.yaml", "02-pass.yaml"] diff --git a/tests/e2e/cases/cli/migrate/test.yaml b/tests/e2e/cases/cli/migrate/test.yaml index 1208d2a5..010f5fc1 100644 --- a/tests/e2e/cases/cli/migrate/test.yaml +++ b/tests/e2e/cases/cli/migrate/test.yaml @@ -10,5 +10,7 @@ run: expect: stdout_contains: - - "oats-schema-version: 3" + - "name: legacy" - "logql" + stdout_not_contains: + - "oats-schema-version" diff --git a/tests/e2e/cases/custom/custom-check-fail/files/oats-case.yaml b/tests/e2e/cases/custom/custom-check-fail/files/oats-case.yaml index 429c3a5e..1bd73271 100644 --- a/tests/e2e/cases/custom/custom-check-fail/files/oats-case.yaml +++ b/tests/e2e/cases/custom/custom-check-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: custom-check-fail fixture: remote: diff --git a/tests/e2e/cases/custom/custom-check/files/oats-case.yaml b/tests/e2e/cases/custom/custom-check/files/oats-case.yaml index 897697ae..44ecbaa5 100644 --- a/tests/e2e/cases/custom/custom-check/files/oats-case.yaml +++ b/tests/e2e/cases/custom/custom-check/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: custom-check fixture: remote: diff --git a/tests/e2e/cases/fixture/compose-app-seed/files/oats-case.yaml b/tests/e2e/cases/fixture/compose-app-seed/files/oats-case.yaml index caef6555..a22dcc6e 100644 --- a/tests/e2e/cases/fixture/compose-app-seed/files/oats-case.yaml +++ b/tests/e2e/cases/fixture/compose-app-seed/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: compose-app-seed fixture: compose: diff --git a/tests/e2e/cases/fixture/compose-logs-fail/files/oats-case.yaml b/tests/e2e/cases/fixture/compose-logs-fail/files/oats-case.yaml index f5fcd78b..cd444fa3 100644 --- a/tests/e2e/cases/fixture/compose-logs-fail/files/oats-case.yaml +++ b/tests/e2e/cases/fixture/compose-logs-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: compose-logs-fail fixture: compose: diff --git a/tests/e2e/cases/fixture/compose-logs/files/oats-case.yaml b/tests/e2e/cases/fixture/compose-logs/files/oats-case.yaml index 4233f713..0de74737 100644 --- a/tests/e2e/cases/fixture/compose-logs/files/oats-case.yaml +++ b/tests/e2e/cases/fixture/compose-logs/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: compose-logs fixture: compose: diff --git a/tests/e2e/cases/fixture/k3d-fail/files/oats-case.yaml b/tests/e2e/cases/fixture/k3d-fail/files/oats-case.yaml index 710f9ab3..508f5e5c 100644 --- a/tests/e2e/cases/fixture/k3d-fail/files/oats-case.yaml +++ b/tests/e2e/cases/fixture/k3d-fail/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: k3d-fail fixture: k3d: diff --git a/tests/e2e/cases/fixture/k3d-smoke/files/oats-case.yaml b/tests/e2e/cases/fixture/k3d-smoke/files/oats-case.yaml index eb185e09..23445bc8 100644 --- a/tests/e2e/cases/fixture/k3d-smoke/files/oats-case.yaml +++ b/tests/e2e/cases/fixture/k3d-smoke/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: k3d-smoke fixture: k3d: diff --git a/tests/e2e/cases/fixture/parallel-compose/files/alpha.yaml b/tests/e2e/cases/fixture/parallel-compose/files/alpha.yaml index 92f36bc4..e43b9eff 100644 --- a/tests/e2e/cases/fixture/parallel-compose/files/alpha.yaml +++ b/tests/e2e/cases/fixture/parallel-compose/files/alpha.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: alpha seed: type: inline-otlp diff --git a/tests/e2e/cases/fixture/parallel-compose/files/beta.yaml b/tests/e2e/cases/fixture/parallel-compose/files/beta.yaml index 1f5dd8ab..048f44a9 100644 --- a/tests/e2e/cases/fixture/parallel-compose/files/beta.yaml +++ b/tests/e2e/cases/fixture/parallel-compose/files/beta.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: beta seed: type: inline-otlp diff --git a/tests/e2e/cases/fixture/parallel-compose/files/oats-config.yaml b/tests/e2e/cases/fixture/parallel-compose/files/oats-config.yaml index 3b6f0f96..02baee8d 100644 --- a/tests/e2e/cases/fixture/parallel-compose/files/oats-config.yaml +++ b/tests/e2e/cases/fixture/parallel-compose/files/oats-config.yaml @@ -1,5 +1,5 @@ meta: - version: 2 + version: 3 suites: - name: alpha cases: ["alpha.yaml"] diff --git a/tests/e2e/cases/fixture/remote-basic/files/oats-case.yaml b/tests/e2e/cases/fixture/remote-basic/files/oats-case.yaml index 1df08e59..da4db902 100644 --- a/tests/e2e/cases/fixture/remote-basic/files/oats-case.yaml +++ b/tests/e2e/cases/fixture/remote-basic/files/oats-case.yaml @@ -1,4 +1,3 @@ -oats-schema-version: 3 name: remote-basic fixture: remote: diff --git a/tests/e2e/cases/fixture/remote-basic/test.yaml b/tests/e2e/cases/fixture/remote-basic/test.yaml index 9d758d55..55fd0d7f 100644 --- a/tests/e2e/cases/fixture/remote-basic/test.yaml +++ b/tests/e2e/cases/fixture/remote-basic/test.yaml @@ -34,7 +34,6 @@ run: pyroscope: pyroscope CFG cat > "$workdir/files/oats-case.yaml" < Date: Wed, 8 Jul 2026 14:49:29 +0000 Subject: [PATCH 115/124] feat(cli): positional path scoping + config discovery from cwd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Positional args to `oats` (and `oats run`) are now file/dir paths that scope which cases run — `oats examples/` runs only cases at or under examples/, `oats x.yaml` runs one. They filter cases; they do not change where the config is loaded from. The config (oats-config.yaml) is discovered from the current directory walking up through parents (like git), so `oats` works from any subdirectory; --config still overrides. `list` uses the same resolution. Clarified root/run/migrate help. Signed-off-by: Gregor Zeitlinger --- README.md | 19 ++++--- discovery/discovery.go | 32 +++++++++++ discovery/discovery_test.go | 38 +++++++++++++ internal/cli/main.go | 109 +++++++++++++++++++++++++++++------- 4 files changed, 171 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 5e66911a..46e3ba70 100644 --- a/README.md +++ b/README.md @@ -71,19 +71,24 @@ bin/oats version ## CLI ```sh -oats [flags] # run the suites (implicit; same as `oats run`) -oats run [flags] # run the suites -oats list --config oats-config.yaml # print the run plan and exit -oats migrate legacy.yaml # convert one legacy yaml to the v3 shape -oats cache clear # delete all cached results -oats version # print the version +oats # run every case (implicit; same as `oats run`) +oats examples/ # run only cases at or under a path +oats run [paths...] # explicit run form +oats list # print the run plan and exit +oats migrate # migrate a legacy file (stdout) or directory (in place) +oats cache clear # delete all cached results +oats version # print the version ``` +`oats-config.yaml` is found in the current directory or any parent (override with +`--config`), so you can run `oats` from anywhere in the project. Positional paths +scope *which* cases run without changing where the config is loaded from. + Common flags: | Flag | Default | Meaning | |------|---------|---------| -| `--config` | `oats-config.yaml` | path to the config file | +| `--config` | found from cwd upward | path to the config file | | `--suite` | all | comma-separated suite names to run | | `--tags` | all | comma-separated tags; a case runs if it matches any | | `--timeout` | `30s` | per-assertion timeout — each assertion is retried until it passes or this elapses | diff --git a/discovery/discovery.go b/discovery/discovery.go index dc042ad9..1bc5dfb2 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -118,6 +118,33 @@ func (c *RootConfig) Validate() error { type Filter struct { Suites []string // exact suite names; empty = all suites Tags []string // any-match + // Paths restricts the run to cases at (or under) these absolute file/dir + // paths. Empty = no path restriction. Backs the positional `oats …` + // scoping, which selects which cases run without changing where the config + // is loaded from. + Paths []string +} + +// keepCasesUnderPaths returns the cases whose source file is one of paths or +// lives under one of them (dir scope). paths must be absolute and cleaned. +func keepCasesUnderPaths(cases []*casefile.Case, paths []string) []*casefile.Case { + if len(paths) == 0 { + return cases + } + var kept []*casefile.Case + for _, tc := range cases { + abs, err := filepath.Abs(tc.SourcePath) + if err != nil { + continue + } + for _, p := range paths { + if abs == p || strings.HasPrefix(abs, p+string(filepath.Separator)) { + kept = append(kept, tc) + break + } + } + } + return kept } // Plan is one suite plus the cases it expanded to. @@ -174,6 +201,11 @@ func (c *RootConfig) PlanRun(f Filter) ([]Plan, error) { if err != nil { return nil, fmt.Errorf("suite %q: %w", suiteLabel(suite), err) } + cases = keepCasesUnderPaths(cases, f.Paths) + if len(cases) == 0 { + // No cases in this suite fall under the requested path scope. + continue + } suite = materializeSuite(suite, cases) if !wantSuite(suite.Name) { continue diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index f84b6dee..fe50bdcc 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -139,6 +139,44 @@ meta: } } +func TestPlanRun_FilterByPath(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 3 +suites: + - name: alpha + cases: ["a/*.yaml"] + - name: beta + cases: ["b/*.yaml"] +`) + writeFile(t, dir, "a/x.yaml", strings.Replace(validCaseYAML, "%s", "ax", 1)) + writeFile(t, dir, "b/y.yaml", strings.Replace(validCaseYAML, "%s", "by", 1)) + + cfg, err := Load(filepath.Join(dir, "oats-config.yaml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + + // Directory scope: only the suite whose cases live under a/ survives. + plans, err := cfg.PlanRun(Filter{Paths: []string{filepath.Join(dir, "a")}}) + if err != nil { + t.Fatalf("PlanRun dir scope: %v", err) + } + if len(plans) != 1 || plans[0].Suite.Name != "alpha" || len(plans[0].Cases) != 1 { + t.Fatalf("dir path filter: %+v", planNames(plans)) + } + + // Single-file scope: the exact case file. + plans, err = cfg.PlanRun(Filter{Paths: []string{filepath.Join(dir, "b", "y.yaml")}}) + if err != nil { + t.Fatalf("PlanRun file scope: %v", err) + } + if len(plans) != 1 || plans[0].Suite.Name != "beta" || len(plans[0].Cases) != 1 { + t.Fatalf("file path filter: %+v", planNames(plans)) + } +} + func TestPlanRun_FilterBySuiteName(t *testing.T) { dir := t.TempDir() writeFile(t, dir, "oats-config.yaml", ` diff --git a/internal/cli/main.go b/internal/cli/main.go index da10f317..1afef270 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -5,16 +5,20 @@ // // Usage: // -// oats [flags] run the suites (implicit; same as `oats run`) -// oats run [flags] run the suites -// oats list print the run plan and exit -// oats migrate convert one legacy yaml to the v3 shape -// oats cache clear delete all cached results -// oats version print the version +// oats [paths...] run the cases (implicit; same as `oats run`) +// oats run [paths...] run the cases; positional paths scope which cases run +// oats list print the run plan and exit +// oats migrate migrate a legacy file (stdout) or directory (in place) +// oats cache clear delete all cached results +// oats version print the version +// +// The config (oats-config.yaml) is found in the current directory or any parent +// unless --config is given. Positional paths (e.g. `oats examples/`) restrict the +// run to cases at or under them. // // Run flags (subset): // -// --config Path to oats-config.yaml (default ./oats-config.yaml) +// --config Path to oats-config.yaml (default: found from cwd upward) // --gcx Path to gcx binary (default "gcx" on PATH) // --format Output format: "text" (default) or "ndjson" // --suite Comma-separated suite names to include @@ -80,13 +84,17 @@ func Run() int { func newRootCmd(exit *int) *cobra.Command { var verbose int root := &cobra.Command{ - Use: "oats", - Short: "OpenTelemetry Acceptance Tests — the gcx-driven runner", + Use: "oats [paths...]", + Short: "OpenTelemetry Acceptance Tests — the gcx-driven runner", + Long: "OpenTelemetry Acceptance Tests — the gcx-driven runner.\n\n" + + "With no subcommand, oats runs the cases in oats-config.yaml (found in the\n" + + "current directory or any parent). Optional positional paths scope the run to\n" + + "cases at or under them, e.g. `oats examples/` or `oats examples/go/oats-case.yaml`.", SilenceUsage: true, SilenceErrors: true, - // Bare `oats [flags]` is an implicit `run` for backward compatibility. - RunE: func(cmd *cobra.Command, _ []string) error { - return runAction(cmd, verbose, exit) + // Bare `oats [paths...] [flags]` is an implicit `run`. + RunE: func(cmd *cobra.Command, args []string) error { + return runAction(cmd, args, verbose, exit) }, } root.PersistentFlags().CountVarP(&verbose, "verbose", "v", "increase verbosity (-v, -vv, -vvv)") @@ -133,12 +141,16 @@ func addRunFlags(fs *pflag.FlagSet) { func newRunCmd(verbose, exit *int) *cobra.Command { cmd := &cobra.Command{ - Use: "run", - Short: "Run the suites in oats-config.yaml (default when no subcommand is given)", + Use: "run [paths...]", + Short: "Run the cases in oats-config.yaml (default when no subcommand is given)", + Long: "Run the cases declared by oats-config.yaml.\n\n" + + "The config is found in the current directory or any parent (override with\n" + + "--config). Optional positional paths scope the run to cases at or under those\n" + + "files/directories, e.g. `oats run examples/` runs only the cases under examples/.", SilenceUsage: true, SilenceErrors: true, - RunE: func(cmd *cobra.Command, _ []string) error { - return runAction(cmd, *verbose, exit) + RunE: func(cmd *cobra.Command, args []string) error { + return runAction(cmd, args, *verbose, exit) }, } addRunFlags(cmd.Flags()) @@ -224,7 +236,10 @@ func newVersionCmd() *cobra.Command { } func listAction(cmd *cobra.Command) error { - configPath, _ := cmd.Flags().GetString("config") + configPath, err := resolveConfigPath(cmd.Flags()) + if err != nil { + return err + } cfg, err := discovery.Load(configPath) if err != nil { return err @@ -265,8 +280,9 @@ func migrateAction(path string) error { } // runAction is the shared implementation behind the implicit-run root and the -// explicit `run` subcommand. -func runAction(cmd *cobra.Command, verbose int, exit *int) error { +// explicit `run` subcommand. Positional args are file/dir paths that scope which +// cases run; the config is resolved from cwd (walking up), independent of them. +func runAction(cmd *cobra.Command, args []string, verbose int, exit *int) error { fs := cmd.Flags() // Honor the deprecated --list / --migrate flags. @@ -277,21 +293,32 @@ func runAction(cmd *cobra.Command, verbose int, exit *int) error { return migrateAction(migratePath) } - configPath, _ := fs.GetString("config") + configPath, err := resolveConfigPath(fs) + if err != nil { + return err + } cfg, err := discovery.Load(configPath) if err != nil { return err } + paths, err := absArgs(args) + if err != nil { + return err + } filter := discovery.Filter{ Suites: splitCSV(flagStr(fs, "suite")), Tags: splitCSV(flagStr(fs, "tags")), + Paths: paths, } plans, err := cfg.PlanRun(filter) if err != nil { return err } if len(plans) == 0 { + if len(paths) > 0 { + return fmt.Errorf("no cases matched the given path(s)") + } return fmt.Errorf("no suites matched the filter") } @@ -704,6 +731,48 @@ func gcxVersion(bin string) string { // defaultCacheDir returns $XDG_STATE_HOME/oats or ~/.cache/oats. The cache // lives under XDG_STATE_HOME on purpose (it is regeneratable state, not // configuration). Falls back to a relative path if HOME is unset. +// resolveConfigPath returns the oats-config.yaml to load. An explicit --config +// is honored as-is; otherwise the default filename is searched for in the +// current directory and each parent (so `oats` works from a subdirectory). +func resolveConfigPath(fs *pflag.FlagSet) (string, error) { + name, _ := fs.GetString("config") + if fs.Changed("config") { + return name, nil + } + cwd, err := os.Getwd() + if err != nil { + return "", err + } + for dir := cwd; ; { + candidate := filepath.Join(dir, name) + if _, statErr := os.Stat(candidate); statErr == nil { + return candidate, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", fmt.Errorf("no %s found in %s or any parent directory (use --config)", name, cwd) + } + dir = parent + } +} + +// absArgs cleans positional path args to absolute paths (relative to cwd) for +// case-path scoping. +func absArgs(args []string) ([]string, error) { + if len(args) == 0 { + return nil, nil + } + out := make([]string, 0, len(args)) + for _, a := range args { + abs, err := filepath.Abs(a) + if err != nil { + return nil, err + } + out = append(out, filepath.Clean(abs)) + } + return out, nil +} + func defaultCacheDir() string { if s := os.Getenv("XDG_STATE_HOME"); s != "" { return filepath.Join(s, "oats") From 2231785dfffce1f1ce487d33316249d21eced10a Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 15:01:01 +0000 Subject: [PATCH 116/124] docs: clarify config file (what/where) and use default invocation Explain in the README quick start and case-reference what oats-config.yaml is and that oats finds it from cwd upward, and switch the quick start to the default invocation (`cd examples/smoke && oats`) instead of an explicit --config path. Drop the now-redundant `--config oats-config.yaml` from the ci.md workflow examples and fix a stale `fixture.type` reference (nested compose/k3d/remote). Also lands the README header (logo + badges) with its nav pointed at the docs that actually exist (case-reference, ci, upgrading) and adds assets/icon.svg. Signed-off-by: Gregor Zeitlinger --- README.md | 29 ++++++++++++++++++++++++++--- assets/icon.svg | 38 ++++++++++++++++++++++++++++++++++++++ docs/case-reference.md | 18 +++++++++++++----- docs/ci.md | 9 +++++---- 4 files changed, 82 insertions(+), 12 deletions(-) create mode 100644 assets/icon.svg diff --git a/README.md b/README.md index 46e3ba70..a6ea78a6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,21 @@ -# OpenTelemetry Acceptance Tests (OATs) + +

+ oats logo +

+ +

OpenTelemetry Acceptance Tests (OATs)

+ +

+ Lint + GitHub Release +

+ +

+ Case reference · + CI · + Upgrading +

+ OATs is a declarative acceptance-test framework for OpenTelemetry. You describe, in yaml, the telemetry an instrumented app *should* produce — traces, logs, @@ -53,12 +70,18 @@ bootstrap gcx itself, but pinning it explicitly keeps runs reproducible. ## Quick start +An OATS project is a directory with an **`oats-config.yaml`** (which suites/cases +to run and the fixtures they use) plus the case files it points at. `oats` finds +that config in the working directory or any parent, so you just `cd` in and run: + ```sh +cd examples/smoke + # Print the run plan without executing -oats list --config examples/smoke/oats-config.yaml +oats list # Run it -oats --config examples/smoke/oats-config.yaml +oats ``` To hack on OATS itself (builds local `oats` + `gcx` into `./bin`): diff --git a/assets/icon.svg b/assets/icon.svg new file mode 100644 index 00000000..3f0200c8 --- /dev/null +++ b/assets/icon.svg @@ -0,0 +1,38 @@ + + + + + + \ No newline at end of file diff --git a/docs/case-reference.md b/docs/case-reference.md index 3536b676..85bbdb8d 100644 --- a/docs/case-reference.md +++ b/docs/case-reference.md @@ -7,17 +7,25 @@ CLI summary see the [README](../README.md); for running in CI see ## oats-config.yaml +`oats-config.yaml` is the entry point of an OATS project: it declares the suites, +the case files each suite runs (path globs), the fixtures they use, and cache +settings. `oats` looks for it in the current directory and then each parent (so +you can run from a subdirectory); `--config ` overrides the search. Its +`meta.version` is the single OATS schema version — case files carry no version of +their own. + ```yaml meta: - version: 3 # the single OATS schema version; cases carry no version field of their own + version: 3 suites: - - cases: ["examples/smoke/cases/*.yaml"] + - cases: ["cases/*.yaml"] # globs, relative to this file's directory cache: - ttl_days: 7 # skip-when-unchanged TTL; 0 → default (7 days) + ttl_days: 7 # skip-when-unchanged TTL; 0 → default (7 days) ``` -A suite may pin its own fixture, or let each case carry a case-local `fixture:` -block (the common one-case-per-suite shape). +A top-level `cases:` list may be used instead of `suites:` for the common +one-case-per-suite shape. A suite may name a shared fixture (from the `fixture:` +map) or let each case carry its own case-local `fixture:` block. ## Case yaml diff --git a/docs/ci.md b/docs/ci.md index dd5f665c..a877a3d2 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -68,7 +68,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: jdx/mise-action@v2 # installs pinned oats + gcx from mise.toml - - run: oats --config oats-config.yaml + - run: oats # finds oats-config.yaml in the repo root ``` The `paths` filter is the cheapest and most important optimization: if a PR @@ -89,7 +89,7 @@ To persist it across CI runs: with: path: ~/.cache/oats key: oats-${{ hashFiles('mise.toml') }} - - run: oats --config oats-config.yaml --cache-dir ~/.cache/oats + - run: oats --cache-dir ~/.cache/oats ``` Keying on `hashFiles('mise.toml')` scopes the cache to a gcx/oats version pair, @@ -97,8 +97,9 @@ so a version bump starts a fresh cache — no stale skips from version drift. ### Correctness caveat — floating image tags -The cache key hashes the fixture **config** (`fixture.type`, compose file paths, -image tags, env, endpoint …), **not** the fixture's **contents**. If the app +The cache key hashes the fixture **config** (the `compose`/`k3d`/`remote` block — +compose file paths, image tags, env, endpoint …), **not** the fixture's +**contents**. If the app under test is baked into an image with a floating tag (`:latest`), rebuilding that image under the same tag produces an identical key — so a case can be skipped against stale bytes (a false green). From 4c2ae261aaeb71d336ef61276bd29065d6155c92 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 15:11:58 +0000 Subject: [PATCH 117/124] docs: recommend per-dir oats-case.yaml layout; name config in --config default Replace the `cases: ["cases/*.yaml"]` example (generic dir, matches any yaml) with the recommended one-case-per-directory layout collected via `*/oats-case.yaml`, and explain globs are one-segment (no recursive `**`). Name oats-config.yaml in the README --config default cell. Signed-off-by: Gregor Zeitlinger --- README.md | 2 +- docs/case-reference.md | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index a6ea78a6..fcfd6307 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Common flags: | Flag | Default | Meaning | |------|---------|---------| -| `--config` | found from cwd upward | path to the config file | +| `--config` | `oats-config.yaml`, searched from cwd upward | path to the config file | | `--suite` | all | comma-separated suite names to run | | `--tags` | all | comma-separated tags; a case runs if it matches any | | `--timeout` | `30s` | per-assertion timeout — each assertion is retried until it passes or this elapses | diff --git a/docs/case-reference.md b/docs/case-reference.md index 85bbdb8d..cb940bb1 100644 --- a/docs/case-reference.md +++ b/docs/case-reference.md @@ -18,14 +18,26 @@ their own. meta: version: 3 suites: - - cases: ["cases/*.yaml"] # globs, relative to this file's directory + - cases: ["*/oats-case.yaml"] # each case in its own subdir; globs are relative to this file cache: - ttl_days: 7 # skip-when-unchanged TTL; 0 → default (7 days) + ttl_days: 7 # skip-when-unchanged TTL; 0 → default (7 days) ``` -A top-level `cases:` list may be used instead of `suites:` for the common -one-case-per-suite shape. A suite may name a shared fixture (from the `fixture:` -map) or let each case carry its own case-local `fixture:` block. +The common layout is one case per directory — a `oats-case.yaml` alongside the +files it needs (compose files, custom-check scripts) — collected with a glob like +`*/oats-case.yaml`: + +```text +oats-config.yaml +go/oats-case.yaml go/docker-compose.oats.yml +python/oats-case.yaml python/docker-compose.oats.yml +``` + +Globs are shell-style (`*` matches one path segment; no recursive `**`), so add a +segment for deeper trees (e.g. `examples/*/oats-case.yaml`) or list several +patterns. A top-level `cases:` list may be used instead of `suites:` for the +one-suite shape. A suite may name a shared fixture (from the `fixture:` map) or +let each case carry its own case-local `fixture:` block. ## Case yaml From b31972bb9d3424343fd27088ab17c35e19c622ad Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 15:20:17 +0000 Subject: [PATCH 118/124] docs: rewrite README getting-started, split CLI into docs/cli.md, align examples README: replace the examples-dependent quick start with a from-scratch getting started (two files + Docker; OATS boots an LGTM stack and seeds inline-otlp, no app needed), call out examples/ as the starting point to copy, drop the leftover "What it covers" list, and move the CLI section to a dedicated docs/cli.md with per-command detail. Add CLI to the header nav. Align the shipped examples to the recommended one-case-per-directory layout: each case is now /oats-case.yaml (with its verify.sh colocated), collected via `*/oats-case.yaml` globs, so the examples model what the docs recommend. Signed-off-by: Gregor Zeitlinger --- README.md | 96 ++++++++----------- docs/cli.md | 89 +++++++++++++++++ .../rolldice/oats-case.yaml} | 0 .../rolldice/oats-case.yaml} | 2 +- .../{scripts => k3d/rolldice}/verify.sh | 0 examples/fixtures/oats-config.yaml | 4 +- .../oats-case.yaml} | 2 +- .../smoke/{scripts => custom-check}/verify.sh | 0 .../oats-case.yaml} | 0 examples/smoke/oats-config.yaml | 2 +- .../profile.yaml => profile/oats-case.yaml} | 0 .../rolldice.yaml => rolldice/oats-case.yaml} | 0 12 files changed, 132 insertions(+), 63 deletions(-) create mode 100644 docs/cli.md rename examples/fixtures/{cases/compose/rolldice.yaml => compose/rolldice/oats-case.yaml} (100%) rename examples/fixtures/{cases/k3d/rolldice.yaml => k3d/rolldice/oats-case.yaml} (84%) rename examples/fixtures/{scripts => k3d/rolldice}/verify.sh (100%) rename examples/smoke/{cases/custom-check.yaml => custom-check/oats-case.yaml} (66%) rename examples/smoke/{scripts => custom-check}/verify.sh (100%) rename examples/smoke/{cases/inline-seed.yaml => inline-seed/oats-case.yaml} (100%) rename examples/smoke/{cases/profile.yaml => profile/oats-case.yaml} (100%) rename examples/smoke/{cases/rolldice.yaml => rolldice/oats-case.yaml} (100%) diff --git a/README.md b/README.md index fcfd6307..d59c76e9 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@

+ CLI · Case reference · CI · Upgrading @@ -68,82 +69,61 @@ Without mise: `oats` drives assertions through `gcx`; for fixture-backed runs OATS can bootstrap gcx itself, but pinning it explicitly keeps runs reproducible. -## Quick start +## Getting started -An OATS project is a directory with an **`oats-config.yaml`** (which suites/cases -to run and the fixtures they use) plus the case files it points at. `oats` finds -that config in the working directory or any parent, so you just `cd` in and run: +An OATS project is a directory with an **`oats-config.yaml`** plus one case per +subdirectory. The smallest project that runs end-to-end is two files and Docker — +OATS boots a throwaway Grafana LGTM stack for you, so you need no running backend +and no app to see it work. -```sh -cd examples/smoke - -# Print the run plan without executing -oats list +**`oats-config.yaml`** — lists the cases: -# Run it -oats +```yaml +meta: + version: 3 +cases: ["*/oats-case.yaml"] # one case per subdirectory ``` -To hack on OATS itself (builds local `oats` + `gcx` into `./bin`): +**`hello/oats-case.yaml`** — boot an LGTM stack, push a log, assert it lands: -```sh -./scripts/build-local-tools.sh -bin/oats version +```yaml +name: hello world +fixture: + compose: + template: lgtm # OATS boots grafana/otel-lgtm; no compose file needed +seed: + type: inline-otlp # push telemetry directly — no instrumented app required + logs: + - service: hello + body: hello from oats +expected: + logs: + - logql: '{service_name="hello"}' + contains: hello from oats ``` -## CLI +Then, from the project directory: ```sh -oats # run every case (implicit; same as `oats run`) -oats examples/ # run only cases at or under a path -oats run [paths...] # explicit run form -oats list # print the run plan and exit -oats migrate # migrate a legacy file (stdout) or directory (in place) -oats cache clear # delete all cached results -oats version # print the version +oats list # show the resolved plan +oats # boot the fixture, seed it, assert ``` -`oats-config.yaml` is found in the current directory or any parent (override with -`--config`), so you can run `oats` from anywhere in the project. Positional paths -scope *which* cases run without changing where the config is loaded from. - -Common flags: - -| Flag | Default | Meaning | -|------|---------|---------| -| `--config` | `oats-config.yaml`, searched from cwd upward | path to the config file | -| `--suite` | all | comma-separated suite names to run | -| `--tags` | all | comma-separated tags; a case runs if it matches any | -| `--timeout` | `30s` | per-assertion timeout — each assertion is retried until it passes or this elapses | -| `--interval` | `500ms` | polling interval between assertion retries | -| `--absent-timeout` | `10s` | window an `absent` assertion must stay empty | -| `--parallel` | `1` | suites to run concurrently, when fixture isolation allows | -| `--fail-fast` | `false` | stop scheduling further cases after the first case failure | -| `--no-cache` | `false` | disable the skip-when-unchanged cache for this run | -| `--format` | `text` | output format: `text` or `ndjson` | -| `--gcx` | `gcx` | path to the gcx binary | -| `--gcx-context` | derived | override the gcx context (otherwise derived from the fixture endpoint) | -| `-v` / `-vv` / `-vvv` | — | increase verbosity | - -Run `oats --help` for the full list, including the inline-OTLP seed host/port -overrides. - -## What it covers - -- traces / logs / metrics / profiles, queried through `gcx` -- structural collector-style row matching (`match` / `match_spans`) -- app-backed and inline-OTLP seed modes -- remote / compose / k3d fixtures -- custom-check scripts -- best-effort migration from legacy OATS yaml (`oats migrate path/to/legacy.yaml`) +To test a **real app** instead of inline telemetry, point the fixture at your +app's compose file and use `seed: {type: app}` with `input:` requests. The +runnable **[`examples/`](examples/)** projects are the best starting point to +copy from — `examples/smoke/` (remote fixture + assertions) and +`examples/fixtures/` (compose and k3d) — see [docs/case-reference.md](docs/case-reference.md) +for the full shape. ## Documentation -- **[docs/case-reference.md](docs/case-reference.md)** — the full case + config +- **[docs/cli.md](docs/cli.md)** — every command and flag +- **[docs/case-reference.md](docs/case-reference.md)** — the full config + case shape: fixtures, seed modes, the assertion vocabulary, custom checks - **[docs/ci.md](docs/ci.md)** — installing and running OATS in CI, plus result caching and its caveats - **[UPGRADING.md](UPGRADING.md)** — migrating older (schema-2) repos to v3 - **[AGENTS.md](AGENTS.md)** — for contributors and coding agents working *on* OATS (build, layout, conventions) -- `examples/smoke/`, `examples/fixtures/` — small runnable examples +- **[`examples/`](examples/)** — small runnable projects to copy from diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 00000000..4d8ea260 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,89 @@ +# CLI reference + +`oats` runs the cases declared by an `oats-config.yaml`. For the config and case +shapes see [case-reference.md](case-reference.md); for CI see [ci.md](ci.md). + +## Config discovery + +Every command that needs the config looks for **`oats-config.yaml`** in the +current directory and then each parent directory (like `git`), so you can run +`oats` from anywhere inside a project. Pass `--config ` to use a specific +file instead. + +## Commands + +### `oats [paths...]` / `oats run [paths...]` + +Run the cases. Bare `oats` is an implicit `run`; the explicit `run` subcommand is +identical. With no positional args every case in the config runs; positional +**paths** (files or directories) restrict the run to cases at or under them — +they scope *which* cases run, they do not change where the config loads from. + +```sh +oats # run every case +oats payments/ # only cases under payments/ +oats payments/checkout/oats-case.yaml # a single case +oats --suite smoke --tags traces # filter by suite name and tag +oats --parallel 4 # run parallel-safe suites concurrently +``` + +A run boots each suite's fixture, seeds it, then polls every assertion until it +passes or `--timeout` elapses. Exit code is non-zero if any case fails. + +Flags: + +| Flag | Default | Meaning | +|------|---------|---------| +| `--config` | `oats-config.yaml`, searched from cwd upward | config file to load | +| `--suite` | all | comma-separated suite names to run | +| `--tags` | all | comma-separated tags; a case runs if it matches any | +| `--parallel` | `1` | suites to run concurrently, where fixture isolation allows | +| `--fail-fast` | `false` | stop scheduling further cases after the first failure | +| `--timeout` | `30s` | per-assertion timeout — each assertion is retried until it passes or this elapses | +| `--interval` | `500ms` | polling interval between assertion retries | +| `--absent-timeout` | `10s` | window an `absent` assertion must stay empty to pass | +| `--seed-settle` | `2s` | wait after seeding before the first assertion | +| `--no-cache` | `false` | ignore the skip-when-unchanged cache for this run | +| `--cache-dir` | `$XDG_STATE_HOME/oats` | directory for the skip-when-unchanged cache | +| `--format` | `text` | output format: `text` or `ndjson` | +| `--gcx` | `gcx` | path to the gcx binary (PATH-resolved if a bare name) | +| `--gcx-context` | derived | gcx context to query (otherwise derived from the fixture endpoint) | +| `--app-host` / `--app-port` | `localhost` / `8080` | where to drive `input` requests when a fixture doesn't resolve the app endpoint itself | +| `--otlp-http` | `http://localhost:4318` | OTLP/HTTP base URL for the `inline-otlp` seed | +| `-v` / `-vv` / `-vvv` | — | increasing verbosity (passes / commands / lifecycle) | + +### `oats list` + +Print the resolved run plan — the suites, their fixtures, and the cases each +expands to — and exit without executing anything. Honors `--config` and the same +discovery. Useful to confirm globs and fixtures before a run. + +### `oats migrate ` + +Convert legacy (schema-2) OATS yaml to the v3 shape. + +- **File** → prints one self-contained v3 case (including its `fixture:` block) + to stdout; warnings about lossy/dropped fields go to stderr. +- **Directory** → migrates every legacy case found under it *in place* (each file + is rewritten as its v3 equivalent, honoring `.oatsignore` and skipping + templates) and writes an `oats-config.yaml` listing them explicitly. A summary + and per-file warnings go to stderr; nothing to stdout. + +```sh +oats migrate ./old-case.yaml > new-case.yaml # one file +oats migrate . # whole tree, in place +``` + +Legacy docker-compose/kubernetes fixtures become case-local `fixture:` blocks. +Migration is best-effort: review the warnings (e.g. multi-entry matrices and +`compose-logs` are not auto-converted). + +### `oats cache clear` + +Delete all cached results under `--cache-dir` (default `$XDG_STATE_HOME/oats`). +The cache lets a re-run skip cases whose `(case, fixture, gcx version, oats +version)` are unchanged and previously passed; clear it to force a full run. + +### `oats version` + +Print the oats version and exit. diff --git a/examples/fixtures/cases/compose/rolldice.yaml b/examples/fixtures/compose/rolldice/oats-case.yaml similarity index 100% rename from examples/fixtures/cases/compose/rolldice.yaml rename to examples/fixtures/compose/rolldice/oats-case.yaml diff --git a/examples/fixtures/cases/k3d/rolldice.yaml b/examples/fixtures/k3d/rolldice/oats-case.yaml similarity index 84% rename from examples/fixtures/cases/k3d/rolldice.yaml rename to examples/fixtures/k3d/rolldice/oats-case.yaml index 71f649ea..377736b1 100644 --- a/examples/fixtures/cases/k3d/rolldice.yaml +++ b/examples/fixtures/k3d/rolldice/oats-case.yaml @@ -8,4 +8,4 @@ expected: - promql: 'http_server_active_requests{service_name="rolldice"}' value: '>= 0' custom-checks: - - script: ../../scripts/verify.sh + - script: ./verify.sh diff --git a/examples/fixtures/scripts/verify.sh b/examples/fixtures/k3d/rolldice/verify.sh similarity index 100% rename from examples/fixtures/scripts/verify.sh rename to examples/fixtures/k3d/rolldice/verify.sh diff --git a/examples/fixtures/oats-config.yaml b/examples/fixtures/oats-config.yaml index dcebca20..c5c228b7 100644 --- a/examples/fixtures/oats-config.yaml +++ b/examples/fixtures/oats-config.yaml @@ -2,11 +2,11 @@ meta: version: 3 suites: - name: compose-app - cases: ["cases/compose/*.yaml"] + cases: ["compose/*/oats-case.yaml"] fixture: compose-dev tags: [compose, app] - name: k3d-app - cases: ["cases/k3d/*.yaml"] + cases: ["k3d/*/oats-case.yaml"] fixture: k3d-dev tags: [k3d, app] fixture: diff --git a/examples/smoke/cases/custom-check.yaml b/examples/smoke/custom-check/oats-case.yaml similarity index 66% rename from examples/smoke/cases/custom-check.yaml rename to examples/smoke/custom-check/oats-case.yaml index 9790ab95..b98fb6cc 100644 --- a/examples/smoke/cases/custom-check.yaml +++ b/examples/smoke/custom-check/oats-case.yaml @@ -3,4 +3,4 @@ seed: type: app expected: custom-checks: - - script: ../scripts/verify.sh + - script: ./verify.sh diff --git a/examples/smoke/scripts/verify.sh b/examples/smoke/custom-check/verify.sh similarity index 100% rename from examples/smoke/scripts/verify.sh rename to examples/smoke/custom-check/verify.sh diff --git a/examples/smoke/cases/inline-seed.yaml b/examples/smoke/inline-seed/oats-case.yaml similarity index 100% rename from examples/smoke/cases/inline-seed.yaml rename to examples/smoke/inline-seed/oats-case.yaml diff --git a/examples/smoke/oats-config.yaml b/examples/smoke/oats-config.yaml index f5cd10a0..a9071931 100644 --- a/examples/smoke/oats-config.yaml +++ b/examples/smoke/oats-config.yaml @@ -2,7 +2,7 @@ meta: version: 3 suites: - name: smoke - cases: ["cases/*.yaml"] + cases: ["*/oats-case.yaml"] fixture: local-lgtm tags: [traces, logs, metrics] fixture: diff --git a/examples/smoke/cases/profile.yaml b/examples/smoke/profile/oats-case.yaml similarity index 100% rename from examples/smoke/cases/profile.yaml rename to examples/smoke/profile/oats-case.yaml diff --git a/examples/smoke/cases/rolldice.yaml b/examples/smoke/rolldice/oats-case.yaml similarity index 100% rename from examples/smoke/cases/rolldice.yaml rename to examples/smoke/rolldice/oats-case.yaml From 77b26ba8a4cce1679a498d8ecf0d1f9fe1c036ff Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 15:26:52 +0000 Subject: [PATCH 119/124] docs(examples): comment the example configs and cases Add explanatory comments to examples/smoke and examples/fixtures so a newcomer can follow them: what oats-config.yaml declares, what each fixture type does (remote vs OATS-booted compose/k3d), the seed modes (app vs inline-otlp), and what each assertion checks. No semantic changes. Signed-off-by: Gregor Zeitlinger --- .../fixtures/compose/rolldice/oats-case.yaml | 3 +++ examples/fixtures/k3d/rolldice/oats-case.yaml | 5 +++- examples/fixtures/oats-config.yaml | 25 +++++++++++++------ examples/smoke/custom-check/oats-case.yaml | 8 +++++- examples/smoke/inline-seed/oats-case.yaml | 11 +++++--- examples/smoke/oats-config.yaml | 18 ++++++++++--- examples/smoke/profile/oats-case.yaml | 8 ++++-- examples/smoke/rolldice/oats-case.yaml | 22 ++++++++++------ 8 files changed, 75 insertions(+), 25 deletions(-) diff --git a/examples/fixtures/compose/rolldice/oats-case.yaml b/examples/fixtures/compose/rolldice/oats-case.yaml index b05545f5..6c733f6a 100644 --- a/examples/fixtures/compose/rolldice/oats-case.yaml +++ b/examples/fixtures/compose/rolldice/oats-case.yaml @@ -1,8 +1,11 @@ +# Runs against the suite's compose-dev fixture (see ../../oats-config.yaml). name: compose fixture app smoke + seed: type: app input: - path: /rolldice?rolls=3 + expected: traces: - traceql: '{ resource.service.name = "rolldice" }' diff --git a/examples/fixtures/k3d/rolldice/oats-case.yaml b/examples/fixtures/k3d/rolldice/oats-case.yaml index 377736b1..a740a12a 100644 --- a/examples/fixtures/k3d/rolldice/oats-case.yaml +++ b/examples/fixtures/k3d/rolldice/oats-case.yaml @@ -1,11 +1,14 @@ +# Runs against the suite's k3d-dev fixture (see ../../oats-config.yaml). name: k3d fixture app smoke + seed: type: app input: - path: /rolldice?rolls=2 + expected: metrics: - promql: 'http_server_active_requests{service_name="rolldice"}' value: '>= 0' custom-checks: - - script: ./verify.sh + - script: ./verify.sh # colocated next to this case diff --git a/examples/fixtures/oats-config.yaml b/examples/fixtures/oats-config.yaml index c5c228b7..f82594be 100644 --- a/examples/fixtures/oats-config.yaml +++ b/examples/fixtures/oats-config.yaml @@ -1,25 +1,36 @@ +# This project shows the two fixtures OATS can boot and tear down itself +# (unlike examples/smoke, which points at a remote stack). Each suite uses a +# different fixture. NOTE: the referenced compose files / k8s dir are illustrative +# — this config is here to show the shape, not to run as-is. + meta: version: 3 + suites: - name: compose-app - cases: ["compose/*/oats-case.yaml"] + cases: ["compose/*/oats-case.yaml"] # one case per subdir under compose/ fixture: compose-dev tags: [compose, app] - name: k3d-app cases: ["k3d/*/oats-case.yaml"] fixture: k3d-dev tags: [k3d, app] + fixture: + # compose: OATS runs `docker compose up` on these files, waits for readiness, + # then tears them down. Add `template: lgtm` to also boot a grafana/otel-lgtm. compose-dev: compose: files: ["docker-compose.yml", "docker-compose.override.yml"] env: ["OTEL_EXPORTER_OTLP_ENDPOINT=http://lgtm:4318"] + # k3d: OATS spins up a k3d (k3s-in-docker) cluster, builds + imports the app + # image, applies the manifests in k8s_dir, and port-forwards to reach it. k3d-dev: k3d: - k8s_dir: k8s - app_service: rolldice - app_docker_file: Dockerfile - app_docker_context: . + k8s_dir: k8s # directory of k8s manifests to apply + app_service: rolldice # service exposing the app + app_docker_file: Dockerfile # build the app image from this Dockerfile… + app_docker_context: . # …with this build context app_docker_tag: rolldice:test - app_port: 18080 - import_images: ["grafana/otel-lgtm:latest"] + app_port: 18080 # app's container port + import_images: ["grafana/otel-lgtm:latest"] # images to load into the cluster diff --git a/examples/smoke/custom-check/oats-case.yaml b/examples/smoke/custom-check/oats-case.yaml index b98fb6cc..18573b30 100644 --- a/examples/smoke/custom-check/oats-case.yaml +++ b/examples/smoke/custom-check/oats-case.yaml @@ -1,6 +1,12 @@ +# A custom-check runs a script when the built-in assertions aren't enough: exit 0 +# = pass, non-zero = fail (retried like any assertion). The script runs with its +# cwd set to this case's directory and gets OATS_* env vars (OATS_GRAFANA_URL, +# OATS_OTLP_HTTP, …) — see docs/case-reference.md. name: custom check smoke + seed: type: app + expected: custom-checks: - - script: ./verify.sh + - script: ./verify.sh # path relative to this case dir (verify.sh sits next to it) diff --git a/examples/smoke/inline-seed/oats-case.yaml b/examples/smoke/inline-seed/oats-case.yaml index 5690fee4..b8c5979d 100644 --- a/examples/smoke/inline-seed/oats-case.yaml +++ b/examples/smoke/inline-seed/oats-case.yaml @@ -1,15 +1,20 @@ +# inline-otlp seeds telemetry directly — no app, no SDK. Use it to exercise the +# pipeline/backend in isolation or to pin an exact payload. Here we push one span +# and assert it comes back. name: inline seed smoke + seed: type: inline-otlp traces: - - service: gcx-e2e-seed + - service: gcx-e2e-seed # becomes resource.service.name spans: - - name: seed-operation + - name: seed-operation # kind defaults to INTERNAL, duration to 200ms + expected: traces: - traceql: '{ resource.service.name = "gcx-e2e-seed" }' match_spans: - name: seed-operation - attributes: + attributes: # optional attribute assertions on the matched span - key: service.name value: gcx-e2e-seed diff --git a/examples/smoke/oats-config.yaml b/examples/smoke/oats-config.yaml index a9071931..dcd65594 100644 --- a/examples/smoke/oats-config.yaml +++ b/examples/smoke/oats-config.yaml @@ -1,11 +1,21 @@ +# oats-config.yaml is the entry point of an OATS project. `oats` finds it in the +# current directory or any parent. It declares the suites, the cases each runs, +# and the fixtures (the backend the cases assert against). + meta: - version: 3 + version: 3 # OATS schema version (the single version; cases carry none) + suites: - name: smoke - cases: ["*/oats-case.yaml"] - fixture: local-lgtm - tags: [traces, logs, metrics] + cases: ["*/oats-case.yaml"] # one case per subdirectory (glob, relative to this file) + fixture: local-lgtm # the fixture (defined below) these cases run against + tags: [traces, logs, metrics] # optional labels; filter with `oats --tags traces` + fixture: + # A "remote" fixture points at an already-running stack — here a local + # grafana/otel-lgtm on the default OTLP port. OATS does not boot or tear it + # down; start it yourself, e.g. + # docker run -p 3000:3000 -p 4317:4317 -p 4318:4318 grafana/otel-lgtm local-lgtm: remote: endpoint: http://localhost:4318 diff --git a/examples/smoke/profile/oats-case.yaml b/examples/smoke/profile/oats-case.yaml index 862cd30e..c3ce3ea7 100644 --- a/examples/smoke/profile/oats-case.yaml +++ b/examples/smoke/profile/oats-case.yaml @@ -1,9 +1,13 @@ +# Profiles are queried from Pyroscope. They can't be inline-seeded, so assert +# them against an app-backed fixture that produces CPU profiles. name: profile smoke + seed: type: app + expected: profiles: - - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{}' + - query: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds{}' # Pyroscope query match: - match_type: regexp - name: 'main|worker' + name: 'main|worker' # a matching flamegraph frame diff --git a/examples/smoke/rolldice/oats-case.yaml b/examples/smoke/rolldice/oats-case.yaml index ea2d1285..5d226514 100644 --- a/examples/smoke/rolldice/oats-case.yaml +++ b/examples/smoke/rolldice/oats-case.yaml @@ -1,20 +1,28 @@ +# A case: drive an app, then assert on the telemetry it produced. This one +# exercises all three main signals (traces, metrics, logs). name: rolldice smoke + +# `input` HTTP requests drive the app so it emits telemetry (method defaults to GET). input: - path: /rolldice?rolls=5 + seed: - type: app - compose: docker-compose.app.yml + type: app # an instrumented app produces the telemetry + compose: docker-compose.app.yml # legacy shorthand to boot the app via compose + +# Each assertion names a query, then what must hold for its result. OATS retries +# every assertion until it passes or --timeout elapses. expected: traces: - - traceql: '{ resource.service.name = "rolldice" }' - match_spans: + - traceql: '{ resource.service.name = "rolldice" }' # TraceQL + match_spans: # a matching span must exist - match_type: regexp name: '^GET /rolldice.*$' metrics: - - promql: 'http_server_active_requests{service_name="rolldice"}' - value: '>= 0' + - promql: 'http_server_active_requests{service_name="rolldice"}' # PromQL + value: '>= 0' # compare the sample value logs: - - logql: '{service_name="rolldice"} |~ `rolling the dice`' + - logql: '{service_name="rolldice"} |~ `rolling the dice`' # LogQL match: - match_type: regexp name: '.*rolling the dice.*' From 8f12c7aeacb1a80f0e7f71061969f0809833ff1d Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 15:31:55 +0000 Subject: [PATCH 120/124] docs: explain suites vs cases (why the two layers exist) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a "Suites vs cases" section to case-reference — a suite groups cases that share one booted fixture, and is the unit of parallelism + filtering; top-level `cases:` is the shorthand when no grouping is needed. Comment the example configs to show the why: smoke is one suite (4 cases, one shared fixture), fixtures is two suites (one per distinct fixture, compose vs k3d). Signed-off-by: Gregor Zeitlinger --- docs/case-reference.md | 26 +++++++++++++++++++++++--- examples/fixtures/oats-config.yaml | 2 ++ examples/smoke/oats-config.yaml | 3 +++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/case-reference.md b/docs/case-reference.md index cb940bb1..6c0ea723 100644 --- a/docs/case-reference.md +++ b/docs/case-reference.md @@ -35,9 +35,29 @@ python/oats-case.yaml python/docker-compose.oats.yml Globs are shell-style (`*` matches one path segment; no recursive `**`), so add a segment for deeper trees (e.g. `examples/*/oats-case.yaml`) or list several -patterns. A top-level `cases:` list may be used instead of `suites:` for the -one-suite shape. A suite may name a shared fixture (from the `fixture:` map) or -let each case carry its own case-local `fixture:` block. +patterns. + +### Suites vs cases + +A **case** is one test. A **suite** is a named group of cases that **share one +booted fixture** — so the (often expensive) backend is stood up once and every +case in the suite runs against it. A suite is also the unit of parallelism +(`--parallel` runs *suites* concurrently; cases within a suite run in order) and +of `--suite` / `--tags` filtering. Use one suite per distinct fixture — e.g. a +`compose` suite and a `k3d` suite side by side. + +When you don't need that grouping, use a **top-level `cases:`** list instead of +`suites:`; each case then becomes its own single-case suite (and typically +carries its own case-local `fixture:` block): + +```yaml +meta: + version: 3 +cases: ["*/oats-case.yaml"] # no suites; one case per subdir, each self-contained +``` + +A suite either names a shared fixture (from the `fixture:` map) or lets each of +its cases carry a case-local `fixture:` block. ## Case yaml diff --git a/examples/fixtures/oats-config.yaml b/examples/fixtures/oats-config.yaml index f82594be..b08b75ea 100644 --- a/examples/fixtures/oats-config.yaml +++ b/examples/fixtures/oats-config.yaml @@ -6,6 +6,8 @@ meta: version: 3 +# Two suites because each needs a different fixture — a suite groups the cases +# that share one fixture, so a compose group and a k3d group are separate suites. suites: - name: compose-app cases: ["compose/*/oats-case.yaml"] # one case per subdir under compose/ diff --git a/examples/smoke/oats-config.yaml b/examples/smoke/oats-config.yaml index dcd65594..93b597c9 100644 --- a/examples/smoke/oats-config.yaml +++ b/examples/smoke/oats-config.yaml @@ -5,6 +5,9 @@ meta: version: 3 # OATS schema version (the single version; cases carry none) +# A suite groups cases that share one fixture (booted once, then every case runs +# against it) and is the unit of parallelism + filtering. These four cases all +# assert against the same local-lgtm stack, so they belong to one suite. suites: - name: smoke cases: ["*/oats-case.yaml"] # one case per subdirectory (glob, relative to this file) From fe5247b4c38067011a1c27618a499595a05037cc Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 16:03:11 +0000 Subject: [PATCH 121/124] feat(fixture): default the compose template to lgtm; add flagship python example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A compose fixture with no `template` now defaults to `template: lgtm`, so OATS boots a builtin grafana/otel-lgtm stack alongside the user's file/files (opt out with `template: none`). A case/suite with no fixture at all defaults to a booted lgtm stack too; external stacks now use an explicit `remote:` fixture. This lets a real-app case declare only its own compose file. Add examples/python — a real instrumented Flask "rolldice" app (app + Dockerfile + compose + case) as the flagship, called out in the README getting-started as the place to start; smoke/fixtures become the advanced examples. A dedicated black-box GitHub workflow (mise task `example-test`) builds oats+gcx and runs oats against it end to end. README: getting-started rebuilt around the real app, nav link renamed to "Test Case Syntax". Signed-off-by: Gregor Zeitlinger --- .github/renovate-tracked-deps.json | 5 ++ .github/workflows/example-python.yml | 36 ++++++++++++ README.md | 75 +++++++++++++------------ casefile/case.go | 29 ++++++++-- casefile/case_test.go | 26 +++++++++ discovery/discovery.go | 7 ++- discovery/discovery_test.go | 45 +++++++++++++++ docs/case-reference.md | 8 ++- examples/fixtures/oats-config.yaml | 4 +- examples/python/Dockerfile | 29 ++++++++++ examples/python/app.py | 27 +++++++++ examples/python/docker-compose.oats.yml | 16 ++++++ examples/python/oats-case.yaml | 32 +++++++++++ examples/python/oats-config.yaml | 4 ++ examples/python/requirements.txt | 7 +++ fixture/compose.go | 14 +++-- fixture/fixture.go | 2 +- fixture/fixture_test.go | 54 +++++++++++++----- internal/cli/main.go | 6 -- mise.toml | 9 +++ 20 files changed, 364 insertions(+), 71 deletions(-) create mode 100644 .github/workflows/example-python.yml create mode 100644 examples/python/Dockerfile create mode 100644 examples/python/app.py create mode 100644 examples/python/docker-compose.oats.yml create mode 100644 examples/python/oats-case.yaml create mode 100644 examples/python/oats-config.yaml create mode 100644 examples/python/requirements.txt diff --git a/.github/renovate-tracked-deps.json b/.github/renovate-tracked-deps.json index 0ce21129..ac6cec8c 100644 --- a/.github/renovate-tracked-deps.json +++ b/.github/renovate-tracked-deps.json @@ -16,6 +16,11 @@ "mise" ] }, + ".github/workflows/example-python.yml": { + "regex": [ + "mise" + ] + }, ".github/workflows/lint.yml": { "regex": [ "mise" diff --git a/.github/workflows/example-python.yml b/.github/workflows/example-python.yml new file mode 100644 index 00000000..483280bd --- /dev/null +++ b/.github/workflows/example-python.yml @@ -0,0 +1,36 @@ +name: Example (python) + +# Black-box test of the flagship python example: build oats + gcx, then run oats +# against examples/python (which boots its own grafana/otel-lgtm via the default +# compose template) and assert it passes — the same way a consumer would use it. +on: + pull_request: + paths: + - "examples/python/**" + - "fixture/**" + - "discovery/**" + - "casefile/**" + - "engine/**" + - "seed/**" + - "runner/**" + - "scripts/build-local-tools.sh" + - ".github/workflows/example-python.yml" + - "mise.toml" + workflow_dispatch: + +permissions: {} + +jobs: + example-python: + runs-on: ubuntu-24.04 + steps: + - name: Check out + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + with: + version: v2026.7.0 + sha256: 0744cb3c303baf0d308ff7b112ed41f22abb6029cb5644fd3a8ce74b29f16a68 + - name: Build oats + gcx and run the python example + run: mise run example-test diff --git a/README.md b/README.md index d59c76e9..77512c0a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@

CLI · - Case reference · + Test Case Syntax · CI · Upgrading

@@ -31,8 +31,7 @@ name: rolldice traces have a route attribute fixture: compose: - template: lgtm - file: docker-compose.oats.yml + file: docker-compose.oats.yml # your app; OATS adds a Grafana LGTM stack by default seed: type: app @@ -71,59 +70,63 @@ bootstrap gcx itself, but pinning it explicitly keeps runs reproducible. ## Getting started -An OATS project is a directory with an **`oats-config.yaml`** plus one case per -subdirectory. The smallest project that runs end-to-end is two files and Docker — -OATS boots a throwaway Grafana LGTM stack for you, so you need no running backend -and no app to see it work. +The best starting point is **[`examples/python/`](examples/python/)** — a real, +instrumented Flask "rolldice" app tested end to end. Copy it and run: -**`oats-config.yaml`** — lists the cases: - -```yaml -meta: - version: 3 -cases: ["*/oats-case.yaml"] # one case per subdirectory +```sh +cp -r examples/python my-oats-test && cd my-oats-test +oats # run it ``` -**`hello/oats-case.yaml`** — boot an LGTM stack, push a log, assert it lands: +With Docker running, `oats` finds `oats-config.yaml` (in the current directory or +any parent), builds the app and boots it next to a throwaway Grafana LGTM stack +(the default fixture — no `template: lgtm` needed), calls `/rolldice`, and checks +the telemetry it produced. The whole test is one file — `oats-case.yaml`: ```yaml -name: hello world +name: python rolldice fixture: compose: - template: lgtm # OATS boots grafana/otel-lgtm; no compose file needed + file: docker-compose.oats.yml # your app; OATS adds a Grafana LGTM stack by default + app_service: python # so OATS can reach the app… + app_port: 8082 # …on its container port seed: - type: inline-otlp # push telemetry directly — no instrumented app required - logs: - - service: hello - body: hello from oats + type: app +input: + - path: /rolldice # drive the app so it emits telemetry expected: + traces: + - traceql: '{ span.http.route = "/rolldice" }' + match_spans: + - name: GET /rolldice + metrics: + - promql: 'http_server_active_requests{http_method="GET"}' + value: '>= 0' logs: - - logql: '{service_name="hello"}' - contains: hello from oats + - logql: '{service_name="rolldice"} |~ `rolling the dice`' + regex: 'rolling the dice' ``` -Then, from the project directory: +No app of your own yet? A case can seed telemetry directly with +`seed: {type: inline-otlp}` and skip the app entirely — see +[docs/case-reference.md](docs/case-reference.md). -```sh -oats list # show the resolved plan -oats # boot the fixture, seed it, assert -``` +## Examples -To test a **real app** instead of inline telemetry, point the fixture at your -app's compose file and use `seed: {type: app}` with `input:` requests. The -runnable **[`examples/`](examples/)** projects are the best starting point to -copy from — `examples/smoke/` (remote fixture + assertions) and -`examples/fixtures/` (compose and k3d) — see [docs/case-reference.md](docs/case-reference.md) -for the full shape. +- **[`examples/python/`](examples/python/)** — flagship: a real Flask app + + compose fixture, asserting traces, metrics, and logs. **Start here.** +- **[`examples/smoke/`](examples/smoke/)** — advanced: a remote fixture with + inline-otlp / app / profile / custom-check cases. +- **[`examples/fixtures/`](examples/fixtures/)** — advanced: compose and k3d + fixtures side by side. ## Documentation - **[docs/cli.md](docs/cli.md)** — every command and flag -- **[docs/case-reference.md](docs/case-reference.md)** — the full config + case - shape: fixtures, seed modes, the assertion vocabulary, custom checks +- **[docs/case-reference.md](docs/case-reference.md)** — test case syntax: the + full config + case shape (fixtures, seed modes, assertion vocabulary, custom checks) - **[docs/ci.md](docs/ci.md)** — installing and running OATS in CI, plus result caching and its caveats - **[UPGRADING.md](UPGRADING.md)** — migrating older (schema-2) repos to v3 - **[AGENTS.md](AGENTS.md)** — for contributors and coding agents working *on* OATS (build, layout, conventions) -- **[`examples/`](examples/)** — small runnable projects to copy from diff --git a/casefile/case.go b/casefile/case.go index 2f790e38..8edf53b0 100644 --- a/casefile/case.go +++ b/casefile/case.go @@ -64,9 +64,13 @@ type FixtureConfig struct { } // ComposeFixture boots a docker-compose stack. template selects a built-in -// stack ("lgtm"); file/files point at compose files relative to the fixture -// source dir. app_service + app_port let OATS publish and discover an -// ephemeral host port for the application under test. +// stack; file/files point at compose files relative to the fixture source dir, +// merged alongside the template. app_service + app_port let OATS publish and +// discover an ephemeral host port for the application under test. +// +// template defaults to "lgtm" when unset, so OATS boots a builtin +// grafana/otel-lgtm stack next to the user's file/files. Set template: "none" +// to opt out and bring your own observability stack via file/files. type ComposeFixture struct { Template string `yaml:"template,omitempty"` File string `yaml:"file,omitempty"` @@ -76,6 +80,16 @@ type ComposeFixture struct { AppPort int `yaml:"app_port,omitempty"` } +// EffectiveTemplate returns the compose template to apply, defaulting an unset +// template to "lgtm" so the builtin stack is booted unless the case explicitly +// opts out with "none". +func (c ComposeFixture) EffectiveTemplate() string { + if c.Template == "" { + return "lgtm" + } + return c.Template +} + // K3DFixture boots a k3d cluster and builds/imports the application image. type K3DFixture struct { K8sDir string `yaml:"k8s_dir,omitempty"` @@ -467,12 +481,15 @@ func (f FixtureConfig) Validate(label string) error { switch { case f.Compose != nil: c := f.Compose - if c.Template == "" && c.File == "" && len(c.Files) == 0 { - return fmt.Errorf("fixture %q: compose requires template, file, or files", label) - } if c.File != "" && len(c.Files) > 0 { return fmt.Errorf("fixture %q: compose sets file or files, not both", label) } + // template defaults to lgtm, so a compose block is always valid on the + // template axis. Only template=none (bring-your-own-stack) needs an + // explicit file/files to boot anything. + if c.EffectiveTemplate() == "none" && c.File == "" && len(c.Files) == 0 { + return fmt.Errorf("fixture %q: compose template=none requires file or files", label) + } case f.K3D != nil: k := f.K3D if k.K8sDir == "" || k.AppService == "" || k.AppDockerFile == "" || k.AppDockerTag == "" || k.AppPort == 0 { diff --git a/casefile/case_test.go b/casefile/case_test.go index e985ac0f..74ea2ffd 100644 --- a/casefile/case_test.go +++ b/casefile/case_test.go @@ -328,6 +328,32 @@ expected: } } +func TestValidate_EmptyComposeDefaultsToLGTM(t *testing.T) { + // A compose block with no template/file/files is valid: template defaults to + // lgtm, so OATS boots the builtin stack. + f := FixtureConfig{Compose: &ComposeFixture{}} + if err := f.Validate("fixture"); err != nil { + t.Fatalf("expected empty compose to validate (defaults to lgtm), got %v", err) + } + if got := f.Compose.EffectiveTemplate(); got != "lgtm" { + t.Fatalf("EffectiveTemplate: got %q, want lgtm", got) + } +} + +func TestValidate_ComposeTemplateNoneRequiresFile(t *testing.T) { + f := FixtureConfig{Compose: &ComposeFixture{Template: "none"}} + err := f.Validate("fixture") + if err == nil || !strings.Contains(err.Error(), "compose template=none requires file or files") { + t.Fatalf("expected template=none requires-file error, got %v", err) + } + + // template=none with a file is valid (bring-your-own-stack). + ok := FixtureConfig{Compose: &ComposeFixture{Template: "none", File: "docker-compose.yml"}} + if err := ok.Validate("fixture"); err != nil { + t.Fatalf("expected template=none with file to validate, got %v", err) + } +} + func TestValidate_RejectsInvalidMatchRegexp(t *testing.T) { _, err := Parse([]byte(` name: bad regexp diff --git a/discovery/discovery.go b/discovery/discovery.go index 1bc5dfb2..cdc8f023 100644 --- a/discovery/discovery.go +++ b/discovery/discovery.go @@ -349,7 +349,12 @@ func (c *RootConfig) resolveSuiteFixture(suite SuiteConfig, cases []*casefile.Ca } } if !seen { - return casefile.FixtureConfig{}, "", nil + // No suite fixture and no case declares one: default to a compose + // fixture with the builtin lgtm template (an empty ComposeFixture + // resolves to template=lgtm). This boots just the lgtm stack, which is + // handy for inline-otlp smoke tests. External stacks now require an + // explicit remote: fixture. Temp compose files land next to the config. + return casefile.FixtureConfig{Compose: &casefile.ComposeFixture{}}, c.SourceDir, nil } return fixture, sourceDir, nil } diff --git a/discovery/discovery_test.go b/discovery/discovery_test.go index fe50bdcc..28fd4765 100644 --- a/discovery/discovery_test.go +++ b/discovery/discovery_test.go @@ -204,6 +204,51 @@ suites: } } +func TestPlanRun_NoFixtureDefaultsToComposeLGTM(t *testing.T) { + // A suite with no fixture (and cases that declare none) now defaults to a + // compose fixture booting the builtin lgtm stack, not an external setup. + dir := t.TempDir() + writeFile(t, dir, "oats-config.yaml", ` +meta: + version: 3 +suites: + - name: inline + cases: ["cases/*.yaml"] +`) + writeFile(t, dir, "cases/a.yaml", `name: inline smoke +seed: + type: inline-otlp + logs: + - service: a + body: line +expected: + logs: + - logql: '{service_name="a"}' + contains: line +`) + + cfg, err := Load(filepath.Join(dir, "oats-config.yaml")) + if err != nil { + t.Fatalf("Load: %v", err) + } + plans, err := cfg.PlanRun(Filter{}) + if err != nil { + t.Fatalf("PlanRun: %v", err) + } + if len(plans) != 1 { + t.Fatalf("plans: got %d, want 1", len(plans)) + } + if plans[0].Fixture.Kind() != "compose" { + t.Fatalf("expected no-fixture suite to default to compose, got %q (%+v)", plans[0].Fixture.Kind(), plans[0].Fixture) + } + if got := plans[0].Fixture.Compose.EffectiveTemplate(); got != "lgtm" { + t.Fatalf("expected default compose template lgtm, got %q", got) + } + if plans[0].FixtureSourceDir != dir { + t.Fatalf("expected fixture source dir %q, got %q", dir, plans[0].FixtureSourceDir) + } +} + func TestValidate_EmptyFixtureRejected(t *testing.T) { cfg := &RootConfig{ Meta: Meta{Version: 3}, diff --git a/docs/case-reference.md b/docs/case-reference.md index 6c0ea723..43826c50 100644 --- a/docs/case-reference.md +++ b/docs/case-reference.md @@ -99,9 +99,15 @@ one of three nested blocks; the block you set selects how the stack is stood up | Block | Meaning | |-------|---------| | `remote` | point at an already-running stack (`endpoint:` / a gcx context) | -| `compose` | OATS boots a docker-compose stack; `template: lgtm` lets OATS own the LGTM ports | +| `compose` | OATS boots a docker-compose stack; `template` defaults to `lgtm`, booting a builtin grafana/otel-lgtm alongside your `file`/`files`. Set `template: none` to bring your own stack | | `k3d` | OATS boots a k3d (k3s-in-docker) cluster | +A `compose` fixture with no `template` defaults to `template: lgtm`, so OATS +boots the builtin LGTM stack next to your `file`/`files`. Omit `file`/`files` +entirely (or drop the `fixture:` block altogether) to boot just the LGTM stack — +handy for `inline-otlp` smoke tests. To skip the builtin stack, set +`template: none` (then `file`/`files` are required). + ```yaml fixture: remote: diff --git a/examples/fixtures/oats-config.yaml b/examples/fixtures/oats-config.yaml index b08b75ea..2660353b 100644 --- a/examples/fixtures/oats-config.yaml +++ b/examples/fixtures/oats-config.yaml @@ -20,7 +20,9 @@ suites: fixture: # compose: OATS runs `docker compose up` on these files, waits for readiness, - # then tears them down. Add `template: lgtm` to also boot a grafana/otel-lgtm. + # then tears them down. A compose fixture defaults to `template: lgtm`, so OATS + # boots a grafana/otel-lgtm stack alongside these files — that's the `lgtm` + # service the env below points at. Set `template: none` to bring your own stack. compose-dev: compose: files: ["docker-compose.yml", "docker-compose.override.yml"] diff --git a/examples/python/Dockerfile b/examples/python/Dockerfile new file mode 100644 index 00000000..d890505d --- /dev/null +++ b/examples/python/Dockerfile @@ -0,0 +1,29 @@ +FROM python:alpine3.19@sha256:8287ca207e905649e9f399b5f91a119e5e9051d8cd110d5f8c3b4bd9458ebd1d + +WORKDIR /app + +COPY requirements.txt . + +# The requirements were generated by following the OpenTelemetry Python getting +# started guide and running `pip freeze > requirements.txt`. +# Alpine package revisions are inherited from the Python base image. +# hadolint ignore=DL3018 +RUN apk add --no-cache build-base +# hadolint ignore=DL3013 +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +# renovate: datasource=pypi depName=opentelemetry-distro +ENV OPENTELEMETRY_DISTRO_VERSION=0.64b0 +RUN pip install --no-cache-dir "opentelemetry-distro[otlp]==$OPENTELEMETRY_DISTRO_VERSION" && \ + opentelemetry-bootstrap -a install + +COPY app.py . + +# Log auto-instrumentation is still alpha, so enable it explicitly. +ENV OTEL_PYTHON_LOGGING_AUTO_INSTRUMENTATION_ENABLED=true +ENV OTEL_LOGS_EXPORTER=otlp + +EXPOSE 8082 + +CMD ["opentelemetry-instrument", "flask", "run", "--host", "0.0.0.0", "--port", "8082"] diff --git a/examples/python/app.py b/examples/python/app.py new file mode 100644 index 00000000..0579e897 --- /dev/null +++ b/examples/python/app.py @@ -0,0 +1,27 @@ +"""Simple Flask app that rolls a dice.""" + +import logging +from random import randint + +from flask import Flask, request + +app = Flask(__name__) +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +@app.route("/rolldice") +def roll_dice(): + """Rolls a dice and returns the result.""" + player = request.args.get("player", default=None, type=str) + result = str(roll()) + if player: + logger.warning("%s is rolling the dice: %s", player, result) + else: + logger.warning("Anonymous player is rolling the dice: %s", result) + return result + + +def roll(): + """Rolls a dice and returns the result.""" + return randint(1, 6) diff --git a/examples/python/docker-compose.oats.yml b/examples/python/docker-compose.oats.yml new file mode 100644 index 00000000..90879e2e --- /dev/null +++ b/examples/python/docker-compose.oats.yml @@ -0,0 +1,16 @@ +--- +# The app under test. OATS boots this alongside a Grafana LGTM stack (the default +# fixture template) and points the app's OTLP exporter at it via the `lgtm` host. +services: + python: + build: + context: . + dockerfile: Dockerfile + environment: + OTEL_SERVICE_NAME: "rolldice" + OTEL_EXPORTER_OTLP_ENDPOINT: http://lgtm:4317 + OTEL_METRIC_EXPORT_INTERVAL: "5000" # so we don't wait 60s for metrics + ports: + # Ephemeral host port so parallel suites don't collide; OATS discovers it + # via fixture.app_service + app_port. + - "127.0.0.1::8082" diff --git a/examples/python/oats-case.yaml b/examples/python/oats-case.yaml new file mode 100644 index 00000000..8ac042c0 --- /dev/null +++ b/examples/python/oats-case.yaml @@ -0,0 +1,32 @@ +# Acceptance-test a real instrumented app end to end: OATS boots this Flask +# "rolldice" app next to a Grafana LGTM stack, calls /rolldice, then asserts on +# the trace, metric, and log the app actually produced. +name: python rolldice + +fixture: + compose: + # Your app's compose file. OATS adds a grafana/otel-lgtm stack by default + # (template: lgtm) — set `template: none` to bring your own. + file: docker-compose.oats.yml + app_service: python # the app service in the compose file… + app_port: 8082 # …and its container port, so OATS can drive it + +seed: + type: app # an instrumented app produces the telemetry +input: + - path: /rolldice # drive it so it emits a trace, metric, and log + +expected: + traces: + - traceql: '{ span.http.route = "/rolldice" }' + match_spans: + - name: GET /rolldice + attributes: + - key: otel.library.name + value: opentelemetry.instrumentation.flask + metrics: + - promql: 'http_server_active_requests{http_method="GET"}' + value: '>= 0' + logs: + - logql: '{service_name="rolldice"} |~ `Anonymous player is rolling the dice.*`' + regex: 'Anonymous player is rolling the dice' diff --git a/examples/python/oats-config.yaml b/examples/python/oats-config.yaml new file mode 100644 index 00000000..2aee6f9b --- /dev/null +++ b/examples/python/oats-config.yaml @@ -0,0 +1,4 @@ +# A single-project OATS config: one case, in this directory. +meta: + version: 3 +cases: ["oats-case.yaml"] diff --git a/examples/python/requirements.txt b/examples/python/requirements.txt new file mode 100644 index 00000000..83bf1bd9 --- /dev/null +++ b/examples/python/requirements.txt @@ -0,0 +1,7 @@ +blinker==1.9.0 +click==8.4.2 +Flask==3.1.3 +itsdangerous==2.2.0 +Jinja2==3.1.6 +MarkupSafe==3.0.3 +Werkzeug==3.1.8 diff --git a/fixture/compose.go b/fixture/compose.go index 52861aba..ea164a7f 100644 --- a/fixture/compose.go +++ b/fixture/compose.go @@ -91,15 +91,19 @@ func startCompose(plan discovery.Plan) (Handle, Runtime, error) { func resolveComposeFiles(sourceDir string, compose *casefile.ComposeFixture) ([]string, func() error, error) { var files []string var cleanup func() error - if compose.Template == "lgtm" { + template := compose.EffectiveTemplate() + switch template { + case "lgtm": f, err := writeBuiltinLGTMCompose(sourceDir) if err != nil { return nil, nil, err } files = append(files, f) cleanup = func() error { return os.Remove(f) } - } else if compose.Template != "" { - return nil, nil, fmt.Errorf("unsupported compose fixture template %q", compose.Template) + case "none": + // Bring-your-own-stack: no builtin is booted, only the user's file/files. + default: + return nil, nil, fmt.Errorf("unsupported compose template %q", compose.Template) } switch { case compose.File != "": @@ -108,8 +112,8 @@ func resolveComposeFiles(sourceDir string, compose *casefile.ComposeFixture) ([] for _, file := range compose.Files { files = append(files, filepath.Join(sourceDir, file)) } - case compose.Template == "": - return nil, nil, fmt.Errorf("compose fixture requires file, files, or supported template") + case template == "none": + return nil, nil, fmt.Errorf("compose template=none requires file or files") } return files, cleanup, nil } diff --git a/fixture/fixture.go b/fixture/fixture.go index f9eb16d0..0581183a 100644 --- a/fixture/fixture.go +++ b/fixture/fixture.go @@ -111,7 +111,7 @@ func SupportsParallel(plan discovery.Plan) (bool, string) { case "", "remote": return true, "" case "compose": - if plan.Fixture.Compose.Template != "lgtm" { + if plan.Fixture.Compose.EffectiveTemplate() != "lgtm" { return false, "compose fixtures are only parallel-safe when OATS owns the LGTM ports via template=lgtm" } for _, c := range plan.Cases { diff --git a/fixture/fixture_test.go b/fixture/fixture_test.go index 49608951..c14c4cf0 100644 --- a/fixture/fixture_test.go +++ b/fixture/fixture_test.go @@ -14,29 +14,52 @@ import ( "github.com/grafana/oats/testhelpers/remote" ) +func isBuiltinLGTMFile(path string) bool { + return strings.Contains(filepath.Base(path), ".oats.lgtm.") && strings.HasSuffix(path, ".compose.yml") +} + func TestResolveComposeFiles(t *testing.T) { - got, cleanup, err := resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{File: "stack/compose.yml"}) + // template=none with a single file: only the user's file, no builtin, no + // cleanup. + got, cleanup, err := resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{Template: "none", File: "stack/compose.yml"}) if err != nil { - t.Fatalf("resolveComposeFiles file: %v", err) + t.Fatalf("resolveComposeFiles template=none file: %v", err) } if cleanup != nil { - t.Fatalf("unexpected cleanup for file fixture") + t.Fatalf("unexpected cleanup for template=none file fixture") } if want := []string{"/tmp/work/stack/compose.yml"}; len(got) != 1 || got[0] != want[0] { t.Fatalf("got %q want %q", got, want) } - got, cleanup, err = resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{Files: []string{"a.yml", "b.yml"}}) + // template=none with files: only the user's files, no builtin. + got, cleanup, err = resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{Template: "none", Files: []string{"a.yml", "b.yml"}}) if err != nil { - t.Fatalf("resolveComposeFiles files: %v", err) + t.Fatalf("resolveComposeFiles template=none files: %v", err) } if cleanup != nil { - t.Fatalf("unexpected cleanup for files fixture") + t.Fatalf("unexpected cleanup for template=none files fixture") } if len(got) != 2 || got[0] != "/tmp/work/a.yml" || got[1] != "/tmp/work/b.yml" { t.Fatalf("unexpected files resolution: %v", got) } + // No template + a user file: the builtin lgtm compose is merged in ahead of + // the user's file (default template=lgtm), with cleanup for the temp file. + got, cleanup, err = resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{File: "stack/compose.yml"}) + if err != nil { + t.Fatalf("resolveComposeFiles default template + file: %v", err) + } + if cleanup == nil { + t.Fatalf("expected cleanup for default (lgtm) fixture") + } + if len(got) != 2 || !isBuiltinLGTMFile(got[0]) || got[1] != "/tmp/work/stack/compose.yml" { + _ = cleanup() + t.Fatalf("unexpected default-template resolution: %v", got) + } + _ = cleanup() + + // Explicit template=lgtm with no file: just the builtin lgtm compose. got, cleanup, err = resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{Template: "lgtm"}) if err != nil { t.Fatalf("resolveComposeFiles template=lgtm: %v", err) @@ -45,7 +68,7 @@ func TestResolveComposeFiles(t *testing.T) { t.Fatalf("expected cleanup for template=lgtm fixture") } defer func() { _ = cleanup() }() - if len(got) != 1 || !strings.Contains(filepath.Base(got[0]), ".oats.lgtm.") || !strings.HasSuffix(got[0], ".compose.yml") { + if len(got) != 1 || !isBuiltinLGTMFile(got[0]) { t.Fatalf("unexpected template=lgtm resolution: %v", got) } } @@ -80,8 +103,11 @@ func TestStart_ComposeLifecycle(t *testing.T) { Suite: discovery.SuiteConfig{Name: "smoke", Fixture: "local"}, Fixture: casefile.FixtureConfig{ Compose: &casefile.ComposeFixture{ - Files: []string{"a.yml", "b.yml"}, - Env: []string{"FOO=bar"}, + // template=none keeps the resolved file list exactly a.yml/b.yml + // so this lifecycle test isn't perturbed by the builtin lgtm file. + Template: "none", + Files: []string{"a.yml", "b.yml"}, + Env: []string{"FOO=bar"}, }, }, FixtureSourceDir: "/tmp/work", @@ -232,15 +258,15 @@ func TestK3DCheckEnv_UsesConfiguredPorts(t *testing.T) { func TestResolveComposeFiles_UnsupportedTemplate(t *testing.T) { _, _, err := resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{Template: "weird"}) - if err == nil || !strings.Contains(err.Error(), `unsupported compose fixture template "weird"`) { + if err == nil || !strings.Contains(err.Error(), `unsupported compose template "weird"`) { t.Fatalf("expected unsupported template error, got %v", err) } } -func TestResolveComposeFiles_MissingConfig(t *testing.T) { - _, _, err := resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{}) - if err == nil || !strings.Contains(err.Error(), "compose fixture requires file, files, or supported template") { - t.Fatalf("expected missing compose config error, got %v", err) +func TestResolveComposeFiles_TemplateNoneRequiresFile(t *testing.T) { + _, _, err := resolveComposeFiles("/tmp/work", &casefile.ComposeFixture{Template: "none"}) + if err == nil || !strings.Contains(err.Error(), "compose template=none requires file or files") { + t.Fatalf("expected template=none missing-file error, got %v", err) } } diff --git a/internal/cli/main.go b/internal/cli/main.go index 1afef270..ab129ac8 100644 --- a/internal/cli/main.go +++ b/internal/cli/main.go @@ -432,12 +432,6 @@ func resolveEndpoint(plan discovery.Plan, rt fixture.Runtime, gcxContextOverride ep.OTLPHTTP = rt.OTLPHTTP } ep.CustomCheckEnv = append(ep.CustomCheckEnv, rt.CustomCheckEnv...) - case "": - // No fixture configured — caller (or --gcx-context) must supply - // everything. Useful while plumbing the new CLI against an external setup. - ep.CustomCheckEnv = append(ep.CustomCheckEnv, - "OATS_APP_URL="+fmt.Sprintf("http://%s:%d", ep.AppHost, ep.AppPort), - ) default: return ep, fmt.Errorf("fixture kind %q is not supported in oats", plan.Fixture.Kind()) } diff --git a/mise.toml b/mise.toml index fb96e5d7..9fb90728 100644 --- a/mise.toml +++ b/mise.toml @@ -40,6 +40,15 @@ run = 'GOFLAGS=-buildvcs=false go test $(GOFLAGS=-buildvcs=false go list ./... | description = "Run e2e case suite" run = "go test -v -buildvcs=false ./tests/e2e -run TestCases" +[tasks.example-test] +description = "Black-box test: build oats+gcx, then run the python example (needs Docker)" +run = ''' +root="$PWD" +./scripts/build-local-tools.sh "$root/bin" +cd examples/python +"$root/bin/oats" --gcx "$root/bin/gcx" --no-cache --timeout 90s --interval 2s -v +''' + [tasks.build] description = "Build the project" run = "go build -buildvcs=false ." From e5dee651c0aefa31dc40a32199ad2b501448835a Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 16:22:21 +0000 Subject: [PATCH 122/124] ci: track examples/python deps in renovate; broaden example-python triggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit config:recommended bundles :ignoreModulesAndTests, which silently excluded examples/** (and tests/**) from renovate, so examples/python deps were never tracked or updated. Disable that preset and add customManagers:dockerfileVersions to pick up the opentelemetry-distro ENV pin, matching docker-otel-lgtm. Regenerate renovate-tracked-deps.json. Also broaden the example-python workflow path filter to all Go source (**/*.go, go.mod, go.sum) rather than an enumerated package list that is easy to let drift — the example is a black-box test of the whole binary. Signed-off-by: Gregor Zeitlinger --- .github/renovate-tracked-deps.json | 60 ++++++++++++++++++++++++++++ .github/renovate.json5 | 6 +++ .github/workflows/example-python.yml | 11 +++-- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/.github/renovate-tracked-deps.json b/.github/renovate-tracked-deps.json index ac6cec8c..976c5b3a 100644 --- a/.github/renovate-tracked-deps.json +++ b/.github/renovate-tracked-deps.json @@ -31,6 +31,25 @@ "mise" ] }, + "examples/python/Dockerfile": { + "dockerfile": [ + "python" + ], + "regex": [ + "opentelemetry-distro" + ] + }, + "examples/python/requirements.txt": { + "pip_requirements": [ + "Flask", + "Jinja2", + "MarkupSafe", + "Werkzeug", + "blinker", + "click", + "itsdangerous" + ] + }, "go.mod": { "gomod": [ "github.com/cenkalti/backoff/v5", @@ -98,6 +117,47 @@ "typos" ] }, + "tests/e2e/Dockerfile": { + "dockerfile": [ + "ghcr.io/open-telemetry/opentelemetry-operator/autoinstrumentation-java", + "springio/petclinic" + ] + }, + "tests/e2e/cases/fixture/compose-app-seed/files/docker-compose.oats.yml": { + "docker-compose": [ + "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen" + ] + }, + "tests/e2e/cases/fixture/compose-logs-fail/files/docker-compose.oats.yml": { + "docker-compose": [ + "alpine" + ] + }, + "tests/e2e/cases/fixture/compose-logs/files/docker-compose.oats.yml": { + "docker-compose": [ + "alpine" + ] + }, + "tests/e2e/cases/fixture/k3d-fail/files/Dockerfile": { + "dockerfile": [ + "docker.io/library/alpine" + ] + }, + "tests/e2e/cases/fixture/k3d-smoke/files/Dockerfile": { + "dockerfile": [ + "docker.io/library/alpine" + ] + }, + "tests/e2e/cases/fixture/parallel-compose/files/docker-compose.oats.yml": { + "docker-compose": [ + "alpine" + ] + }, + "tests/e2e/cases/fixture/remote-basic/files/docker-compose.remote.yml": { + "docker-compose": [ + "docker.io/grafana/otel-lgtm" + ] + }, "yaml/testdata/docker-compose-addition.yaml": { "docker-compose": [ "mongo" diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 89d6bc21..56b5aff0 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -3,7 +3,13 @@ extends: [ "config:best-practices", "config:recommended", + // Track `# renovate:`-annotated ENV pins in Dockerfiles (opentelemetry-distro). + "customManagers:dockerfileVersions", "github>grafana/flint#v0.22.6", ], + // config:recommended pulls in :ignoreModulesAndTests, which ignores + // **/examples/** (and tests, vendor, …). Disable it so example deps + // (examples/python Dockerfile, requirements.txt) get tracked/updated. + ignorePresets: [":ignoreModulesAndTests"], automerge: true, } diff --git a/.github/workflows/example-python.yml b/.github/workflows/example-python.yml index 483280bd..1a670adc 100644 --- a/.github/workflows/example-python.yml +++ b/.github/workflows/example-python.yml @@ -6,13 +6,12 @@ name: Example (python) on: pull_request: paths: + # Black-box test of the whole oats binary — any Go change can affect it, + # so match all Go source rather than naming packages (easy to forget one). + - "**/*.go" + - "go.mod" + - "go.sum" - "examples/python/**" - - "fixture/**" - - "discovery/**" - - "casefile/**" - - "engine/**" - - "seed/**" - - "runner/**" - "scripts/build-local-tools.sh" - ".github/workflows/example-python.yml" - "mise.toml" From e74bf9028ccec97d18be310d4a7a84db2fbce10c Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 18:23:14 +0200 Subject: [PATCH 123/124] remove dep task Signed-off-by: Gregor Zeitlinger --- mise.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mise.toml b/mise.toml index 9fb90728..4fb461d9 100644 --- a/mise.toml +++ b/mise.toml @@ -56,7 +56,3 @@ run = "go build -buildvcs=false ." [tasks.check] description = "Run all checks" depends = ["lint", "test"] - -[tasks.deps] -description = "Update dependencies" -run = "go mod tidy" From bacd92954bdaf383b1fe2ebb61b9775801e7a2e3 Mon Sep 17 00:00:00 2001 From: Gregor Zeitlinger Date: Wed, 8 Jul 2026 19:48:57 +0000 Subject: [PATCH 124/124] fix: address Copilot review comments (leaks, cancellation, portability) Resolve the remaining actionable Copilot findings on #334: - report/text.go: escape the GHA annotation file= property (:, ,) so paths with colons don't corrupt the annotation (message body was already escaped). - testhelpers/compose: drain stdout to EOF (log chunk before checking err) so a trailing line without a newline isn't dropped and the child can't block on a full pipe; add mergeEnv so provided vars (e.g. COMPOSE_PROJECT_NAME) deterministically override parent-env duplicates. - testhelpers/kubernetes: teardown attempts all kills AND the cluster delete (errors.Join) instead of bailing on the first kill error; only port-forward Grafana/OTLP when those ports are explicitly set (v3 k3d always allocates them; legacy v1 callers left them 0 and got fixed-port collisions). - fixture/k3d.go: best-effort ep.Stop when ep.Start fails, to avoid leaking a partially-created cluster. - wait.go: Until() no longer reports success when the context is already cancelled (consistent with While). - remote endpoint: EffectiveOTLPHTTPPort falls back to TracesHTTPPort. - legacyyaml (was yaml/): quote paths with %q and return errors from absolutePath instead of panicking. - seed_test.go: drop dead `before` var; internal/cli/testdata/fake-gcx.sh: portable last-arg (bash 3.2 lacks ${*: -1}); not-contains-fail tag naming; AGENTS.md list/version subcommands. Move the legacy parser out of the public API: yaml/ -> internal/legacyyaml/ (only importer is migrate). Update flint excludes + regenerate tracked deps. Fold the python example into e2e_test.yml, reusing the build-e2e-tools artifact instead of a second build; example-test reuses OATS_EXAMPLE_BIN_DIR when set. Drop the standalone example-python workflow. gitignore built bins. Signed-off-by: Gregor Zeitlinger --- .github/config/flint.toml | 6 ++- .github/renovate-tracked-deps.json | 37 ++++++++--------- .github/workflows/e2e_test.yml | 26 ++++++++++++ .github/workflows/example-python.yml | 35 ---------------- .gitignore | 4 ++ AGENTS.md | 11 +++-- fixture/k3d.go | 1 + internal/cli/testdata/fake-gcx.sh | 4 +- .../docker-compose-docker-lgtm-template.yml | 0 {yaml => internal/legacyyaml}/generator.go | 2 +- .../legacyyaml}/generator_test.go | 2 +- {yaml => internal/legacyyaml}/logs.go | 2 +- {yaml => internal/legacyyaml}/logs_test.go | 2 +- {yaml => internal/legacyyaml}/metrics.go | 2 +- {yaml => internal/legacyyaml}/profiles.go | 2 +- {yaml => internal/legacyyaml}/runner.go | 2 +- {yaml => internal/legacyyaml}/testcase.go | 33 +++++++++------ .../legacyyaml}/testcase_test.go | 14 ++++--- .../testdata/docker-compose-addition.yaml | 0 .../testdata/docker-compose-expected.yaml | 0 .../testdata/docker-compose-template.yaml | 0 .../invalid-tests/malformed-yaml.yaml | 0 .../invalid-tests/outdated-version.yaml | 0 .../testdata/invalid-tests/unknown-field.yaml | 0 .../invalid-tests/version-not-int.yaml | 0 .../legacyyaml}/testdata/loki_response.json | 0 .../legacyyaml}/testdata/oats-merged.yaml | 0 .../valid-tests/expect-absent.oats.yaml | 0 .../testdata/valid-tests/ignored/.oatsignore | 0 .../ignored/should-not-appear.oats.yaml | 0 .../testdata/valid-tests/input.oats.yaml | 0 .../valid-tests/matrix-test.oats.yaml | 0 .../testdata/valid-tests/more-oats.yml | 0 .../testdata/valid-tests/oats-template.yaml | 0 .../testdata/valid-tests/oats.yaml | 0 {yaml => internal/legacyyaml}/traces.go | 2 +- {yaml => internal/legacyyaml}/validate.go | 2 +- .../legacyyaml}/validate_test.go | 2 +- migrate/migrate.go | 2 +- migrate/migrate_test.go | 4 +- mise.toml | 9 ++++- report/report_test.go | 26 ++++++++++++ report/text.go | 15 +++++++ seed/seed_test.go | 3 -- testhelpers/compose/compose.go | 40 ++++++++++++++++--- testhelpers/kubernetes/kubernetes.go | 28 ++++++++----- testhelpers/kubernetes/kubernetes_test.go | 18 ++------- .../remote/remote_observability_endpoint.go | 3 ++ .../cases/assert/not-contains-fail/test.yaml | 2 +- wait/wait.go | 6 +++ wait/wait_test.go | 18 +++++++++ 51 files changed, 236 insertions(+), 129 deletions(-) delete mode 100644 .github/workflows/example-python.yml rename {yaml => internal/legacyyaml}/docker-compose-docker-lgtm-template.yml (100%) rename {yaml => internal/legacyyaml}/generator.go (99%) rename {yaml => internal/legacyyaml}/generator_test.go (98%) rename {yaml => internal/legacyyaml}/logs.go (98%) rename {yaml => internal/legacyyaml}/logs_test.go (98%) rename {yaml => internal/legacyyaml}/metrics.go (98%) rename {yaml => internal/legacyyaml}/profiles.go (98%) rename {yaml => internal/legacyyaml}/runner.go (99%) rename {yaml => internal/legacyyaml}/testcase.go (88%) rename {yaml => internal/legacyyaml}/testcase_test.go (88%) rename {yaml => internal/legacyyaml}/testdata/docker-compose-addition.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/docker-compose-expected.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/docker-compose-template.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/invalid-tests/malformed-yaml.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/invalid-tests/outdated-version.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/invalid-tests/unknown-field.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/invalid-tests/version-not-int.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/loki_response.json (100%) rename {yaml => internal/legacyyaml}/testdata/oats-merged.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/valid-tests/expect-absent.oats.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/valid-tests/ignored/.oatsignore (100%) rename {yaml => internal/legacyyaml}/testdata/valid-tests/ignored/should-not-appear.oats.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/valid-tests/input.oats.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/valid-tests/matrix-test.oats.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/valid-tests/more-oats.yml (100%) rename {yaml => internal/legacyyaml}/testdata/valid-tests/oats-template.yaml (100%) rename {yaml => internal/legacyyaml}/testdata/valid-tests/oats.yaml (100%) rename {yaml => internal/legacyyaml}/traces.go (98%) rename {yaml => internal/legacyyaml}/validate.go (98%) rename {yaml => internal/legacyyaml}/validate_test.go (99%) diff --git a/.github/config/flint.toml b/.github/config/flint.toml index f4826089..65dc0084 100644 --- a/.github/config/flint.toml +++ b/.github/config/flint.toml @@ -1,5 +1,9 @@ [settings] -exclude = ["CHANGELOG.md", "yaml/testdata/**", "yaml/docker-compose-docker-lgtm-template.yml"] +exclude = [ + "CHANGELOG.md", + "internal/legacyyaml/testdata/**", + "internal/legacyyaml/docker-compose-docker-lgtm-template.yml", +] [checks.renovate-deps] exclude_managers = ["github-actions", "github-runners"] diff --git a/.github/renovate-tracked-deps.json b/.github/renovate-tracked-deps.json index 976c5b3a..ad2c2a0f 100644 --- a/.github/renovate-tracked-deps.json +++ b/.github/renovate-tracked-deps.json @@ -16,11 +16,6 @@ "mise" ] }, - ".github/workflows/example-python.yml": { - "regex": [ - "mise" - ] - }, ".github/workflows/lint.yml": { "regex": [ "mise" @@ -94,6 +89,22 @@ "gopkg.in/yaml.v3" ] }, + "internal/legacyyaml/testdata/docker-compose-addition.yaml": { + "docker-compose": [ + "mongo" + ] + }, + "internal/legacyyaml/testdata/docker-compose-expected.yaml": { + "docker-compose": [ + "grafana/grafana", + "mongo" + ] + }, + "internal/legacyyaml/testdata/docker-compose-template.yaml": { + "docker-compose": [ + "grafana/grafana" + ] + }, "mise.toml": { "mise": [ "actionlint", @@ -157,22 +168,6 @@ "docker-compose": [ "docker.io/grafana/otel-lgtm" ] - }, - "yaml/testdata/docker-compose-addition.yaml": { - "docker-compose": [ - "mongo" - ] - }, - "yaml/testdata/docker-compose-expected.yaml": { - "docker-compose": [ - "grafana/grafana", - "mongo" - ] - }, - "yaml/testdata/docker-compose-template.yaml": { - "docker-compose": [ - "grafana/grafana" - ] } } } diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index ad6f1b31..1dd03562 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -78,3 +78,29 @@ jobs: mise run e2e-test end_ts="$(date +%s)" echo "e2e timing: shard=${{ matrix.shard }} total=$((end_ts-start_ts))s" + + # Black-box test of the flagship python example: run the already-built oats + + # gcx (from build-e2e-tools) against examples/python, which boots its own + # grafana/otel-lgtm via the default compose template — the same way a consumer + # would use it. Reuses the e2e-tools artifact rather than rebuilding. + example-python: + needs: build-e2e-tools + runs-on: ubuntu-24.04 + steps: + - name: Check out + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 + with: + version: v2026.7.0 + sha256: 0744cb3c303baf0d308ff7b112ed41f22abb6029cb5644fd3a8ce74b29f16a68 + - name: Download e2e tools + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: e2e-tools + path: .e2e-tools/bin + - name: Restore e2e tool permissions + run: chmod +x .e2e-tools/bin/oats .e2e-tools/bin/gcx + - name: Run the python example + run: OATS_EXAMPLE_BIN_DIR="$PWD/.e2e-tools/bin" mise run example-test diff --git a/.github/workflows/example-python.yml b/.github/workflows/example-python.yml deleted file mode 100644 index 1a670adc..00000000 --- a/.github/workflows/example-python.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Example (python) - -# Black-box test of the flagship python example: build oats + gcx, then run oats -# against examples/python (which boots its own grafana/otel-lgtm via the default -# compose template) and assert it passes — the same way a consumer would use it. -on: - pull_request: - paths: - # Black-box test of the whole oats binary — any Go change can affect it, - # so match all Go source rather than naming packages (easy to forget one). - - "**/*.go" - - "go.mod" - - "go.sum" - - "examples/python/**" - - "scripts/build-local-tools.sh" - - ".github/workflows/example-python.yml" - - "mise.toml" - workflow_dispatch: - -permissions: {} - -jobs: - example-python: - runs-on: ubuntu-24.04 - steps: - - name: Check out - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - uses: jdx/mise-action@e6a8b3978addb5a52f2b4cd9d91eafa7f0ab959d # v4.2.0 - with: - version: v2026.7.0 - sha256: 0744cb3c303baf0d308ff7b112ed41f22abb6029cb5644fd3a8ce74b29f16a68 - - name: Build oats + gcx and run the python example - run: mise run example-test diff --git a/.gitignore b/.gitignore index 9af9e94c..c7c8daf7 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,7 @@ build/ *.exe dist/ + +# Locally built binaries (scripts/build-local-tools.sh, mise run example-test) +/bin/ +/.e2e-tools/ diff --git a/AGENTS.md b/AGENTS.md index 3d02da4f..01367bca 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,8 +53,9 @@ EditorConfig rules live in `.editorconfig`. - **`main.go`** — Root `oats` CLI entry point - **`internal/cli/`** — The gcx-driven CLI implementation used by the root binary - **`model/`** — Core data models (`TestCaseDefinition`, expected signals) -- **`yaml/`** — Test case parsing, execution, signal-specific assertions - (`runner.go`, `traces.go`, `metrics.go`, `logs.go`, `profiles.go`) +- **`internal/legacyyaml/`** — legacy (v1/v2) test-case parsing, execution, and + signal-specific assertions (`runner.go`, `traces.go`, `metrics.go`, + `logs.go`, `profiles.go`); used only by `migrate` - **`testhelpers/`** — Docker Compose management, Kubernetes (k3d), HTTP request helpers, response parsing - **`observability/`** — Observability endpoint interface - **`tests/`** — Integration and e2e test fixtures @@ -69,14 +70,16 @@ the current `oats-config.yaml` + case-yaml flow. ```bash # Print a plan -oats --config oats-config.yaml --list +oats list # With flags oats --config oats-config.yaml --timeout 1m ``` +Subcommands: `run` (default), `list`, `migrate`, `cache clear`, `version`. + Key flags: `--config`, `--suite`, `--tags`, `--timeout`, `--interval`, -`--absent-timeout`, `--parallel`, `--gcx`, `--gcx-context`, `--version` +`--absent-timeout`, `--parallel`, `--gcx`, `--gcx-context` ## Code Conventions diff --git a/fixture/k3d.go b/fixture/k3d.go index bda05331..a181bb26 100644 --- a/fixture/k3d.go +++ b/fixture/k3d.go @@ -17,6 +17,7 @@ func startK3D(ctx context.Context, plan discovery.Plan) (Handle, Runtime, error) } ep := newKubernetesEndpoint(plan, ports) if err := ep.Start(ctx); err != nil { + _ = ep.Stop(context.Background()) return nil, Runtime{}, err } appPort := plan.Fixture.K3D.AppPort diff --git a/internal/cli/testdata/fake-gcx.sh b/internal/cli/testdata/fake-gcx.sh index 9d99d84f..cd9302d1 100755 --- a/internal/cli/testdata/fake-gcx.sh +++ b/internal/cli/testdata/fake-gcx.sh @@ -27,7 +27,9 @@ for arg in "$@"; do fi done -query="${*: -1}" +# Last positional arg (portable: bash 3.2 lacks ${*: -1} negative indexing). +query="" +for query in "$@"; do :; done is_missing=false if [[ "$query" == *"missing"* ]]; then is_missing=true diff --git a/yaml/docker-compose-docker-lgtm-template.yml b/internal/legacyyaml/docker-compose-docker-lgtm-template.yml similarity index 100% rename from yaml/docker-compose-docker-lgtm-template.yml rename to internal/legacyyaml/docker-compose-docker-lgtm-template.yml diff --git a/yaml/generator.go b/internal/legacyyaml/generator.go similarity index 99% rename from yaml/generator.go rename to internal/legacyyaml/generator.go index f9c4303b..6d1a118c 100644 --- a/yaml/generator.go +++ b/internal/legacyyaml/generator.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "bytes" diff --git a/yaml/generator_test.go b/internal/legacyyaml/generator_test.go similarity index 98% rename from yaml/generator_test.go rename to internal/legacyyaml/generator_test.go index 4fa58963..9c2218d7 100644 --- a/yaml/generator_test.go +++ b/internal/legacyyaml/generator_test.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "os" diff --git a/yaml/logs.go b/internal/legacyyaml/logs.go similarity index 98% rename from yaml/logs.go rename to internal/legacyyaml/logs.go index dd2137ed..9401c036 100644 --- a/yaml/logs.go +++ b/internal/legacyyaml/logs.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "encoding/json" diff --git a/yaml/logs_test.go b/internal/legacyyaml/logs_test.go similarity index 98% rename from yaml/logs_test.go rename to internal/legacyyaml/logs_test.go index 4012416d..fbf3d23c 100644 --- a/yaml/logs_test.go +++ b/internal/legacyyaml/logs_test.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "os" diff --git a/yaml/metrics.go b/internal/legacyyaml/metrics.go similarity index 98% rename from yaml/metrics.go rename to internal/legacyyaml/metrics.go index 791a7ccb..570c4b6f 100644 --- a/yaml/metrics.go +++ b/internal/legacyyaml/metrics.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "strconv" diff --git a/yaml/profiles.go b/internal/legacyyaml/profiles.go similarity index 98% rename from yaml/profiles.go rename to internal/legacyyaml/profiles.go index ecdd9e3b..05c868c5 100644 --- a/yaml/profiles.go +++ b/internal/legacyyaml/profiles.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "encoding/json" diff --git a/yaml/runner.go b/internal/legacyyaml/runner.go similarity index 99% rename from yaml/runner.go rename to internal/legacyyaml/runner.go index d06246ca..dbaa276f 100644 --- a/yaml/runner.go +++ b/internal/legacyyaml/runner.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "context" diff --git a/yaml/testcase.go b/internal/legacyyaml/testcase.go similarity index 88% rename from yaml/testcase.go rename to internal/legacyyaml/testcase.go index 4eb15812..e7f2066f 100644 --- a/yaml/testcase.go +++ b/internal/legacyyaml/testcase.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "bytes" @@ -22,7 +22,10 @@ func ReadTestCases(input []string, evaluateIgnoreFile bool) ([]model.TestCase, e var cases []model.TestCase for _, base := range input { - base = absolutePath(base) + base, err := absolutePath(base) + if err != nil { + return nil, err + } c, err := collectTestCases(base, evaluateIgnoreFile) if err != nil { @@ -117,12 +120,12 @@ func addTestCase(cases []model.TestCase, base string, path string) ([]model.Test return cases, nil } -func absolutePath(dir string) string { +func absolutePath(dir string) (string, error) { abs, err := filepath.Abs(dir) if err != nil { - panic(err) + return "", fmt.Errorf("failed to resolve path %q: %w", dir, err) } - return abs + return abs, nil } func readTestCase(testBase, filePath string) (*model.TestCase, error) { @@ -134,9 +137,16 @@ func readTestCase(testBase, filePath string) (*model.TestCase, error) { return nil, nil } - absoluteFilePath := absolutePath(filePath) + absoluteFilePath, err := absolutePath(filePath) + if err != nil { + return nil, err + } dir := filepath.Dir(absoluteFilePath) - name := strings.TrimPrefix(dir, absolutePath(testBase)) + "-" + strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath)) + absoluteTestBase, err := absolutePath(testBase) + if err != nil { + return nil, err + } + name := strings.TrimPrefix(dir, absoluteTestBase) + "-" + strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath)) sep := string(filepath.Separator) name = strings.TrimPrefix(name, sep) name = strings.ReplaceAll(name, sep, "-") @@ -151,19 +161,18 @@ func readTestCase(testBase, filePath string) (*model.TestCase, error) { } func readTestCaseDefinition(filePath string, templateMode bool) (*model.TestCaseDefinition, error) { - absPath, err := filepath.Abs(filePath) + filePath, err := absolutePath(filePath) if err != nil { - return nil, fmt.Errorf("failed to resolve path %s: %w", filePath, err) + return nil, err } - filePath = absPath content, err := os.ReadFile(filePath) if err != nil { - return nil, fmt.Errorf("failed to read file %s: %w", filePath, err) + return nil, fmt.Errorf("failed to read file %q: %w", filePath, err) } var parsed map[string]interface{} err = yaml.Unmarshal(content, &parsed) if err != nil { - return nil, fmt.Errorf("failed to parse file %s: %w", filePath, err) + return nil, fmt.Errorf("failed to parse file %q: %w", filePath, err) } schemaVersion, ok := parsed["oats-schema-version"] if !ok { diff --git a/yaml/testcase_test.go b/internal/legacyyaml/testcase_test.go similarity index 88% rename from yaml/testcase_test.go rename to internal/legacyyaml/testcase_test.go index 968297e2..16607bcf 100644 --- a/yaml/testcase_test.go +++ b/internal/legacyyaml/testcase_test.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "path/filepath" @@ -21,7 +21,9 @@ func TestReadTestCase(t *testing.T) { tc, err := readTestCase("testdata", "testdata/valid-tests/oats.yaml") require.NoError(t, err) require.Equal(t, "runvalid-tests-oats", tc.Name) - require.Equal(t, absolutePath("testdata/valid-tests"), tc.Dir) + expectedDir, err := absolutePath("testdata/valid-tests") + require.NoError(t, err) + require.Equal(t, expectedDir, tc.Dir) } func TestIncludePath(t *testing.T) { @@ -73,24 +75,24 @@ func TestInputDefinitionsInvalidFiles(t *testing.T) { { name: "malformed yaml", filePath: "testdata/invalid-tests/malformed-yaml.yaml", - errorMsg: "failed to parse file .*/yaml/testdata/invalid-tests/malformed-yaml.yaml: yaml: mapping values are not allowed in this context", + errorMsg: "failed to parse file \".*/legacyyaml/testdata/invalid-tests/malformed-yaml.yaml\": yaml: mapping values are not allowed in this context", }, { name: "outdated file version", filePath: "testdata/invalid-tests/outdated-version.yaml", - errorMsg: "error parsing test case definition .*/yaml/testdata/invalid-tests/outdated-version.yaml - " + + errorMsg: "error parsing test case definition .*/legacyyaml/testdata/invalid-tests/outdated-version.yaml - " + "see migration notes at https://github.com/grafana/oats/blob/main/UPGRADING.md - unsupported oats-schema-version '1' required version is '2'", }, { name: "file version is not a number", filePath: "testdata/invalid-tests/version-not-int.yaml", - errorMsg: "error parsing test case definition .*/yaml/testdata/invalid-tests/version-not-int.yaml - " + + errorMsg: "error parsing test case definition .*/legacyyaml/testdata/invalid-tests/version-not-int.yaml - " + "see migration notes at https://github.com/grafana/oats/blob/main/UPGRADING.md - oats-schema-version '1' is not a number", }, { name: "unknown field", filePath: "testdata/invalid-tests/unknown-field.yaml", - errorMsg: "error parsing test case definition .*/yaml/testdata/invalid-tests/unknown-field.yaml - " + + errorMsg: "error parsing test case definition .*/legacyyaml/testdata/invalid-tests/unknown-field.yaml - " + "see migration notes at https://github.com/grafana/oats/blob/main/UPGRADING.md - yaml: unmarshal errors:\n" + ".*line 5: field spans not found in type model.ExpectedTraces", }, diff --git a/yaml/testdata/docker-compose-addition.yaml b/internal/legacyyaml/testdata/docker-compose-addition.yaml similarity index 100% rename from yaml/testdata/docker-compose-addition.yaml rename to internal/legacyyaml/testdata/docker-compose-addition.yaml diff --git a/yaml/testdata/docker-compose-expected.yaml b/internal/legacyyaml/testdata/docker-compose-expected.yaml similarity index 100% rename from yaml/testdata/docker-compose-expected.yaml rename to internal/legacyyaml/testdata/docker-compose-expected.yaml diff --git a/yaml/testdata/docker-compose-template.yaml b/internal/legacyyaml/testdata/docker-compose-template.yaml similarity index 100% rename from yaml/testdata/docker-compose-template.yaml rename to internal/legacyyaml/testdata/docker-compose-template.yaml diff --git a/yaml/testdata/invalid-tests/malformed-yaml.yaml b/internal/legacyyaml/testdata/invalid-tests/malformed-yaml.yaml similarity index 100% rename from yaml/testdata/invalid-tests/malformed-yaml.yaml rename to internal/legacyyaml/testdata/invalid-tests/malformed-yaml.yaml diff --git a/yaml/testdata/invalid-tests/outdated-version.yaml b/internal/legacyyaml/testdata/invalid-tests/outdated-version.yaml similarity index 100% rename from yaml/testdata/invalid-tests/outdated-version.yaml rename to internal/legacyyaml/testdata/invalid-tests/outdated-version.yaml diff --git a/yaml/testdata/invalid-tests/unknown-field.yaml b/internal/legacyyaml/testdata/invalid-tests/unknown-field.yaml similarity index 100% rename from yaml/testdata/invalid-tests/unknown-field.yaml rename to internal/legacyyaml/testdata/invalid-tests/unknown-field.yaml diff --git a/yaml/testdata/invalid-tests/version-not-int.yaml b/internal/legacyyaml/testdata/invalid-tests/version-not-int.yaml similarity index 100% rename from yaml/testdata/invalid-tests/version-not-int.yaml rename to internal/legacyyaml/testdata/invalid-tests/version-not-int.yaml diff --git a/yaml/testdata/loki_response.json b/internal/legacyyaml/testdata/loki_response.json similarity index 100% rename from yaml/testdata/loki_response.json rename to internal/legacyyaml/testdata/loki_response.json diff --git a/yaml/testdata/oats-merged.yaml b/internal/legacyyaml/testdata/oats-merged.yaml similarity index 100% rename from yaml/testdata/oats-merged.yaml rename to internal/legacyyaml/testdata/oats-merged.yaml diff --git a/yaml/testdata/valid-tests/expect-absent.oats.yaml b/internal/legacyyaml/testdata/valid-tests/expect-absent.oats.yaml similarity index 100% rename from yaml/testdata/valid-tests/expect-absent.oats.yaml rename to internal/legacyyaml/testdata/valid-tests/expect-absent.oats.yaml diff --git a/yaml/testdata/valid-tests/ignored/.oatsignore b/internal/legacyyaml/testdata/valid-tests/ignored/.oatsignore similarity index 100% rename from yaml/testdata/valid-tests/ignored/.oatsignore rename to internal/legacyyaml/testdata/valid-tests/ignored/.oatsignore diff --git a/yaml/testdata/valid-tests/ignored/should-not-appear.oats.yaml b/internal/legacyyaml/testdata/valid-tests/ignored/should-not-appear.oats.yaml similarity index 100% rename from yaml/testdata/valid-tests/ignored/should-not-appear.oats.yaml rename to internal/legacyyaml/testdata/valid-tests/ignored/should-not-appear.oats.yaml diff --git a/yaml/testdata/valid-tests/input.oats.yaml b/internal/legacyyaml/testdata/valid-tests/input.oats.yaml similarity index 100% rename from yaml/testdata/valid-tests/input.oats.yaml rename to internal/legacyyaml/testdata/valid-tests/input.oats.yaml diff --git a/yaml/testdata/valid-tests/matrix-test.oats.yaml b/internal/legacyyaml/testdata/valid-tests/matrix-test.oats.yaml similarity index 100% rename from yaml/testdata/valid-tests/matrix-test.oats.yaml rename to internal/legacyyaml/testdata/valid-tests/matrix-test.oats.yaml diff --git a/yaml/testdata/valid-tests/more-oats.yml b/internal/legacyyaml/testdata/valid-tests/more-oats.yml similarity index 100% rename from yaml/testdata/valid-tests/more-oats.yml rename to internal/legacyyaml/testdata/valid-tests/more-oats.yml diff --git a/yaml/testdata/valid-tests/oats-template.yaml b/internal/legacyyaml/testdata/valid-tests/oats-template.yaml similarity index 100% rename from yaml/testdata/valid-tests/oats-template.yaml rename to internal/legacyyaml/testdata/valid-tests/oats-template.yaml diff --git a/yaml/testdata/valid-tests/oats.yaml b/internal/legacyyaml/testdata/valid-tests/oats.yaml similarity index 100% rename from yaml/testdata/valid-tests/oats.yaml rename to internal/legacyyaml/testdata/valid-tests/oats.yaml diff --git a/yaml/traces.go b/internal/legacyyaml/traces.go similarity index 98% rename from yaml/traces.go rename to internal/legacyyaml/traces.go index 5475c3ef..2b5f6ec7 100644 --- a/yaml/traces.go +++ b/internal/legacyyaml/traces.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "context" diff --git a/yaml/validate.go b/internal/legacyyaml/validate.go similarity index 98% rename from yaml/validate.go rename to internal/legacyyaml/validate.go index 481ef4e6..153c9533 100644 --- a/yaml/validate.go +++ b/internal/legacyyaml/validate.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "github.com/grafana/oats/model" diff --git a/yaml/validate_test.go b/internal/legacyyaml/validate_test.go similarity index 99% rename from yaml/validate_test.go rename to internal/legacyyaml/validate_test.go index afac090f..3bcb142d 100644 --- a/yaml/validate_test.go +++ b/internal/legacyyaml/validate_test.go @@ -1,4 +1,4 @@ -package yaml +package legacyyaml import ( "testing" diff --git a/migrate/migrate.go b/migrate/migrate.go index 31ee73c3..c311413f 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -11,8 +11,8 @@ import ( "github.com/grafana/oats/casefile" "github.com/grafana/oats/discovery" + "github.com/grafana/oats/internal/legacyyaml" "github.com/grafana/oats/model" - legacyyaml "github.com/grafana/oats/yaml" goyaml "go.yaml.in/yaml/v3" ) diff --git a/migrate/migrate_test.go b/migrate/migrate_test.go index e199a175..f2946895 100644 --- a/migrate/migrate_test.go +++ b/migrate/migrate_test.go @@ -104,7 +104,7 @@ func TestConvertDefinition_SingleMatrixComposeRequiresFile(t *testing.T) { } func TestConvertFile_RendersYAML(t *testing.T) { - sample := filepath.Join("..", "yaml", "testdata", "valid-tests", "oats.yaml") + sample := filepath.Join("..", "internal", "legacyyaml", "testdata", "valid-tests", "oats.yaml") out, warnings, err := ConvertFile(sample) if err != nil { fatalf(t, "ConvertFile: %v", err) @@ -121,7 +121,7 @@ func TestConvertFile_RendersYAML(t *testing.T) { } func TestConvertFile_MatrixSampleIsParseable(t *testing.T) { - sample := filepath.Join("..", "yaml", "testdata", "valid-tests", "matrix-test.oats.yaml") + sample := filepath.Join("..", "internal", "legacyyaml", "testdata", "valid-tests", "matrix-test.oats.yaml") out, warnings, err := ConvertFile(sample) if err != nil { fatalf(t, "ConvertFile matrix sample: %v", err) diff --git a/mise.toml b/mise.toml index 4fb461d9..0f25d103 100644 --- a/mise.toml +++ b/mise.toml @@ -44,9 +44,14 @@ run = "go test -v -buildvcs=false ./tests/e2e -run TestCases" description = "Black-box test: build oats+gcx, then run the python example (needs Docker)" run = ''' root="$PWD" -./scripts/build-local-tools.sh "$root/bin" +# Reuse a prebuilt bin dir when given one (CI shares the e2e-tools artifact), +# otherwise build the binaries locally — same pattern as e2e-test. +bin_dir="${OATS_EXAMPLE_BIN_DIR:-$root/bin}" +if [ ! -x "$bin_dir/oats" ] || [ ! -x "$bin_dir/gcx" ]; then + ./scripts/build-local-tools.sh "$bin_dir" +fi cd examples/python -"$root/bin/oats" --gcx "$root/bin/gcx" --no-cache --timeout 90s --interval 2s -v +"$bin_dir/oats" --gcx "$bin_dir/gcx" --no-cache --timeout 90s --interval 2s -v ''' [tasks.build] diff --git a/report/report_test.go b/report/report_test.go index 517594c8..baa57f52 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -110,6 +110,32 @@ func TestTextReporter_GHAAnnotationsWithoutSource(t *testing.T) { } } +func TestTextReporter_GHAAnnotationEscaping(t *testing.T) { + t.Setenv("GITHUB_ACTIONS", "true") + + var buf bytes.Buffer + r := NewTextReporter(&buf, VerboseDefault) + r.Emit(Event{Type: EventRunStart}) + r.Emit(Event{ + Type: EventAssertFail, + Case: "x", + Source: "path/with,comma:x/a.yaml:8", + Msg: "line one\nline two 50% off", + }) + r.Emit(Event{Type: EventCaseFail, Case: "x"}) + r.Emit(Event{Type: EventRunEnd}) + + out := buf.String() + // Message: %, CR and LF encoded so the annotation is not truncated. + if !strings.Contains(out, "line one%0Aline two 50%25 off") { + t.Errorf("message not escaped:\n%s", out) + } + // Property: message rules plus : and , so delimiters are not misread. + if !strings.Contains(out, "file=path/with%2Ccomma%3Ax/a.yaml,line=8::") { + t.Errorf("file property not escaped:\n%s", out) + } +} + func TestTextReporter_VerbosePassPrintsPasses(t *testing.T) { var buf bytes.Buffer r := NewTextReporter(&buf, VerbosePasses) diff --git a/report/text.go b/report/text.go index 6dd1e612..ed8639c7 100644 --- a/report/text.go +++ b/report/text.go @@ -140,6 +140,7 @@ func (r *TextReporter) emitGHAAnnotation(e Event) { msg = "OATS assertion failed" } msg = ghaEscape(msg) + file = ghaEscapeProp(file) if line > 0 { r.write("::error file=%s,line=%d::%s\n", file, line, msg) } else { @@ -147,6 +148,9 @@ func (r *TextReporter) emitGHAAnnotation(e Event) { } } +// ghaEscape percent-encodes a workflow-command message body. GitHub requires +// %, CR and LF to be encoded or the annotation truncates at the first newline +// (failures often carry multi-line HTTP bodies or stderr excerpts). func ghaEscape(s string) string { s = strings.ReplaceAll(s, "%", "%25") s = strings.ReplaceAll(s, "\r", "%0D") @@ -154,6 +158,17 @@ func ghaEscape(s string) string { return s } +// ghaEscapeProp percent-encodes a workflow-command property value such as the +// file= path. Properties carry the message rules plus : and , which would +// otherwise be read as property delimiters (file paths can contain colons — +// see splitSource). +func ghaEscapeProp(s string) string { + s = ghaEscape(s) + s = strings.ReplaceAll(s, ":", "%3A") + s = strings.ReplaceAll(s, ",", "%2C") + return s +} + func (r *TextReporter) flushRunEnd(e Event) { // Failure blocks first so the summary line is the final thing the reader // sees — useful for both humans (scroll to bottom) and CI logs (tail). diff --git a/seed/seed_test.go b/seed/seed_test.go index 644592f3..9435f699 100644 --- a/seed/seed_test.go +++ b/seed/seed_test.go @@ -9,7 +9,6 @@ import ( "strings" "sync" "testing" - "time" ) type recordingHandler struct { @@ -106,7 +105,6 @@ func TestSender_SpanDefaultsApply(t *testing.T) { srv, h := newRecorder() defer srv.Close() - before := time.Now() s := &Sender{OTLPEndpoint: srv.URL} err := s.Send(context.Background(), Payload{Traces: []Trace{{ Service: "svc", @@ -138,5 +136,4 @@ func TestSender_SpanDefaultsApply(t *testing.T) { if span.StartTimeUnixNano == "" || span.EndTimeUnixNano == "" { t.Errorf("timestamps empty: %+v", span) } - _ = before } diff --git a/testhelpers/compose/compose.go b/testhelpers/compose/compose.go index fbd6c214..a4bdfa20 100644 --- a/testhelpers/compose/compose.go +++ b/testhelpers/compose/compose.go @@ -29,6 +29,27 @@ func defaultEnv() []string { return os.Environ() } +// mergeEnv combines the parent environment with explicitly provided vars, +// ensuring the provided vars deterministically override any parent duplicates +// (e.g. COMPOSE_PROJECT_NAME) regardless of platform-specific exec behavior. +func mergeEnv(parent, override []string) []string { + merged := make([]string, 0, len(parent)+len(override)) + index := make(map[string]int, len(parent)+len(override)) + for _, kv := range append(append([]string{}, parent...), override...) { + key := kv + if i := strings.IndexByte(kv, '='); i >= 0 { + key = kv[:i] + } + if pos, ok := index[key]; ok { + merged[pos] = kv + continue + } + index[key] = len(merged) + merged = append(merged, kv) + } + return merged +} + func Suite(composeFile string) (*Compose, error) { return SuiteFiles([]string{composeFile}, nil) } @@ -43,8 +64,7 @@ func SuiteFiles(composeFiles []string, env []string) (*Compose, error) { if len(composeFiles) == 0 { return nil, fmt.Errorf("at least one compose file is required") } - mergedEnv := defaultEnv() - mergedEnv = append(mergedEnv, env...) + mergedEnv := mergeEnv(defaultEnv(), env) return &Compose{ Command: command, DefaultArgs: defaultArgs, @@ -128,10 +148,18 @@ func (c *Compose) runDocker(cc command) error { go func() { defer wg.Done() reader := bufio.NewReader(stdout) - line, err := reader.ReadString('\n') - for err == nil { - slog.Info(line) - line, err = reader.ReadString('\n') + for { + // ReadString returns any final data together with io.EOF, so + // log the chunk before checking err to avoid dropping a + // trailing line without a newline. Reading to EOF also fully + // drains the pipe so the child never blocks on a full pipe. + line, err := reader.ReadString('\n') + if line != "" { + slog.Info(line) + } + if err != nil { + return + } } }() diff --git a/testhelpers/kubernetes/kubernetes.go b/testhelpers/kubernetes/kubernetes.go index be2554e0..a2c76f1e 100644 --- a/testhelpers/kubernetes/kubernetes.go +++ b/testhelpers/kubernetes/kubernetes.go @@ -2,6 +2,7 @@ package kubernetes import ( "context" + "errors" "fmt" "io" "log/slog" @@ -45,13 +46,16 @@ func NewEndpoint(host string, model *Kubernetes, ports remote.PortsConfig, testN return remote.NewEndpoint(host, ports, func(ctx context.Context) error { return start(model, ports, testName, run) }, func(ctx context.Context) error { + var errs []error for _, p := range killList { - err := p.Kill() - if err != nil { - return err + if err := p.Kill(); err != nil { + errs = append(errs, err) } } - return run(exec.Command("k3d", "cluster", "delete", cluster), false) + if err := run(exec.Command("k3d", "cluster", "delete", cluster), false); err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) }, func(f func(io.ReadCloser, *sync.WaitGroup)) error { return fmt.Errorf("compose log reading is not implemented for kubernetes fixtures") @@ -123,13 +127,17 @@ func start(model *Kubernetes, ports remote.PortsConfig, testName string, run fun if err != nil { return err } - err = portForward(ports.EffectiveGrafanaHTTPPort(), 3000) - if err != nil { - return err + if ports.GrafanaHTTPPort != 0 { + err = portForward(ports.GrafanaHTTPPort, 3000) + if err != nil { + return err + } } - err = portForward(ports.EffectiveOTLPHTTPPort(), 4318) - if err != nil { - return err + if ports.OTLPHTTPPort != 0 { + err = portForward(ports.OTLPHTTPPort, 4318) + if err != nil { + return err + } } err = portForward(ports.PrometheusHTTPPort, 9090) if err != nil { diff --git a/testhelpers/kubernetes/kubernetes_test.go b/testhelpers/kubernetes/kubernetes_test.go index f1e158da..fbfd4455 100644 --- a/testhelpers/kubernetes/kubernetes_test.go +++ b/testhelpers/kubernetes/kubernetes_test.go @@ -125,7 +125,7 @@ func TestStartWaitsForLgtmDeploymentAvailability(t *testing.T) { }) } -func TestStart_FallsBackToLegacyGrafanaAndOTLPPorts(t *testing.T) { +func TestStart_SkipsGrafanaAndOTLPPortsWhenUnset(t *testing.T) { t.Parallel() model := &Kubernetes{ @@ -156,19 +156,9 @@ func TestStart_FallsBackToLegacyGrafanaAndOTLPPorts(t *testing.T) { if err := start(model, ports, "legacy-ports", run); err != nil { t.Fatalf("start: %v", err) } - if !contains(calls, "bg: kubectl port-forward service/lgtm 3000:3000") { - t.Fatalf("expected Grafana legacy fallback port-forward, got %#v", calls) - } - if !contains(calls, "bg: kubectl port-forward service/lgtm 4318:4318") { - t.Fatalf("expected OTLP legacy fallback port-forward, got %#v", calls) - } -} - -func contains(items []string, want string) bool { - for _, item := range items { - if item == want { - return true + for _, call := range calls { + if strings.Contains(call, ":3000") || strings.Contains(call, ":4318") { + t.Fatalf("expected no Grafana/OTLP port-forward when ports are unset, got %#v", calls) } } - return false } diff --git a/testhelpers/remote/remote_observability_endpoint.go b/testhelpers/remote/remote_observability_endpoint.go index 7bd023dc..da1720ae 100644 --- a/testhelpers/remote/remote_observability_endpoint.go +++ b/testhelpers/remote/remote_observability_endpoint.go @@ -41,6 +41,9 @@ func (p PortsConfig) EffectiveOTLPHTTPPort() int { if p.OTLPHTTPPort != 0 { return p.OTLPHTTPPort } + if p.TracesHTTPPort != 0 { + return p.TracesHTTPPort + } return 4318 } diff --git a/tests/e2e/cases/assert/not-contains-fail/test.yaml b/tests/e2e/cases/assert/not-contains-fail/test.yaml index 792a595a..5e2ff085 100644 --- a/tests/e2e/cases/assert/not-contains-fail/test.yaml +++ b/tests/e2e/cases/assert/not-contains-fail/test.yaml @@ -1,5 +1,5 @@ name: not-contains-fail -tags: [assert, not_contains, fail] +tags: [assert, not-contains, fail] run: args: diff --git a/wait/wait.go b/wait/wait.go index 5583c2ca..cdf1ee08 100644 --- a/wait/wait.go +++ b/wait/wait.go @@ -65,6 +65,12 @@ func Until[F any](ctx context.Context, opts Options, asserter Asserter[F]) Resul iter++ last = asserter() if len(last) == 0 { + // A cancelled context must not be reported as success, even when + // the asserter passed — consistent with While. The asserter has + // still run at least once per contract. + if ctx.Err() != nil { + return Result[F]{OK: false, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} + } return Result[F]{OK: true, Iterations: iter, Elapsed: time.Since(start), LastFailures: nil} } if !time.Now().Before(deadline) { diff --git a/wait/wait_test.go b/wait/wait_test.go index 9556bb49..b6e9cfed 100644 --- a/wait/wait_test.go +++ b/wait/wait_test.go @@ -69,6 +69,24 @@ func TestUntil_RunsAtLeastOnceEvenWithTightDeadline(t *testing.T) { } } +func TestUntil_CancelledContextDoesNotPass(t *testing.T) { + // Even when the asserter passes, an already-cancelled context must not be + // reported as success — consistent with While. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + called := false + r := Until[string](ctx, Options{Timeout: time.Second, Interval: 5 * time.Millisecond}, func() []string { + called = true + return nil // passes + }) + if !called { + t.Error("asserter should still run once before honoring cancel") + } + if r.OK { + t.Errorf("cancelled context should not pass even when asserter succeeds: %+v", r) + } +} + func TestWhile_HoldsForEntireWindow(t *testing.T) { start := time.Now() r := While[string](context.Background(), Options{Timeout: 30 * time.Millisecond, Interval: 5 * time.Millisecond}, func() []string {