From 04d2e72c4fb5988f75020d3f55ff2f15b10ef041 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 23 Jul 2026 04:37:04 +0000 Subject: [PATCH 1/7] feat(validation): add repeatable process telemetry --- go.mod | 8 + go.sum | 20 ++ internal/corpus/migration_test.go | 1 + .../migrations/007_validation_run_groups.sql | 24 ++ .../corpus/validation_observation_test.go | 26 +- internal/corpus/workflow.go | 39 +++ internal/evidence/mcp_runner.go | 255 ++++++++++++++ internal/evidence/mcp_runner_test.go | 96 ++++++ internal/evidence/models.go | 172 +++++++++- internal/evidence/repeat.go | 323 ++++++++++++++++++ internal/evidence/repeat_test.go | 132 +++++++ internal/evidence/repository.go | 2 + internal/evidence/runner.go | 58 +++- internal/evidence/runner_test.go | 34 ++ internal/evidence/service.go | 57 +++- internal/evidence/service_test.go | 48 +++ internal/evidence/telemetry.go | 277 +++++++++++++++ 17 files changed, 1553 insertions(+), 19 deletions(-) create mode 100644 internal/corpus/migrations/007_validation_run_groups.sql create mode 100644 internal/evidence/mcp_runner.go create mode 100644 internal/evidence/mcp_runner_test.go create mode 100644 internal/evidence/repeat.go create mode 100644 internal/evidence/repeat_test.go create mode 100644 internal/evidence/telemetry.go diff --git a/go.mod b/go.mod index 51441e5..7ac58f8 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/modelcontextprotocol/go-sdk v1.6.1 github.com/pelletier/go-toml/v2 v2.4.3 github.com/pressly/goose/v3 v3.24.0 + github.com/shirou/gopsutil/v4 v4.26.6 github.com/sourcegraph/go-diff v0.8.0 github.com/zalando/go-keyring v0.2.8 golang.org/x/mod v0.37.0 @@ -45,10 +46,13 @@ require ( github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/danieljoos/wincred v1.2.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.20 // indirect @@ -58,13 +62,17 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.5.4 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/oauth2 v0.35.0 // indirect golang.org/x/sync v0.21.0 // indirect diff --git a/go.sum b/go.sum index 3a72924..83bf473 100644 --- a/go.sum +++ b/go.sum @@ -66,14 +66,19 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -95,6 +100,8 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= @@ -119,6 +126,8 @@ github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdD github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pressly/goose/v3 v3.24.0 h1:sFbNms7Bd++2VMq6HSgDHDLWa7kHz1qXzPb3ZIU72VU= github.com/pressly/goose/v3 v3.24.0/go.mod h1:rEWreU9uVtt0DHCyLzF9gRcWiiTF/V+528DV+4DORug= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -133,14 +142,22 @@ github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= github.com/sourcegraph/go-diff v0.8.0 h1:ipIyu4cTsLbIrln4l0qtHA3r0a7gyK4ntKjtQytHhvY= github.com/sourcegraph/go-diff v0.8.0/go.mod h1:hWlcO7Al+UZStZAP8rBumHpCK5ZHQ5BXsMls8p4+F5E= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -153,6 +170,8 @@ golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= @@ -163,6 +182,7 @@ golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= diff --git a/internal/corpus/migration_test.go b/internal/corpus/migration_test.go index 875d92a..12b5432 100644 --- a/internal/corpus/migration_test.go +++ b/internal/corpus/migration_test.go @@ -24,6 +24,7 @@ func TestBaselineMigrationCreatesCurrentSchema(t *testing.T) { "investigations", "opportunities", "workspaces", "dossiers", "cluster_runs", "clusters", "contribution_manifests", "concerns", "concern_links", "concerns_fts", + "validation_run_groups", } { if !migrationTableExists(ctx, t, c.db, table) { t.Fatalf("table %s missing after baseline migration", table) diff --git a/internal/corpus/migrations/007_validation_run_groups.sql b/internal/corpus/migrations/007_validation_run_groups.sql new file mode 100644 index 0000000..a3878d3 --- /dev/null +++ b/internal/corpus/migrations/007_validation_run_groups.sql @@ -0,0 +1,24 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE validation_run_groups ( + id TEXT PRIMARY KEY, + definition_id TEXT NOT NULL, + investigation_id TEXT NOT NULL, + opportunity_id TEXT NOT NULL DEFAULT '', + classification TEXT NOT NULL CHECK (classification IN ('stable_pass', 'stable_fail', 'flaky', 'inconclusive', 'cancelled')), + requested_runs INTEGER NOT NULL CHECK (requested_runs > 0 AND requested_runs <= 200), + completed_runs INTEGER NOT NULL CHECK (completed_runs >= 0 AND completed_runs <= requested_runs), + payload TEXT NOT NULL, + started_at INTEGER NOT NULL, + completed_at INTEGER NOT NULL, + FOREIGN KEY (definition_id) REFERENCES validation_definitions (id) ON DELETE CASCADE, + FOREIGN KEY (investigation_id) REFERENCES investigations (id) ON DELETE CASCADE +); +CREATE INDEX idx_validation_run_groups_definition_started + ON validation_run_groups (definition_id, started_at DESC, id); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS validation_run_groups; +-- +goose StatementEnd diff --git a/internal/corpus/validation_observation_test.go b/internal/corpus/validation_observation_test.go index 74fb16f..7f2b34a 100644 --- a/internal/corpus/validation_observation_test.go +++ b/internal/corpus/validation_observation_test.go @@ -5,7 +5,9 @@ import ( "testing" "time" + "github.com/morluto/gitcontribute/internal/domain" "github.com/morluto/gitcontribute/internal/evidence" + "github.com/morluto/gitcontribute/internal/investigation" ) func TestValidationObservationPayloadRoundTrip(t *testing.T) { @@ -13,8 +15,12 @@ func TestValidationObservationPayloadRoundTrip(t *testing.T) { ctx := context.Background() c, _ := openTestCorpus(t) now := time.Now().UTC() + inv, err := investigation.NewService(c, c).StartInvestigation(ctx, domain.RepoRef{Owner: "owner", Repo: "repo"}, "abc123", "") + if err != nil { + t.Fatal(err) + } definition := &evidence.ValidationDefinition{ - ID: "definition", Command: []string{"test"}, WorkingDir: "/tmp", CreatedAt: now, + ID: "definition", InvestigationID: inv.ID, Command: []string{"test"}, WorkingDir: "/tmp", CreatedAt: now, Observation: &evidence.ObservationContract{ Intent: "observe artifact", Candidate: []evidence.ExpectedObservation{{ @@ -54,4 +60,22 @@ func TestValidationObservationPayloadRoundTrip(t *testing.T) { if gotRun.ObservationStatus != evidence.ObservationMatched || len(gotRun.Observations) != 1 || gotRun.Observations[0].Excerpt != "fixed" { t.Fatalf("run observation = %#v", gotRun) } + group := &evidence.ValidationRunGroup{ + ID: "group", DefinitionID: definition.ID, InvestigationID: inv.ID, RequestedRuns: 1, CompletedRuns: 1, + Concurrency: 1, PerRunTimeout: time.Second, OverallTimeout: time.Second, + SampleInterval: 100 * time.Millisecond, Classification: evidence.RunGroupStablePass, + Attempts: []evidence.ValidationAttempt{{Index: 1, Kind: evidence.RunKindCandidate, RunID: run.ID, + Classification: evidence.RunClassificationPassing, Phases: evidence.RunPhases{InitializedAt: now}}}, + StartedAt: now, CompletedAt: now, + } + if err := c.SaveValidationRunGroup(ctx, group); err != nil { + t.Fatalf("save validation group: %v", err) + } + gotGroup, err := c.GetValidationRunGroup(ctx, group.ID) + if err != nil { + t.Fatalf("get validation group: %v", err) + } + if len(gotGroup.Attempts) != 1 || !gotGroup.Attempts[0].Phases.InitializedAt.Equal(now) { + t.Fatalf("validation group = %#v", gotGroup) + } } diff --git a/internal/corpus/workflow.go b/internal/corpus/workflow.go index 1bfb7cd..8d2fcfd 100644 --- a/internal/corpus/workflow.go +++ b/internal/corpus/workflow.go @@ -476,6 +476,45 @@ func (c *Corpus) GetValidationRun(ctx context.Context, id string) (*evidence.Val return &item, nil } +// SaveValidationRunGroup persists one bounded repeat/stress aggregate. +func (c *Corpus) SaveValidationRunGroup(ctx context.Context, item *evidence.ValidationRunGroup) error { + if item == nil || item.ID == "" { + return errors.New("validation run group id is required") + } + payload, err := marshalWorkflow(item) + if err != nil { + return err + } + _, err = c.db.ExecContext(ctx, ` + INSERT INTO validation_run_groups ( + id, definition_id, investigation_id, opportunity_id, classification, + requested_runs, completed_runs, payload, started_at, completed_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, item.ID, item.DefinitionID, item.InvestigationID, item.OpportunityID, item.Classification, + item.RequestedRuns, item.CompletedRuns, payload, encodeTime(item.StartedAt), encodeTime(item.CompletedAt)) + if err != nil { + return fmt.Errorf("save validation run group: %w", err) + } + return nil +} + +// GetValidationRunGroup returns one persisted repeat/stress aggregate. +func (c *Corpus) GetValidationRunGroup(ctx context.Context, id string) (*evidence.ValidationRunGroup, error) { + var payload string + err := c.db.QueryRowContext(ctx, `SELECT payload FROM validation_run_groups WHERE id=?`, id).Scan(&payload) + if errors.Is(err, sql.ErrNoRows) { + return nil, evidence.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("get validation run group: %w", err) + } + var item evidence.ValidationRunGroup + if err := unmarshalWorkflow(payload, &item); err != nil { + return nil, err + } + return &item, nil +} + // ListValidationRuns returns validation runs scoped to an opportunity. func (c *Corpus) ListValidationRuns(ctx context.Context, opportunityID string) ([]*evidence.ValidationRun, error) { return listWorkflowPayloads[evidence.ValidationRun]( diff --git a/internal/evidence/mcp_runner.go b/internal/evidence/mcp_runner.go new file mode 100644 index 0000000..167a55f --- /dev/null +++ b/internal/evidence/mcp_runner.go @@ -0,0 +1,255 @@ +package evidence + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +const mcpShutdownGrace = 500 * time.Millisecond + +// MCPStdioRunner uses the official MCP SDK to measure declared protocol +// milestones. It does not parse or infer protocol state from process output. +type MCPStdioRunner struct{} + +// NewMCPStdioRunner returns an SDK-backed stdio validation runner. +func NewMCPStdioRunner() *MCPStdioRunner { return &MCPStdioRunner{} } + +type connectedTransport struct{ connection mcp.Connection } + +func (t connectedTransport) Connect(context.Context) (mcp.Connection, error) { + return t.connection, nil +} + +// Run starts an MCP stdio server, initializes a client session, lists tools, +// and closes the session while recording each protocol boundary. +func (r *MCPStdioRunner) Run(ctx context.Context, req RunRequest) (*RunResult, error) { + if len(req.Args) == 0 { + return nil, ErrMissingCommand + } + if req.Dir == "" { + return nil, ErrMissingWorkspace + } + info, err := os.Stat(req.Dir) + if err != nil || !info.IsDir() { + return nil, fmt.Errorf("%w: %s", ErrInvalidWorkspace, req.Dir) + } + outputLimit := req.MaxOutputBytes + if outputLimit < 0 || outputLimit > maxOutputBytes { + return nil, ErrInvalidOutputLimit + } + if outputLimit == 0 { + outputLimit = defaultMaxOutputBytes + } + readinessTimeout := req.ReadinessTimeout + if readinessTimeout <= 0 { + readinessTimeout = 30 * time.Second + } + + started := time.Now().UTC() + phases := RunPhases{SpawnStartedAt: started} + stderr := newBoundedWriter(int(outputLimit)) + // #nosec G204 -- argv is a stored shell-free validation command whose execution requires explicit authorization. + cmd := exec.CommandContext(ctx, req.Args[0], req.Args[1:]...) + cmd.Dir = req.Dir + cmd.Env = req.Env + cmd.Stderr = stderr + configureCommandCancellation(cmd) + transport := &mcp.CommandTransport{Command: cmd, TerminateDuration: mcpShutdownGrace} + connection, err := transport.Connect(ctx) + if err != nil { + return protocolStartResult(ctx, started, phases, stderr, err), nil + } + phases.ProcessStartedAt = time.Now().UTC() + // #nosec G115 -- gopsutil models OS process IDs as int32 on supported platforms. + sampler := startProcessSampler(ctx, int32(cmd.Process.Pid), req.SampleInterval) + + client := mcp.NewClient(&mcp.Implementation{Name: "gitcontribute-validation", Version: "v1"}, nil) + session, readinessTimedOut, err := connectMCP(ctx, client, connection, cmd, readinessTimeout) + if err != nil { + var closer protocolCloser = connection + if readinessTimedOut || ctx.Err() != nil { + phases.ShutdownStartedAt = time.Now().UTC() + closer = nil + } + return finishProtocolResult(ctx, cmd, sampler, started, phases, stderr, newBoundedWriter(int(outputLimit)), err, "readiness", readinessTimedOut, closer), nil + } + phases.InitializedAt = time.Now().UTC() + phases.FirstResponseAt = phases.InitializedAt + + tools, listErr := session.ListTools(ctx, nil) + if listErr == nil { + phases.ToolsListedAt = time.Now().UTC() + } + stdout := newBoundedWriter(int(outputLimit)) + if encodeErr := json.NewEncoder(stdout).Encode(tools); encodeErr != nil && listErr == nil { + listErr = fmt.Errorf("encode tools/list response: %w", encodeErr) + } + return finishProtocolResult(ctx, cmd, sampler, started, phases, stderr, stdout, listErr, "execution", errors.Is(ctx.Err(), context.DeadlineExceeded), session), nil +} + +func protocolStartResult(ctx context.Context, started time.Time, phases RunPhases, stderr *boundedWriter, runErr error) *RunResult { + completed := time.Now().UTC() + timeoutPhase := "" + classification := RunClassificationError + if ctx.Err() != nil { + classification = RunClassificationCancelled + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + timeoutPhase = "startup" + } + } + return &RunResult{ + ExitCode: -1, Stderr: stderr.String(), Truncated: stderr.Overflow(), StartedAt: started, + CompletedAt: completed, Error: runErr.Error(), Classification: classification, + Phases: phases, TimeoutPhase: timeoutPhase, FailurePhase: "startup", + Cleanup: CleanupResult{Status: "unavailable", Reason: "process did not start", CheckedAt: completed}, + } +} + +func finishProtocolResult( + ctx context.Context, + cmd *exec.Cmd, + sampler *processSampler, + started time.Time, + phases RunPhases, + stderr *boundedWriter, + stdout *boundedWriter, + runErr error, + errorPhase string, + timedOut bool, + closer protocolCloser, +) *RunResult { + phases.ExecutionEndedAt = time.Now().UTC() + failurePhase := "" + timeoutPhase := "" + classification := RunClassificationPassing + if runErr != nil { + classification = RunClassificationError + failurePhase = errorPhase + if timedOut { + timeoutPhase = errorPhase + } + } + if phases.ShutdownStartedAt.IsZero() { + phases.ShutdownStartedAt = time.Now().UTC() + } + var closeErr error + shutdownTimedOut := false + if closer != nil { + shutdownTimedOut, closeErr = closeProtocol(ctx, closer, cmd) + } + if closeErr != nil && runErr == nil { + runErr = fmt.Errorf("close MCP session: %w", closeErr) + classification = RunClassificationError + failurePhase = "shutdown" + } + if shutdownTimedOut { + failurePhase = "shutdown" + timeoutPhase = "shutdown" + classification = RunClassificationCancelled + } + sampled := sampler.finish() + phases.ShutdownCheckedAt = sampled.cleanup.CheckedAt + if ctx.Err() != nil && classification != RunClassificationPassing { + classification = RunClassificationCancelled + } + errText := "" + if runErr != nil { + errText = runErr.Error() + } + return &RunResult{ + ExitCode: exitCode(cmd), Stdout: stdout.String(), Stderr: stderr.String(), + Truncated: stdout.Overflow() || stderr.Overflow(), StartedAt: started, CompletedAt: time.Now().UTC(), + Error: errText, Classification: classification, Process: sampled.identity, Phases: phases, + TimeoutPhase: timeoutPhase, FailurePhase: failurePhase, + Resources: sampled.telemetry, Cleanup: sampled.cleanup, + } +} + +type protocolCloser interface { + Close() error +} + +type mcpConnectResult struct { + session *mcp.ClientSession + err error +} + +func connectMCP(ctx context.Context, client *mcp.Client, connection mcp.Connection, cmd *exec.Cmd, timeout time.Duration) (*mcp.ClientSession, bool, error) { + connected := make(chan mcpConnectResult, 1) + go func() { + session, err := client.Connect(ctx, connectedTransport{connection: connection}, nil) + connected <- mcpConnectResult{session: session, err: err} + }() + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case result := <-connected: + return result.session, false, result.err + case <-timer.C: + return nil, true, abortMCPConnect(connection, cmd, connected, context.DeadlineExceeded) + case <-ctx.Done(): + return nil, errors.Is(ctx.Err(), context.DeadlineExceeded), abortMCPConnect(connection, cmd, connected, ctx.Err()) + } +} + +func abortMCPConnect(connection mcp.Connection, cmd *exec.Cmd, connected <-chan mcpConnectResult, cause error) error { + var cancelErr error + if cmd.Cancel != nil { + cancelErr = cmd.Cancel() + } + closed := make(chan error, 1) + go func() { closed <- connection.Close() }() + timer := time.NewTimer(mcpShutdownGrace) + defer timer.Stop() + var connectErr, closeErr error + for remaining := 2; remaining > 0; { + select { + case result := <-connected: + connectErr = result.err + remaining-- + connected = nil + case closeErr = <-closed: + remaining-- + closed = nil + case <-timer.C: + return errors.Join(cause, cancelErr, connectErr, closeErr, errors.New("MCP readiness shutdown exceeded its grace period")) + } + } + return errors.Join(cause, cancelErr, connectErr, closeErr) +} + +func closeProtocol(ctx context.Context, closer protocolCloser, cmd *exec.Cmd) (bool, error) { + done := make(chan error, 1) + go func() { done <- closer.Close() }() + select { + case err := <-done: + return false, err + case <-ctx.Done(): + cancelErr := os.ErrProcessDone + if cmd.Cancel != nil { + cancelErr = cmd.Cancel() + } + timer := time.NewTimer(mcpShutdownGrace) + defer timer.Stop() + select { + case closeErr := <-done: + return true, errors.Join(ctx.Err(), cancelErr, closeErr) + case <-timer.C: + return true, errors.Join(ctx.Err(), cancelErr, errors.New("MCP shutdown exceeded its grace period")) + } + } +} + +func exitCode(cmd *exec.Cmd) int { + if cmd.ProcessState == nil { + return -1 + } + return cmd.ProcessState.ExitCode() +} diff --git a/internal/evidence/mcp_runner_test.go b/internal/evidence/mcp_runner_test.go new file mode 100644 index 0000000..acf1b27 --- /dev/null +++ b/internal/evidence/mcp_runner_test.go @@ -0,0 +1,96 @@ +package evidence + +import ( + "context" + "os" + "os/exec" + "runtime" + "strings" + "testing" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type blockingProtocolCloser struct{ done <-chan struct{} } + +func (c blockingProtocolCloser) Close() error { + <-c.done + return nil +} + +func TestMCPStdioHelper(_ *testing.T) { + if os.Getenv("GITCONTRIBUTE_MCP_HELPER") != "1" { + return + } + server := mcp.NewServer(&mcp.Implementation{Name: "validation-fixture", Version: "v1"}, nil) + mcp.AddTool(server, &mcp.Tool{Name: "fixture.echo", Description: "fixture tool"}, + func(context.Context, *mcp.CallToolRequest, struct{}) (*mcp.CallToolResult, any, error) { + return &mcp.CallToolResult{}, nil, nil + }) + if err := server.Run(context.Background(), &mcp.StdioTransport{}); err != nil { + os.Exit(2) + } + os.Exit(0) +} + +func TestMCPStdioRunnerRecordsSDKMilestones(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + result, err := NewMCPStdioRunner().Run(ctx, RunRequest{ + Args: []string{os.Args[0], "-test.run=^TestMCPStdioHelper$"}, Dir: t.TempDir(), + Env: []string{"GITCONTRIBUTE_MCP_HELPER=1"}, MaxOutputBytes: 4096, + ReadinessTimeout: time.Second, SampleInterval: 10 * time.Millisecond, + }) + if err != nil { + t.Fatal(err) + } + if result.Classification != RunClassificationPassing || result.FailurePhase != "" || result.TimeoutPhase != "" { + t.Fatalf("result = %#v", result) + } + if result.Phases.InitializedAt.IsZero() || result.Phases.ToolsListedAt.IsZero() || result.Phases.ShutdownStartedAt.IsZero() { + t.Fatalf("phases = %#v", result.Phases) + } + if !strings.Contains(result.Stdout, "fixture.echo") { + t.Fatalf("tools/list output = %q", result.Stdout) + } +} + +func TestMCPStdioRunnerClassifiesReadinessDeadline(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell fixture is POSIX-only") + } + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + result, err := NewMCPStdioRunner().Run(ctx, RunRequest{ + Args: []string{"sh", "-c", "sleep 10"}, Dir: t.TempDir(), + MaxOutputBytes: 1024, ReadinessTimeout: 20 * time.Millisecond, + SampleInterval: 10 * time.Millisecond, + }) + if err != nil { + t.Fatal(err) + } + if result.TimeoutPhase != "readiness" || result.FailurePhase != "readiness" { + t.Fatalf("phase result = %#v", result) + } + if result.Phases.ProcessStartedAt.IsZero() || !result.Phases.InitializedAt.IsZero() { + t.Fatalf("phases = %#v", result.Phases) + } +} + +func TestCloseProtocolBoundsStuckShutdownAndUsesConfiguredCancellation(t *testing.T) { + done := make(chan struct{}) + defer close(done) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + cancelCalled := false + cmd := &exec.Cmd{Cancel: func() error { cancelCalled = true; return nil }} + started := time.Now() + timedOut, err := closeProtocol(ctx, blockingProtocolCloser{done: done}, cmd) + if !timedOut || err == nil || !cancelCalled { + t.Fatalf("close result = timedOut:%v err:%v cancelCalled:%v", timedOut, err, cancelCalled) + } + if elapsed := time.Since(started); elapsed > 2*mcpShutdownGrace { + t.Fatalf("shutdown took %s", elapsed) + } +} diff --git a/internal/evidence/models.go b/internal/evidence/models.go index 83fd7f5..5ebb131 100644 --- a/internal/evidence/models.go +++ b/internal/evidence/models.go @@ -54,10 +54,12 @@ const ( // RunRequest is a shell-free command execution request. type RunRequest struct { - Args []string - Dir string - Env []string - MaxOutputBytes int64 + Args []string + Dir string + Env []string + MaxOutputBytes int64 + SampleInterval time.Duration + ReadinessTimeout time.Duration } // RunResult is the captured output of one command execution. @@ -70,6 +72,64 @@ type RunResult struct { CompletedAt time.Time Error string Classification RunClassification + Process ProcessIdentity + Phases RunPhases + TimeoutPhase string + FailurePhase string + Resources ResourceTelemetry + Cleanup CleanupResult +} + +// ProcessIdentity prevents PID reuse from merging unrelated process samples. +type ProcessIdentity struct { + PID int32 + CreateTimeUnixMilli int64 +} + +// RunPhases records generic process boundaries. Protocol readiness milestones +// require a declared adapter and are never inferred from command output. +type RunPhases struct { + SpawnStartedAt time.Time + ProcessStartedAt time.Time + InitializedAt time.Time + ToolsListedAt time.Time + FirstResponseAt time.Time + ExecutionEndedAt time.Time + ShutdownStartedAt time.Time + ShutdownCheckedAt time.Time +} + +// Int64Metric represents a sampled value. Nil means unavailable, never zero. +type Int64Metric struct { + Value *int64 + UnavailableReason string +} + +// Uint64Metric represents a sampled unsigned value. +type Uint64Metric struct { + Value *uint64 + UnavailableReason string +} + +// ResourceTelemetry contains bounded process-tree high-water marks. +type ResourceTelemetry struct { + Provider string + Platform string + SampleInterval time.Duration + SampleCount int + CPUTimeMillis Int64Metric + PeakRSSBytes Uint64Metric + PeakChildCount Int64Metric + SamplerOverheadNanoseconds int64 +} + +// CleanupResult records whether sampled descendants survived the shutdown +// boundary. Survivors are matched by PID and creation time. +type CleanupResult struct { + Status string + Reason string + Survivors []ProcessIdentity + CheckedAt time.Time } // Runner executes an explicit argv inside a workspace directory without a shell. @@ -165,9 +225,20 @@ type ValidationDefinition struct { Timeout time.Duration MaxOutputBytes int64 Observation *ObservationContract + Protocol ValidationProtocol + ReadinessTimeout time.Duration CreatedAt time.Time } +// ValidationProtocol selects an explicit structured adapter. Empty means the +// generic command runner; protocol milestones are never inferred from stdout. +type ValidationProtocol string + +const ( + // ValidationProtocolMCPStdio measures initialize and tools/list through the official MCP SDK. + ValidationProtocolMCPStdio ValidationProtocol = "mcp_stdio" +) + // ValidationRun records the outcome of one execution of a validation definition. type ValidationRun struct { ID string @@ -190,6 +261,99 @@ type ValidationRun struct { WorkspaceSnapshotAfter string WorkspaceBindingStatus string WorkspaceBindingReason string + Process ProcessIdentity + Phases RunPhases + TimeoutPhase string + FailurePhase string + Resources ResourceTelemetry + Cleanup CleanupResult +} + +// RunGroupClassification summarizes repeated, semantically comparable runs. +type RunGroupClassification string + +const ( + // RunGroupStablePass means every comparable attempt passed. + RunGroupStablePass RunGroupClassification = "stable_pass" + // RunGroupStableFail means every comparable attempt failed. + RunGroupStableFail RunGroupClassification = "stable_fail" + // RunGroupFlaky means comparable attempts disagreed. + RunGroupFlaky RunGroupClassification = "flaky" + // RunGroupInconclusive means attempts could not support a semantic conclusion. + RunGroupInconclusive RunGroupClassification = "inconclusive" + // RunGroupCancelled means the requested sample was not completed. + RunGroupCancelled RunGroupClassification = "cancelled" +) + +// RepeatValidationOptions bounds one repeat/stress request. +type RepeatValidationOptions struct { + Kinds []RunKind + RunCount int + Concurrency int + PerRunTimeout time.Duration + OverallTimeout time.Duration + SampleInterval time.Duration +} + +// ValidationAttempt is a bounded summary of one independently timed run. Full +// bounded output remains in the referenced ValidationRun record. +type ValidationAttempt struct { + Index int + Kind RunKind + RunID string + StartedAt time.Time + CompletedAt time.Time + ExitCode int + Classification RunClassification + ObservationStatus ObservationStatus + TimeoutPhase string + FailurePhase string + Error string + Process ProcessIdentity + Phases RunPhases + Resources ResourceTelemetry + Cleanup CleanupResult +} + +// ValidationAggregate preserves semantic and resource conclusions separately. +type ValidationAggregate struct { + Kind RunKind + Requested int + Completed int + Passing int + Failing int + Inconclusive int + Cancelled int + Classification RunGroupClassification + ResourceClassification string +} + +// ValidationGroupComparison compares stable base and candidate aggregates. +type ValidationGroupComparison struct { + Classification ComparisonClassification + Explanation string +} + +// ValidationRunGroup is one persisted bounded repeat/stress execution. +type ValidationRunGroup struct { + ID string + DefinitionID string + InvestigationID string + HypothesisID string + OpportunityID string + ConfigurationSHA256 string + RequestedRuns int + CompletedRuns int + Concurrency int + PerRunTimeout time.Duration + OverallTimeout time.Duration + SampleInterval time.Duration + Attempts []ValidationAttempt + Aggregates []ValidationAggregate + Classification RunGroupClassification + Comparison *ValidationGroupComparison + StartedAt time.Time + CompletedAt time.Time } // Evidence is a piece of supporting, contradicting, or inconclusive proof. diff --git a/internal/evidence/repeat.go b/internal/evidence/repeat.go new file mode 100644 index 0000000..5cd4d69 --- /dev/null +++ b/internal/evidence/repeat.go @@ -0,0 +1,323 @@ +package evidence + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "sync" + "time" + + "github.com/google/uuid" +) + +const ( + maxRepeatRuns = 100 + maxRepeatConcurrency = 16 + minSampleInterval = 10 * time.Millisecond + maxSampleInterval = 10 * time.Second +) + +type validationTask struct { + index int + kind RunKind +} + +type validationTaskResult struct { + attempt ValidationAttempt + run *ValidationRun +} + +// RunValidationGroup executes independently timed attempts, persists every +// successful run record, and then persists one bounded aggregate. +func (s *Service) RunValidationGroup(ctx context.Context, defID string, opts RepeatValidationOptions) (*ValidationRunGroup, error) { + def, err := s.repo.GetValidationDefinition(ctx, defID) + if err != nil { + return nil, err + } + opts, err = normalizeRepeatOptions(def, opts) + if err != nil { + return nil, err + } + configurationDigest, err := validationConfigurationDigest(def) + if err != nil { + return nil, err + } + group := &ValidationRunGroup{ + ID: uuid.NewString(), DefinitionID: def.ID, InvestigationID: def.InvestigationID, + HypothesisID: def.HypothesisID, OpportunityID: def.OpportunityID, + ConfigurationSHA256: configurationDigest, RequestedRuns: len(opts.Kinds) * opts.RunCount, + Concurrency: opts.Concurrency, PerRunTimeout: opts.PerRunTimeout, OverallTimeout: opts.OverallTimeout, + SampleInterval: opts.SampleInterval, StartedAt: time.Now().UTC(), + } + groupCtx, cancel := context.WithTimeout(ctx, opts.OverallTimeout) + defer cancel() + tasks := repeatTasks(opts.Kinds, opts.RunCount) + results := s.executeValidationTasks(groupCtx, def, opts, tasks) + for result := range results { + group.Attempts = append(group.Attempts, result.attempt) + if result.run != nil { + group.CompletedRuns++ + } + } + sort.Slice(group.Attempts, func(i, j int) bool { + if group.Attempts[i].Kind != group.Attempts[j].Kind { + return group.Attempts[i].Kind == RunKindBase + } + return group.Attempts[i].Index < group.Attempts[j].Index + }) + group.CompletedAt = time.Now().UTC() + group.Aggregates = aggregateAttempts(group.Attempts, opts.Kinds, opts.RunCount, def.Observation != nil) + group.Classification = aggregateGroupClassification(group.Aggregates) + group.Comparison = compareValidationAggregates(group.Aggregates) + saveCtx := ctx + if ctx.Err() != nil { + var saveCancel context.CancelFunc + saveCtx, saveCancel = context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer saveCancel() + } + if err := s.repo.SaveValidationRunGroup(saveCtx, group); err != nil { + return nil, err + } + return group, nil +} + +func normalizeRepeatOptions(def *ValidationDefinition, opts RepeatValidationOptions) (RepeatValidationOptions, error) { + if opts.RunCount < 1 || opts.RunCount > maxRepeatRuns { + return opts, fmt.Errorf("repeat run count must be between 1 and %d", maxRepeatRuns) + } + if len(opts.Kinds) == 0 || len(opts.Kinds) > 2 { + return opts, errors.New("one or two validation kinds are required") + } + seen := map[RunKind]bool{} + for _, kind := range opts.Kinds { + if (kind != RunKindBase && kind != RunKindCandidate) || seen[kind] { + return opts, ErrMissingRunKind + } + seen[kind] = true + } + total := len(opts.Kinds) * opts.RunCount + if opts.Concurrency < 1 || opts.Concurrency > maxRepeatConcurrency || opts.Concurrency > total { + return opts, fmt.Errorf("repeat concurrency must be between 1 and %d and no greater than total attempts", maxRepeatConcurrency) + } + if opts.PerRunTimeout == 0 { + opts.PerRunTimeout = def.Timeout + } + if opts.PerRunTimeout <= 0 || opts.PerRunTimeout > maxValidationTimeout { + return opts, ErrInvalidTimeout + } + if opts.SampleInterval == 0 { + opts.SampleInterval = defaultSampleInterval + } + if opts.SampleInterval < minSampleInterval || opts.SampleInterval > maxSampleInterval { + return opts, fmt.Errorf("sample interval must be between %s and %s", minSampleInterval, maxSampleInterval) + } + if opts.OverallTimeout == 0 { + waves := (total + opts.Concurrency - 1) / opts.Concurrency + opts.OverallTimeout = time.Duration(waves) * opts.PerRunTimeout + } + if opts.OverallTimeout <= 0 || opts.OverallTimeout > maxValidationTimeout { + return opts, ErrInvalidTimeout + } + return opts, nil +} + +func repeatTasks(kinds []RunKind, count int) []validationTask { + tasks := make([]validationTask, 0, len(kinds)*count) + for _, kind := range kinds { + for index := 1; index <= count; index++ { + tasks = append(tasks, validationTask{index: index, kind: kind}) + } + } + return tasks +} + +func (s *Service) executeValidationTasks(ctx context.Context, def *ValidationDefinition, opts RepeatValidationOptions, pending []validationTask) <-chan validationTaskResult { + tasks := make(chan validationTask, len(pending)) + results := make(chan validationTaskResult, len(pending)) + for _, task := range pending { + tasks <- task + } + close(tasks) + var workers sync.WaitGroup + workers.Add(opts.Concurrency) + for range opts.Concurrency { + go func() { + defer workers.Done() + for { + if ctx.Err() != nil { + return + } + select { + case <-ctx.Done(): + return + case task, ok := <-tasks: + if !ok { + return + } + if ctx.Err() != nil { + return + } + results <- s.executeValidationTask(ctx, def, opts, task) + } + } + }() + } + go func() { + workers.Wait() + close(results) + }() + return results +} + +func (s *Service) executeValidationTask(ctx context.Context, def *ValidationDefinition, opts RepeatValidationOptions, task validationTask) validationTaskResult { + startedAt := time.Now().UTC() + run, err := s.runDefinition(ctx, def, task.kind, opts.PerRunTimeout, opts.SampleInterval) + if err != nil { + classification := RunClassificationError + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + classification = RunClassificationCancelled + } + return validationTaskResult{attempt: ValidationAttempt{Index: task.index, Kind: task.kind, StartedAt: startedAt, CompletedAt: time.Now().UTC(), Classification: classification, Error: err.Error()}} + } + return validationTaskResult{run: run, attempt: ValidationAttempt{ + Index: task.index, Kind: task.kind, RunID: run.ID, StartedAt: run.StartedAt, CompletedAt: run.CompletedAt, + ExitCode: run.ExitCode, Classification: run.Classification, ObservationStatus: run.ObservationStatus, + TimeoutPhase: run.TimeoutPhase, FailurePhase: run.FailurePhase, + Error: run.Error, Process: run.Process, Phases: run.Phases, + Resources: run.Resources, Cleanup: run.Cleanup, + }} +} + +func aggregateAttempts(attempts []ValidationAttempt, kinds []RunKind, requested int, hasObservations bool) []ValidationAggregate { + aggregates := make([]ValidationAggregate, 0, len(kinds)) + for _, kind := range kinds { + aggregate := ValidationAggregate{Kind: kind, Requested: requested, ResourceClassification: "available"} + for _, attempt := range attempts { + if attempt.Kind != kind { + continue + } + aggregate.Completed++ + if attempt.Cleanup.Status == "failed" { + aggregate.ResourceClassification = "cleanup_failed" + } else if aggregate.ResourceClassification == "available" && resourcesUnavailable(attempt.Resources) { + aggregate.ResourceClassification = "inconclusive" + } + switch semanticAttempt(attempt, hasObservations) { + case RunGroupStablePass: + aggregate.Passing++ + case RunGroupStableFail: + aggregate.Failing++ + case RunGroupCancelled: + aggregate.Cancelled++ + default: + aggregate.Inconclusive++ + } + } + aggregate.Classification = classifyAggregate(aggregate) + aggregates = append(aggregates, aggregate) + } + return aggregates +} + +func semanticAttempt(attempt ValidationAttempt, hasObservations bool) RunGroupClassification { + if attempt.Classification == RunClassificationCancelled { + return RunGroupCancelled + } + if attempt.Classification == RunClassificationError || attempt.RunID == "" { + return RunGroupInconclusive + } + if hasObservations && attempt.ObservationStatus != ObservationMatched { + return RunGroupInconclusive + } + if attempt.Classification == RunClassificationPassing { + return RunGroupStablePass + } + if attempt.Classification == RunClassificationFailing { + return RunGroupStableFail + } + return RunGroupInconclusive +} + +func classifyAggregate(aggregate ValidationAggregate) RunGroupClassification { + if aggregate.Inconclusive > 0 { + return RunGroupInconclusive + } + if aggregate.Passing > 0 && aggregate.Failing > 0 { + return RunGroupFlaky + } + if aggregate.Cancelled > 0 || aggregate.Completed < aggregate.Requested { + return RunGroupCancelled + } + if aggregate.Passing == aggregate.Requested { + return RunGroupStablePass + } + if aggregate.Failing == aggregate.Requested { + return RunGroupStableFail + } + return RunGroupInconclusive +} + +func aggregateGroupClassification(aggregates []ValidationAggregate) RunGroupClassification { + if len(aggregates) == 1 { + return aggregates[0].Classification + } + for _, aggregate := range aggregates { + if aggregate.Classification == RunGroupCancelled { + return RunGroupCancelled + } + if aggregate.Classification == RunGroupFlaky || aggregate.Classification == RunGroupInconclusive { + return RunGroupInconclusive + } + } + return RunGroupInconclusive +} + +func compareValidationAggregates(aggregates []ValidationAggregate) *ValidationGroupComparison { + if len(aggregates) != 2 { + return nil + } + byKind := map[RunKind]ValidationAggregate{aggregates[0].Kind: aggregates[0], aggregates[1].Kind: aggregates[1]} + base, baseOK := byKind[RunKindBase] + candidate, candidateOK := byKind[RunKindCandidate] + if !baseOK || !candidateOK { + return nil + } + if base.Classification == RunGroupStableFail && candidate.Classification == RunGroupStablePass { + return &ValidationGroupComparison{Classification: ComparisonFixed, Explanation: "base fails consistently and candidate passes consistently"} + } + if base.Classification == RunGroupStableFail && candidate.Classification == RunGroupStableFail { + return &ValidationGroupComparison{Classification: ComparisonNotFixed, Explanation: "base and candidate fail consistently"} + } + if base.Classification == RunGroupStablePass && candidate.Classification == RunGroupStablePass { + return &ValidationGroupComparison{Classification: ComparisonNoDifference, Explanation: "base and candidate pass consistently"} + } + if base.Classification == RunGroupStablePass && candidate.Classification == RunGroupStableFail { + return &ValidationGroupComparison{Classification: ComparisonRegression, Explanation: "base passes consistently and candidate fails consistently"} + } + return &ValidationGroupComparison{Classification: ComparisonInconclusive, Explanation: "flaky, cancelled, or semantically incomparable attempts prevent a fixed/not-fixed conclusion"} +} + +func resourcesUnavailable(resources ResourceTelemetry) bool { + return resources.CPUTimeMillis.Value == nil || resources.PeakRSSBytes.Value == nil || resources.PeakChildCount.Value == nil +} + +func validationConfigurationDigest(def *ValidationDefinition) (string, error) { + payload, err := json.Marshal(struct { + Command []string + Environment []string + Timeout time.Duration + MaxOutputBytes int64 + Observation *ObservationContract + Protocol ValidationProtocol + ReadinessTimeout time.Duration + }{def.Command, def.Env, def.Timeout, def.MaxOutputBytes, def.Observation, def.Protocol, def.ReadinessTimeout}) + if err != nil { + return "", fmt.Errorf("encode validation configuration: %w", err) + } + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]), nil +} diff --git a/internal/evidence/repeat_test.go b/internal/evidence/repeat_test.go new file mode 100644 index 0000000..b36d896 --- /dev/null +++ b/internal/evidence/repeat_test.go @@ -0,0 +1,132 @@ +package evidence + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +type sequenceRunner struct { + mu sync.Mutex + results []*RunResult + errors []error + delay time.Duration + index int +} + +func (r *sequenceRunner) Run(ctx context.Context, _ RunRequest) (*RunResult, error) { + if r.delay > 0 { + timer := time.NewTimer(r.delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return &RunResult{ExitCode: -1, StartedAt: time.Now().UTC(), CompletedAt: time.Now().UTC(), Classification: RunClassificationCancelled, Error: ctx.Err().Error()}, nil + case <-timer.C: + } + } + r.mu.Lock() + defer r.mu.Unlock() + index := r.index + r.index++ + if index < len(r.errors) && r.errors[index] != nil { + return nil, r.errors[index] + } + if index >= len(r.results) { + return nil, errors.New("missing scripted result") + } + return r.results[index], nil +} + +func repeatFixture(t *testing.T, runner Runner, observation *ObservationContract) (*Service, *fakeRepo, string) { + t.Helper() + repo := newFakeRepo() + definition := &ValidationDefinition{ + ID: "def-repeat", InvestigationID: "inv-1", Command: []string{"test"}, WorkingDir: t.TempDir(), + Timeout: time.Second, MaxOutputBytes: 1024, Observation: observation, + } + if err := repo.SaveValidationDefinition(context.Background(), definition); err != nil { + t.Fatal(err) + } + return NewService(repo, runner), repo, definition.ID +} + +func runResult(classification RunClassification) *RunResult { + now := time.Now().UTC() + exitCode := 0 + if classification == RunClassificationFailing { + exitCode = 1 + } + zero64, zeroU64 := int64(0), uint64(0) + return &RunResult{ + ExitCode: exitCode, Classification: classification, StartedAt: now, CompletedAt: now, + Resources: ResourceTelemetry{CPUTimeMillis: Int64Metric{Value: &zero64}, PeakRSSBytes: Uint64Metric{Value: &zeroU64}, PeakChildCount: Int64Metric{Value: &zero64}}, + Cleanup: CleanupResult{Status: "clean"}, + } +} + +func TestRunValidationGroupClassifiesOneOffFailureAsFlaky(t *testing.T) { + runner := &sequenceRunner{results: []*RunResult{runResult(RunClassificationPassing), runResult(RunClassificationFailing), runResult(RunClassificationPassing)}} + svc, repo, id := repeatFixture(t, runner, nil) + group, err := svc.RunValidationGroup(context.Background(), id, RepeatValidationOptions{Kinds: []RunKind{RunKindCandidate}, RunCount: 3, Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if group.Classification != RunGroupFlaky || group.CompletedRuns != 3 || len(group.Attempts) != 3 { + t.Fatalf("group = %+v", group) + } + if repo.groups[group.ID] == nil || len(repo.runs) != 3 { + t.Fatalf("group or attempts were not persisted") + } +} + +func TestRunValidationGroupKeepsResultsAfterMalformedAttempt(t *testing.T) { + runner := &sequenceRunner{ + results: []*RunResult{nil, runResult(RunClassificationPassing), runResult(RunClassificationPassing)}, + } + svc, _, id := repeatFixture(t, runner, nil) + group, err := svc.RunValidationGroup(context.Background(), id, RepeatValidationOptions{Kinds: []RunKind{RunKindCandidate}, RunCount: 3, Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if len(group.Attempts) != 3 || group.CompletedRuns != 2 || group.Classification != RunGroupInconclusive || group.Attempts[0].RunID != "" { + t.Fatalf("group = %+v", group) + } +} + +func TestRunValidationGroupCancellationReturnsPartialResults(t *testing.T) { + runner := &sequenceRunner{delay: 100 * time.Millisecond, results: []*RunResult{runResult(RunClassificationPassing)}} + svc, _, id := repeatFixture(t, runner, nil) + group, err := svc.RunValidationGroup(context.Background(), id, RepeatValidationOptions{ + Kinds: []RunKind{RunKindCandidate}, RunCount: 5, Concurrency: 1, + PerRunTimeout: time.Second, OverallTimeout: 25 * time.Millisecond, + }) + if err != nil { + t.Fatal(err) + } + if len(group.Attempts) >= 5 || group.Classification != RunGroupCancelled { + t.Fatalf("group = %+v", group) + } +} + +func TestRunValidationGroupComparisonRejectsUnrelatedCandidateError(t *testing.T) { + contract := &ObservationContract{ + Intent: "candidate removes expected symptom", + Base: []ExpectedObservation{{Name: "symptom", Source: ObservationStderr, Matcher: ObservationExact, Pattern: "expected symptom", Occurrence: ObservationPresent}}, + Candidate: []ExpectedObservation{{Name: "symptom absent", Source: ObservationStderr, Matcher: ObservationExact, Pattern: "expected symptom", Occurrence: ObservationAbsent}}, + } + base := runResult(RunClassificationFailing) + base.Stderr = "expected symptom" + candidate := runResult(RunClassificationError) + candidate.Stderr, candidate.Error = "unrelated startup failure", "start failed" + runner := &sequenceRunner{results: []*RunResult{base, candidate}} + svc, _, id := repeatFixture(t, runner, contract) + group, err := svc.RunValidationGroup(context.Background(), id, RepeatValidationOptions{Kinds: []RunKind{RunKindBase, RunKindCandidate}, RunCount: 1, Concurrency: 1}) + if err != nil { + t.Fatal(err) + } + if group.Comparison == nil || group.Comparison.Classification != ComparisonInconclusive || group.Aggregates[1].Classification != RunGroupInconclusive { + t.Fatalf("group = %+v", group) + } +} diff --git a/internal/evidence/repository.go b/internal/evidence/repository.go index efbe003..9cd0d76 100644 --- a/internal/evidence/repository.go +++ b/internal/evidence/repository.go @@ -10,6 +10,8 @@ type Repository interface { GetValidationDefinition(ctx context.Context, id string) (*ValidationDefinition, error) SaveValidationRun(ctx context.Context, r *ValidationRun) error GetValidationRun(ctx context.Context, id string) (*ValidationRun, error) + SaveValidationRunGroup(ctx context.Context, group *ValidationRunGroup) error + GetValidationRunGroup(ctx context.Context, id string) (*ValidationRunGroup, error) SaveEvidence(ctx context.Context, e *Evidence) error ListEvidence(ctx context.Context, filter EvidenceFilter) ([]*Evidence, error) } diff --git a/internal/evidence/runner.go b/internal/evidence/runner.go index a06b9f2..cd0806c 100644 --- a/internal/evidence/runner.go +++ b/internal/evidence/runner.go @@ -55,24 +55,51 @@ func (r *ExecRunner) Run(ctx context.Context, req RunRequest) (*RunResult, error configureCommandCancellation(cmd) cmd.WaitDelay = 2 * time.Second - started := time.Now() + started := time.Now().UTC() + phases := RunPhases{SpawnStartedAt: started} if err := cmd.Start(); err != nil { if ctx.Err() != nil { + completed := time.Now().UTC() + timeoutPhase := "" + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + timeoutPhase = "startup" + } return &RunResult{ ExitCode: -1, StartedAt: started, - CompletedAt: time.Now(), + CompletedAt: completed, Error: ctx.Err().Error(), Classification: RunClassificationCancelled, + Phases: phases, + TimeoutPhase: timeoutPhase, + FailurePhase: "startup", + Cleanup: CleanupResult{Status: "unavailable", Reason: "process did not start", CheckedAt: completed}, }, nil } - return nil, fmt.Errorf("runner: start: %w", err) + completed := time.Now().UTC() + return &RunResult{ + ExitCode: -1, StartedAt: started, CompletedAt: completed, + Error: fmt.Sprintf("runner: start: %v", err), Classification: RunClassificationError, + Phases: phases, FailurePhase: "startup", + Cleanup: CleanupResult{Status: "unavailable", Reason: "process did not start", CheckedAt: completed}, + }, nil } + phases.ProcessStartedAt = time.Now().UTC() + // #nosec G115 -- gopsutil models OS process IDs as int32 on supported platforms. + sampler := startProcessSampler(ctx, int32(cmd.Process.Pid), req.SampleInterval) runErr := cmd.Wait() - completed := time.Now() + completed := time.Now().UTC() + phases.ExecutionEndedAt = completed + phases.ShutdownStartedAt = completed + sampled := sampler.finish() + phases.ShutdownCheckedAt = sampled.cleanup.CheckedAt if ctx.Err() != nil { + timeoutPhase := "" + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + timeoutPhase = "execution" + } return &RunResult{ ExitCode: -1, Stdout: stdoutBuf.String(), @@ -82,6 +109,12 @@ func (r *ExecRunner) Run(ctx context.Context, req RunRequest) (*RunResult, error CompletedAt: completed, Error: ctx.Err().Error(), Classification: RunClassificationCancelled, + Process: sampled.identity, + Phases: phases, + TimeoutPhase: timeoutPhase, + FailurePhase: "execution", + Resources: sampled.telemetry, + Cleanup: sampled.cleanup, }, nil } @@ -101,6 +134,17 @@ func (r *ExecRunner) Run(ctx context.Context, req RunRequest) (*RunResult, error } runErrStr = runErr.Error() } + timeoutPhase := "" + if errors.Is(runErr, exec.ErrWaitDelay) { + timeoutPhase = "shutdown" + } + failurePhase := "" + if runErr != nil { + failurePhase = "execution" + } + if timeoutPhase == "shutdown" { + failurePhase = "shutdown" + } return &RunResult{ ExitCode: exitCode, @@ -111,6 +155,12 @@ func (r *ExecRunner) Run(ctx context.Context, req RunRequest) (*RunResult, error CompletedAt: completed, Error: runErrStr, Classification: classification, + Process: sampled.identity, + Phases: phases, + TimeoutPhase: timeoutPhase, + FailurePhase: failurePhase, + Resources: sampled.telemetry, + Cleanup: sampled.cleanup, }, nil } diff --git a/internal/evidence/runner_test.go b/internal/evidence/runner_test.go index ccbfc07..452071f 100644 --- a/internal/evidence/runner_test.go +++ b/internal/evidence/runner_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "os/exec" + "runtime" "strings" "testing" "time" @@ -136,3 +137,36 @@ func TestExecRunnerCancellation(t *testing.T) { t.Fatalf("classification: got %q, want cancelled", res.Classification) } } + +func TestExecRunnerCapturesProcessTreeTelemetryAndCleanup(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX process-group cleanup test") + } + sh := findOrSkip(t, "sh") + r := NewExecRunner() + ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) + defer cancel() + res, err := r.Run(ctx, RunRequest{Args: []string{sh, "-c", "sleep 10 & wait"}, Dir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + if res.Process.PID <= 0 || res.Process.CreateTimeUnixMilli <= 0 { + t.Fatalf("process identity = %+v", res.Process) + } + if res.Resources.Provider != "gopsutil/v4" || res.Resources.SampleCount == 0 || res.Resources.PeakChildCount.Value == nil || *res.Resources.PeakChildCount.Value < 1 { + t.Fatalf("telemetry = %+v", res.Resources) + } + if (res.Cleanup.Status != "clean" && res.Cleanup.Status != "unavailable") || len(res.Cleanup.Survivors) != 0 { + t.Fatalf("cleanup = %+v", res.Cleanup) + } + if res.TimeoutPhase != "execution" { + t.Fatalf("timeout phase = %q", res.TimeoutPhase) + } +} + +func TestUnavailableMetricIsNotEncodedAsZero(t *testing.T) { + metric := metricInt64(0, errors.New("unsupported platform metric")) + if metric.Value != nil || metric.UnavailableReason == "" { + t.Fatalf("metric = %+v", metric) + } +} diff --git a/internal/evidence/service.go b/internal/evidence/service.go index 5b70542..7571fc4 100644 --- a/internal/evidence/service.go +++ b/internal/evidence/service.go @@ -2,6 +2,7 @@ package evidence import ( "context" + "errors" "fmt" "os" "regexp" @@ -20,13 +21,14 @@ var environmentNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) // Service manages validation definitions, runs, evidence, and base-vs-candidate comparisons. type Service struct { - repo Repository - runner Runner + repo Repository + runner Runner + mcpRunner Runner } // NewService returns an EvidenceService backed by repo and runner. func NewService(repo Repository, runner Runner) *Service { - return &Service{repo: repo, runner: runner} + return &Service{repo: repo, runner: runner, mcpRunner: NewMCPStdioRunner()} } // DefineValidation validates and stores a validation definition. @@ -46,6 +48,20 @@ func (s *Service) DefineValidation(ctx context.Context, d *ValidationDefinition) if d.Timeout == 0 { d.Timeout = defaultValidationTimeout } + if d.Protocol != "" && d.Protocol != ValidationProtocolMCPStdio { + return fmt.Errorf("unsupported validation protocol %q", d.Protocol) + } + if d.Protocol == "" && d.ReadinessTimeout != 0 { + return errors.New("readiness timeout requires a declared protocol adapter") + } + if d.Protocol == ValidationProtocolMCPStdio { + if d.ReadinessTimeout == 0 { + d.ReadinessTimeout = 30 * time.Second + } + if d.ReadinessTimeout < 0 || d.ReadinessTimeout > d.Timeout { + return errors.New("readiness timeout must be positive and no greater than validation timeout") + } + } if d.MaxOutputBytes < 0 || d.MaxOutputBytes > maxOutputBytes { return ErrInvalidOutputLimit } @@ -75,6 +91,10 @@ func (s *Service) RunValidation(ctx context.Context, defID string, kind RunKind) if err != nil { return nil, err } + return s.runDefinition(ctx, def, kind, def.Timeout, 0) +} + +func (s *Service) runDefinition(ctx context.Context, def *ValidationDefinition, kind RunKind, timeout, sampleInterval time.Duration) (*ValidationRun, error) { workingDir := def.WorkingDir if kind == RunKindBase && def.BaseWorkingDir != "" { workingDir = def.BaseWorkingDir @@ -86,7 +106,6 @@ func (s *Service) RunValidation(ctx context.Context, defID string, kind RunKind) return nil, ErrMissingWorkspace } - timeout := def.Timeout if timeout <= 0 || timeout > maxValidationTimeout { return nil, ErrInvalidTimeout } @@ -98,19 +117,31 @@ func (s *Service) RunValidation(ctx context.Context, defID string, kind RunKind) maxOutput = def.MaxOutputBytes } - result, err := s.runner.Run(runCtx, RunRequest{ - Args: def.Command, - Dir: workingDir, - Env: resolveEnvironment(def.Env), - MaxOutputBytes: maxOutput, + runner := s.runner + if def.Protocol == ValidationProtocolMCPStdio { + runner = s.mcpRunner + } + if runner == nil { + return nil, errors.New("validation runner is unavailable") + } + result, err := runner.Run(runCtx, RunRequest{ + Args: def.Command, + Dir: workingDir, + Env: resolveEnvironment(def.Env), + MaxOutputBytes: maxOutput, + SampleInterval: sampleInterval, + ReadinessTimeout: def.ReadinessTimeout, }) if err != nil { return nil, err } + if result == nil { + return nil, errors.New("validation runner returned no result") + } run := &ValidationRun{ ID: uuid.NewString(), - DefinitionID: defID, + DefinitionID: def.ID, InvestigationID: def.InvestigationID, HypothesisID: def.HypothesisID, OpportunityID: def.OpportunityID, @@ -123,6 +154,12 @@ func (s *Service) RunValidation(ctx context.Context, defID string, kind RunKind) Truncated: result.Truncated, Error: result.Error, Classification: result.Classification, + Process: result.Process, + Phases: result.Phases, + TimeoutPhase: result.TimeoutPhase, + FailurePhase: result.FailurePhase, + Resources: result.Resources, + Cleanup: result.Cleanup, } run.ObservationStatus, run.Observations = evaluateObservations(ctx, def.Observation, kind, workingDir, result.Stdout, result.Stderr, maxOutput) saveCtx := ctx diff --git a/internal/evidence/service_test.go b/internal/evidence/service_test.go index bc8a2fd..0e2fe76 100644 --- a/internal/evidence/service_test.go +++ b/internal/evidence/service_test.go @@ -3,6 +3,7 @@ package evidence import ( "context" "errors" + "sync" "testing" "time" @@ -11,8 +12,10 @@ import ( ) type fakeRepo struct { + mu sync.Mutex defs map[string]*ValidationDefinition runs map[string]*ValidationRun + groups map[string]*ValidationRunGroup evidence map[string]*Evidence } @@ -20,16 +23,21 @@ func newFakeRepo() *fakeRepo { return &fakeRepo{ defs: make(map[string]*ValidationDefinition), runs: make(map[string]*ValidationRun), + groups: make(map[string]*ValidationRunGroup), evidence: make(map[string]*Evidence), } } func (r *fakeRepo) SaveValidationDefinition(_ context.Context, d *ValidationDefinition) error { + r.mu.Lock() + defer r.mu.Unlock() r.defs[d.ID] = d return nil } func (r *fakeRepo) GetValidationDefinition(_ context.Context, id string) (*ValidationDefinition, error) { + r.mu.Lock() + defer r.mu.Unlock() d, ok := r.defs[id] if !ok { return nil, ErrNotFound @@ -38,11 +46,15 @@ func (r *fakeRepo) GetValidationDefinition(_ context.Context, id string) (*Valid } func (r *fakeRepo) SaveValidationRun(_ context.Context, run *ValidationRun) error { + r.mu.Lock() + defer r.mu.Unlock() r.runs[run.ID] = run return nil } func (r *fakeRepo) GetValidationRun(_ context.Context, id string) (*ValidationRun, error) { + r.mu.Lock() + defer r.mu.Unlock() run, ok := r.runs[id] if !ok { return nil, ErrNotFound @@ -50,12 +62,33 @@ func (r *fakeRepo) GetValidationRun(_ context.Context, id string) (*ValidationRu return run, nil } +func (r *fakeRepo) SaveValidationRunGroup(_ context.Context, group *ValidationRunGroup) error { + r.mu.Lock() + defer r.mu.Unlock() + r.groups[group.ID] = group + return nil +} + +func (r *fakeRepo) GetValidationRunGroup(_ context.Context, id string) (*ValidationRunGroup, error) { + r.mu.Lock() + defer r.mu.Unlock() + group, ok := r.groups[id] + if !ok { + return nil, ErrNotFound + } + return group, nil +} + func (r *fakeRepo) SaveEvidence(_ context.Context, e *Evidence) error { + r.mu.Lock() + defer r.mu.Unlock() r.evidence[e.ID] = e return nil } func (r *fakeRepo) ListEvidence(_ context.Context, filter EvidenceFilter) ([]*Evidence, error) { + r.mu.Lock() + defer r.mu.Unlock() var out []*Evidence for _, e := range r.evidence { if filter.OpportunityID != "" && e.OpportunityID != filter.OpportunityID { @@ -366,6 +399,21 @@ func TestRunValidationPassesOutputBound(t *testing.T) { } } +func TestDefineValidationRequiresDeclaredProtocolForReadinessDeadline(t *testing.T) { + svc := NewService(newFakeRepo(), &fakeRunner{}) + withoutProtocol := &ValidationDefinition{Command: []string{"server"}, WorkingDir: "/tmp", ReadinessTimeout: time.Second} + if err := svc.DefineValidation(context.Background(), withoutProtocol); err == nil { + t.Fatal("readiness deadline without protocol was accepted") + } + withProtocol := &ValidationDefinition{Command: []string{"server"}, WorkingDir: "/tmp", Protocol: ValidationProtocolMCPStdio} + if err := svc.DefineValidation(context.Background(), withProtocol); err != nil { + t.Fatal(err) + } + if withProtocol.ReadinessTimeout != 30*time.Second { + t.Fatalf("readiness timeout = %s", withProtocol.ReadinessTimeout) + } +} + func TestEvidenceWithSourceRef(t *testing.T) { repo := newFakeRepo() svc := NewService(repo, &fakeRunner{}) diff --git a/internal/evidence/telemetry.go b/internal/evidence/telemetry.go new file mode 100644 index 0000000..0623946 --- /dev/null +++ b/internal/evidence/telemetry.go @@ -0,0 +1,277 @@ +package evidence + +import ( + "context" + "errors" + "fmt" + "runtime" + "sort" + "time" + + "github.com/shirou/gopsutil/v4/process" +) + +const ( + defaultSampleInterval = 100 * time.Millisecond + maxSampledProcesses = 1024 + cleanupGracePeriod = 500 * time.Millisecond +) + +type samplerResult struct { + identity ProcessIdentity + telemetry ResourceTelemetry + cleanup CleanupResult +} + +type processSampler struct { + stop chan chan samplerResult +} + +func startProcessSampler(ctx context.Context, pid int32, interval time.Duration) *processSampler { + if interval <= 0 { + interval = defaultSampleInterval + } + s := &processSampler{stop: make(chan chan samplerResult)} + go s.run(ctx, pid, interval) + return s +} + +func (s *processSampler) finish() samplerResult { + result := make(chan samplerResult, 1) + s.stop <- result + return <-result +} + +func (s *processSampler) run(ctx context.Context, pid int32, interval time.Duration) { + state := newSamplerState(pid, interval) + state.sample(ctx) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + state.sample(ctx) + case result := <-s.stop: + result <- state.result(ctx) + return + } + } +} + +type samplerState struct { + rootPID int32 + rootCreateAt int64 + interval time.Duration + samples int + overhead time.Duration + peakCPUms int64 + peakRSS uint64 + peakChildren int64 + cpuAvailable bool + memoryAvailable bool + childrenAvailable bool + cpuErr error + memoryErr error + childrenErr error + trackingErr error + seen map[ProcessIdentity]struct{} +} + +func newSamplerState(pid int32, interval time.Duration) *samplerState { + return &samplerState{rootPID: pid, interval: interval, seen: make(map[ProcessIdentity]struct{})} +} + +func (s *samplerState) sample(parent context.Context) { + started := time.Now() + ctx, cancel := context.WithTimeout(context.WithoutCancel(parent), min(s.interval, time.Second)) + defer cancel() + root, err := process.NewProcessWithContext(ctx, s.rootPID) + if err != nil { + s.markUnavailable(err) + s.overhead += time.Since(started) + return + } + createdAt, err := root.CreateTimeWithContext(ctx) + if err != nil { + s.markUnavailable(err) + s.overhead += time.Since(started) + return + } + if s.rootCreateAt == 0 { + s.rootCreateAt = createdAt + } else if createdAt != s.rootCreateAt { + s.markUnavailable(errors.New("root PID was reused")) + s.overhead += time.Since(started) + return + } + processes, err := processTree(ctx, root) + if err != nil { + s.childrenErr = err + s.trackingErr = err + } + var cpuMillis int64 + var rss uint64 + cpuComplete := true + memoryComplete := true + for _, current := range processes { + identity, identityErr := processIdentity(ctx, current) + if identityErr != nil { + s.trackingErr = identityErr + } else { + s.seen[identity] = struct{}{} + } + times, timesErr := current.TimesWithContext(ctx) + if timesErr != nil { + s.cpuErr = timesErr + cpuComplete = false + } else { + cpuMillis += int64((times.User + times.System) * 1000) + } + memory, memoryErr := current.MemoryInfoWithContext(ctx) + if memoryErr != nil { + s.memoryErr = memoryErr + memoryComplete = false + } else { + rss += memory.RSS + } + } + s.samples++ + if cpuComplete { + s.cpuAvailable = true + s.peakCPUms = max(s.peakCPUms, cpuMillis) + } + if memoryComplete { + s.memoryAvailable = true + s.peakRSS = max(s.peakRSS, rss) + } + if err == nil { + s.childrenAvailable = true + s.peakChildren = max(s.peakChildren, int64(max(0, len(processes)-1))) + } + s.overhead += time.Since(started) +} + +func (s *samplerState) markUnavailable(err error) { + if s.cpuErr == nil { + s.cpuErr = err + } + if s.memoryErr == nil { + s.memoryErr = err + } + if s.childrenErr == nil { + s.childrenErr = err + } +} + +func processTree(ctx context.Context, root *process.Process) ([]*process.Process, error) { + queue := []*process.Process{root} + all := make([]*process.Process, 0, 8) + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + all = append(all, current) + if len(all) == maxSampledProcesses { + return all, fmt.Errorf("process tree exceeds %d entries", maxSampledProcesses) + } + children, err := current.ChildrenWithContext(ctx) + if err != nil { + return all, err + } + queue = append(queue, children...) + } + return all, nil +} + +func processIdentity(ctx context.Context, item *process.Process) (ProcessIdentity, error) { + createdAt, err := item.CreateTimeWithContext(ctx) + if err != nil { + return ProcessIdentity{}, err + } + return ProcessIdentity{PID: item.Pid, CreateTimeUnixMilli: createdAt}, nil +} + +func (s *samplerState) result(parent context.Context) samplerResult { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(parent), 2*time.Second) + defer cancel() + survivors := s.survivors(cleanupCtx) + deadline := time.Now().Add(cleanupGracePeriod) + for len(survivors) > 0 && time.Now().Before(deadline) { + timer := time.NewTimer(25 * time.Millisecond) + select { + case <-cleanupCtx.Done(): + timer.Stop() + case <-timer.C: + } + survivors = s.survivors(cleanupCtx) + } + sort.Slice(survivors, func(i, j int) bool { + if survivors[i].PID != survivors[j].PID { + return survivors[i].PID < survivors[j].PID + } + return survivors[i].CreateTimeUnixMilli < survivors[j].CreateTimeUnixMilli + }) + checkedAt := time.Now().UTC() + cleanup := CleanupResult{Status: "clean", CheckedAt: checkedAt} + if len(survivors) > 0 { + cleanup.Status, cleanup.Reason, cleanup.Survivors = "failed", "sampled descendants survived shutdown", survivors + } + telemetry := ResourceTelemetry{ + Provider: "gopsutil/v4", Platform: runtime.GOOS + "/" + runtime.GOARCH, + SampleInterval: s.interval, SampleCount: s.samples, SamplerOverheadNanoseconds: s.overhead.Nanoseconds(), + CPUTimeMillis: metricInt64(s.peakCPUms, unavailableMetricError(s.cpuAvailable, s.cpuErr)), + PeakRSSBytes: metricUint64(s.peakRSS, unavailableMetricError(s.memoryAvailable, s.memoryErr)), + PeakChildCount: metricInt64(s.peakChildren, unavailableMetricError(s.childrenAvailable, s.childrenErr)), + } + if s.rootCreateAt == 0 && cleanup.Status == "clean" { + cleanup.Status, cleanup.Reason = "unavailable", "root process identity was unavailable" + } else if s.trackingErr != nil && cleanup.Status == "clean" { + cleanup.Status, cleanup.Reason = "unavailable", "process tree tracking incomplete: "+s.trackingErr.Error() + } + return samplerResult{identity: ProcessIdentity{PID: s.rootPID, CreateTimeUnixMilli: s.rootCreateAt}, telemetry: telemetry, cleanup: cleanup} +} + +func unavailableMetricError(available bool, lastErr error) error { + if available { + return nil + } + if lastErr != nil { + return lastErr + } + return errors.New("no complete process sample was available") +} + +func (s *samplerState) survivors(ctx context.Context) []ProcessIdentity { + survivors := make([]ProcessIdentity, 0) + for identity := range s.seen { + if identity.PID == s.rootPID && identity.CreateTimeUnixMilli == s.rootCreateAt { + continue + } + item, err := process.NewProcessWithContext(ctx, identity.PID) + if err != nil { + continue + } + createdAt, err := item.CreateTimeWithContext(ctx) + if err != nil || createdAt != identity.CreateTimeUnixMilli { + continue + } + running, err := item.IsRunningWithContext(ctx) + if err == nil && running { + survivors = append(survivors, identity) + } + } + return survivors +} + +func metricInt64(value int64, err error) Int64Metric { + if err != nil { + return Int64Metric{UnavailableReason: err.Error()} + } + return Int64Metric{Value: &value} +} + +func metricUint64(value uint64, err error) Uint64Metric { + if err != nil { + return Uint64Metric{UnavailableReason: err.Error()} + } + return Uint64Metric{Value: &value} +} From 1b71018c10b4ed6bc90e6d1882f3a127c2019b2d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 23 Jul 2026 04:37:11 +0000 Subject: [PATCH 2/7] feat(validation): expose repeatable validation workflows --- internal/app/evidence.go | 101 ++++++++++++++++++++++ internal/app/mcp_v1.go | 72 ++++++++++++++++ internal/cli/cli.go | 15 ++++ internal/cli/extended_test.go | 40 +++++++-- internal/cli/interfaces.go | 130 +++++++++++++++++++++++++++++ internal/cli/output.go | 2 + internal/cli/validation_cli.go | 30 ++++++- internal/cli/validation_output.go | 14 ++++ internal/mcpserver/catalog.go | 4 +- internal/mcpserver/catalog_test.go | 11 +++ internal/mcpserver/server.go | 1 + internal/mcpserver/server_test.go | 24 ++++++ internal/mcpserver/v1.go | 69 ++++++++++++++- 13 files changed, 502 insertions(+), 11 deletions(-) diff --git a/internal/app/evidence.go b/internal/app/evidence.go index 5192a80..d420ccb 100644 --- a/internal/app/evidence.go +++ b/internal/app/evidence.go @@ -60,6 +60,8 @@ func (s *Service) DefineValidation(ctx context.Context, investigationID string, Timeout: opts.Timeout, MaxOutputBytes: opts.MaxOutputBytes, Observation: observationContractToEvidence(opts.Observation), + Protocol: evidence.ValidationProtocol(opts.Protocol), + ReadinessTimeout: opts.ReadinessTimeout, } evSvc := evidence.NewService(c, evidence.NewExecRunner()) @@ -223,6 +225,29 @@ func bindValidationWorkspace(ctx context.Context, service *Service, c *corpus.Co return nil } +// RunValidationGroup executes a bounded repeat/stress validation group. +func (s *Service) RunValidationGroup(ctx context.Context, id string, opts cli.RepeatValidationOptions) (*cli.ValidationRunGroupResult, error) { + if !opts.Execute { + return nil, evidence.ErrExecutionNotAuthorized + } + kinds := make([]evidence.RunKind, len(opts.Kinds)) + for index, kind := range opts.Kinds { + kinds[index] = evidence.RunKind(kind) + } + c, err := s.openCorpus(ctx) + if err != nil { + return nil, err + } + group, err := evidence.NewService(c, evidence.NewExecRunner()).RunValidationGroup(ctx, id, evidence.RepeatValidationOptions{ + Kinds: kinds, RunCount: opts.RunCount, Concurrency: opts.Concurrency, + PerRunTimeout: opts.PerRunTimeout, OverallTimeout: opts.OverallTimeout, SampleInterval: opts.SampleInterval, + }) + if err != nil { + return nil, mapEvidenceError(err) + } + return validationRunGroupResult(group), nil +} + // CompareValidation compares a base validation run with a candidate validation run. func (s *Service) CompareValidation(ctx context.Context, baseRunID, candidateRunID string) (*cli.ValidationComparisonResult, error) { c, err := s.openReadOnlyCorpus(ctx) @@ -363,6 +388,10 @@ func validationResult(def *evidence.ValidationDefinition) *cli.ValidationResult if def.Timeout > 0 { timeout = def.Timeout.String() } + readinessTimeout := "" + if def.ReadinessTimeout > 0 { + readinessTimeout = def.ReadinessTimeout.String() + } return &cli.ValidationResult{ ID: def.ID, InvestigationID: def.InvestigationID, @@ -378,6 +407,8 @@ func validationResult(def *evidence.ValidationDefinition) *cli.ValidationResult Timeout: timeout, MaxOutputBytes: def.MaxOutputBytes, Observation: observationContractToCLI(def.Observation), + Protocol: string(def.Protocol), + ReadinessTimeout: readinessTimeout, CreatedAt: formatTime(def.CreatedAt), } } @@ -402,7 +433,77 @@ func validationRunResult(run *evidence.ValidationRun) *cli.ValidationRunResult { WorkspaceSnapshotAfter: run.WorkspaceSnapshotAfter, WorkspaceBindingStatus: run.WorkspaceBindingStatus, WorkspaceBindingReason: run.WorkspaceBindingReason, + Process: validationProcessIdentity(run.Process), + Phases: validationPhases(run.Phases), + TimeoutPhase: run.TimeoutPhase, + FailurePhase: run.FailurePhase, + Resources: validationResources(run.Resources), + Cleanup: validationCleanup(run.Cleanup), + } +} + +func validationRunGroupResult(group *evidence.ValidationRunGroup) *cli.ValidationRunGroupResult { + result := &cli.ValidationRunGroupResult{ + ID: group.ID, DefinitionID: group.DefinitionID, InvestigationID: group.InvestigationID, + ConfigurationSHA256: group.ConfigurationSHA256, RequestedRuns: group.RequestedRuns, CompletedRuns: group.CompletedRuns, + Concurrency: group.Concurrency, PerRunTimeout: group.PerRunTimeout.String(), OverallTimeout: group.OverallTimeout.String(), + SampleInterval: group.SampleInterval.String(), Classification: string(group.Classification), + StartedAt: formatTime(group.StartedAt), CompletedAt: formatTime(group.CompletedAt), + } + for _, attempt := range group.Attempts { + result.Attempts = append(result.Attempts, cli.ValidationAttemptResult{ + Index: attempt.Index, Kind: string(attempt.Kind), RunID: attempt.RunID, + StartedAt: formatTime(attempt.StartedAt), CompletedAt: formatTime(attempt.CompletedAt), ExitCode: attempt.ExitCode, + Classification: string(attempt.Classification), ObservationStatus: string(attempt.ObservationStatus), + TimeoutPhase: attempt.TimeoutPhase, FailurePhase: attempt.FailurePhase, + Error: attempt.Error, Process: validationProcessIdentity(attempt.Process), + Phases: validationPhases(attempt.Phases), + Resources: validationResources(attempt.Resources), Cleanup: validationCleanup(attempt.Cleanup), + }) + } + for _, aggregate := range group.Aggregates { + result.Aggregates = append(result.Aggregates, cli.ValidationAggregateResult{ + Kind: string(aggregate.Kind), Requested: aggregate.Requested, Completed: aggregate.Completed, + Passing: aggregate.Passing, Failing: aggregate.Failing, Inconclusive: aggregate.Inconclusive, + Cancelled: aggregate.Cancelled, Classification: string(aggregate.Classification), + ResourceClassification: aggregate.ResourceClassification, + }) + } + if group.Comparison != nil { + result.Comparison = &cli.ValidationGroupComparisonResult{Classification: string(group.Comparison.Classification), Explanation: group.Comparison.Explanation} + } + return result +} + +func validationPhases(value evidence.RunPhases) cli.ValidationRunPhases { + return cli.ValidationRunPhases{ + SpawnStartedAt: formatTime(value.SpawnStartedAt), ProcessStartedAt: formatTime(value.ProcessStartedAt), + InitializedAt: formatTime(value.InitializedAt), ToolsListedAt: formatTime(value.ToolsListedAt), + FirstResponseAt: formatTime(value.FirstResponseAt), ExecutionEndedAt: formatTime(value.ExecutionEndedAt), + ShutdownStartedAt: formatTime(value.ShutdownStartedAt), ShutdownCheckedAt: formatTime(value.ShutdownCheckedAt), + } +} + +func validationProcessIdentity(value evidence.ProcessIdentity) cli.ValidationProcessIdentity { + return cli.ValidationProcessIdentity{PID: value.PID, CreateTimeUnixMilli: value.CreateTimeUnixMilli} +} + +func validationResources(value evidence.ResourceTelemetry) cli.ValidationResourceTelemetry { + return cli.ValidationResourceTelemetry{ + Provider: value.Provider, Platform: value.Platform, SampleInterval: value.SampleInterval.String(), SampleCount: value.SampleCount, + CPUTimeMillis: cli.ValidationInt64Metric{Value: value.CPUTimeMillis.Value, UnavailableReason: value.CPUTimeMillis.UnavailableReason}, + PeakRSSBytes: cli.ValidationUint64Metric{Value: value.PeakRSSBytes.Value, UnavailableReason: value.PeakRSSBytes.UnavailableReason}, + PeakChildCount: cli.ValidationInt64Metric{Value: value.PeakChildCount.Value, UnavailableReason: value.PeakChildCount.UnavailableReason}, + SamplerOverheadNanoseconds: value.SamplerOverheadNanoseconds, + } +} + +func validationCleanup(value evidence.CleanupResult) cli.ValidationCleanupResult { + result := cli.ValidationCleanupResult{Status: value.Status, Reason: value.Reason, CheckedAt: formatTime(value.CheckedAt)} + for _, survivor := range value.Survivors { + result.Survivors = append(result.Survivors, validationProcessIdentity(survivor)) } + return result } func observationContractToEvidence(contract *cli.ValidationObservationContract) *evidence.ObservationContract { diff --git a/internal/app/mcp_v1.go b/internal/app/mcp_v1.go index 3c9a0d9..d438c19 100644 --- a/internal/app/mcp_v1.go +++ b/internal/app/mcp_v1.go @@ -351,6 +351,66 @@ func (r *MCPReader) RunValidation(ctx context.Context, in mcpserver.RunValidatio return queuedJobReference(id, "run_validation", "validation run started"), nil } +// RunRepeatedValidation submits a durable repeat/stress validation group. +func (r *MCPReader) RunRepeatedValidation(ctx context.Context, in mcpserver.RunRepeatedValidationInput) (mcpserver.JobReference, error) { + if !in.Execute { + return mcpserver.JobReference{}, errors.New("execute must be true to authorize host command execution") + } + if in.RunCount == 0 { + in.RunCount = 3 + } + if in.Concurrency == 0 { + in.Concurrency = 1 + } + if in.SampleInterval == "" { + in.SampleInterval = "100ms" + } + kinds := []string{in.Target} + if in.Target == "both" { + kinds = []string{"base", "candidate"} + } + perRunTimeout, err := parseOptionalDuration(in.PerRunTimeout) + if err != nil { + return mcpserver.JobReference{}, fmt.Errorf("per_run_timeout: %w", err) + } + overallTimeout, err := parseOptionalDuration(in.OverallTimeout) + if err != nil { + return mcpserver.JobReference{}, fmt.Errorf("overall_timeout: %w", err) + } + sampleInterval, err := parseOptionalDuration(in.SampleInterval) + if err != nil { + return mcpserver.JobReference{}, fmt.Errorf("sample_interval: %w", err) + } + opts := cli.RepeatValidationOptions{ + Kinds: kinds, RunCount: in.RunCount, Concurrency: in.Concurrency, + PerRunTimeout: perRunTimeout, OverallTimeout: overallTimeout, SampleInterval: sampleInterval, Execute: true, + } + id, err := r.submitJob(ctx, "run_validation_group", in, func(ctx context.Context, report func(progress, statistics string) error) (any, error) { + if err := report("validation", jobProgressCounts(0, in.RunCount*len(kinds))); err != nil { + return nil, err + } + result, err := r.RunValidationGroup(ctx, in.ID, opts) + if err != nil { + return nil, err + } + if err := report("validation", jobProgressCounts(result.CompletedRuns, result.RequestedRuns)); err != nil { + return nil, err + } + return result, nil + }) + if err != nil { + return mcpserver.JobReference{}, err + } + return queuedJobReference(id, "run_validation_group", "repeat validation job started"), nil +} + +func parseOptionalDuration(value string) (time.Duration, error) { + if strings.TrimSpace(value) == "" { + return 0, nil + } + return time.ParseDuration(value) +} + // StartInvestigation creates a new investigation workspace. func (r *MCPReader) StartInvestigation(ctx context.Context, in mcpserver.StartInvestigationInput) (mcpserver.InvestigationOutput, error) { if in.Number > 0 { @@ -598,6 +658,14 @@ func (r *MCPReader) DefineValidation(ctx context.Context, in mcpserver.DefineVal } timeout = d } + var readinessTimeout time.Duration + if in.ReadinessTimeout != "" { + d, err := time.ParseDuration(in.ReadinessTimeout) + if err != nil { + return mcpserver.ValidationOutput{}, fmt.Errorf("invalid readiness timeout: %w", err) + } + readinessTimeout = d + } opts := cli.DefineValidationOptions{ Kind: in.Kind, Command: in.Command, @@ -608,6 +676,8 @@ func (r *MCPReader) DefineValidation(ctx context.Context, in mcpserver.DefineVal Timeout: timeout, MaxOutputBytes: in.MaxOutputBytes, Observation: observationContractMCPToCLI(in.Observation), + Protocol: in.Protocol, + ReadinessTimeout: readinessTimeout, } res, err := r.Service.DefineValidation(ctx, in.InvestigationID, opts) if err != nil { @@ -632,6 +702,8 @@ func validationResultToMCP(res *cli.ValidationResult) mcpserver.ValidationOutput Timeout: res.Timeout, MaxOutputBytes: res.MaxOutputBytes, Observation: observationContractCLIToMCP(res.Observation), + Protocol: res.Protocol, + ReadinessTimeout: res.ReadinessTimeout, CreatedAt: res.CreatedAt, } } diff --git a/internal/cli/cli.go b/internal/cli/cli.go index efcce6e..d4abc6a 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -392,6 +392,7 @@ type setStatusOpportunityCmd struct { type validationCmd struct { Define defineValidationCmd `cmd:"" help:"Define a validation"` Run runValidationCmd `cmd:"" help:"Run a validation definition"` + Repeat repeatValidationCmd `cmd:"" help:"Run a bounded repeat/stress validation"` Compare compareValidationCmd `cmd:"" help:"Compare two validation runs"` } @@ -409,6 +410,8 @@ type defineValidationCmd struct { Timeout time.Duration `name:"timeout" help:"Maximum execution time"` MaxOutput int64 `name:"max-output" help:"Maximum captured output bytes per stream"` Observation string `name:"observation-contract" help:"JSON observation contract for base and candidate output"` + Protocol string `name:"protocol" help:"Structured protocol adapter (mcp_stdio)"` + ReadinessTimeout time.Duration `name:"readiness-timeout" help:"Protocol initialization deadline"` JSON bool `name:"json" help:"Print the result as JSON"` } @@ -419,6 +422,18 @@ type runValidationCmd struct { JSON bool `name:"json" help:"Print the result as JSON"` } +type repeatValidationCmd struct { + ID string `arg:"" help:"Validation definition ID"` + Kind string `name:"kind" default:"candidate" enum:"base,candidate,both" help:"Run target"` + Runs int `name:"runs" default:"3" help:"Attempts per target (1-100)"` + Concurrency int `name:"concurrency" default:"1" help:"Concurrent attempts (1-16)"` + PerRunTimeout time.Duration `name:"per-run-timeout" help:"Per-attempt timeout; defaults to the validation definition"` + OverallTimeout time.Duration `name:"overall-timeout" help:"Overall group timeout"` + SampleInterval time.Duration `name:"sample-interval" default:"100ms" help:"Process telemetry interval (10ms-10s)"` + Execute bool `name:"execute" help:"Authorize execution of the displayed command on the host"` + JSON bool `name:"json" help:"Print the result as JSON"` +} + type compareValidationCmd struct { BaseRunID string `arg:"" help:"Base run ID"` CandidateRunID string `arg:"" help:"Candidate run ID"` diff --git a/internal/cli/extended_test.go b/internal/cli/extended_test.go index c1f2eab..778d8cf 100644 --- a/internal/cli/extended_test.go +++ b/internal/cli/extended_test.go @@ -16,6 +16,7 @@ type fakeExtendedService struct { showWorkspaceCalled bool defineValidationCalled bool runValidationCalled bool + repeatValidationCalled bool compareValidationCalled bool showEvidenceCalled bool readinessCalled bool @@ -23,14 +24,15 @@ type fakeExtendedService struct { prepareIssueCalled bool preparePRCalled bool - workspaceResult *cli.WorkspaceResult - validationResult *cli.ValidationResult - validationRunResult *cli.ValidationRunResult - comparisonResult *cli.ValidationComparisonResult - evidenceResult *cli.EvidenceResult - readinessResult *cli.ReadinessResult - readinessCheck *cli.ReadinessCheck - draftResult *cli.DraftResult + workspaceResult *cli.WorkspaceResult + validationResult *cli.ValidationResult + validationRunResult *cli.ValidationRunResult + validationGroupResult *cli.ValidationRunGroupResult + comparisonResult *cli.ValidationComparisonResult + evidenceResult *cli.EvidenceResult + readinessResult *cli.ReadinessResult + readinessCheck *cli.ReadinessCheck + draftResult *cli.DraftResult lastWorkspaceInvestigation string lastCreateWorkspaceOpts cli.WorkspaceCreateOptions @@ -40,6 +42,7 @@ type fakeExtendedService struct { lastRunValidationID string lastRunKind string lastRunExecute bool + lastRepeatValidationOpts cli.RepeatValidationOptions lastCompareBase string lastCompareCandidate string lastEvidenceInvestigation string @@ -83,6 +86,13 @@ func (f *fakeExtendedService) RunValidation(ctx context.Context, id string, opts return f.validationRunResult, f.err } +func (f *fakeExtendedService) RunValidationGroup(_ context.Context, id string, opts cli.RepeatValidationOptions) (*cli.ValidationRunGroupResult, error) { + f.repeatValidationCalled = true + f.lastRunValidationID = id + f.lastRepeatValidationOpts = opts + return f.validationGroupResult, f.err +} + func (f *fakeExtendedService) CompareValidation(ctx context.Context, baseRunID, candidateRunID string) (*cli.ValidationComparisonResult, error) { f.compareValidationCalled = true f.lastCompareBase = baseRunID @@ -201,6 +211,10 @@ func TestValidationDefineRunAndCompare(t *testing.T) { Status: "matched", Excerpt: "expected failure", }}, }, + validationGroupResult: &cli.ValidationRunGroupResult{ + ID: "group-1", DefinitionID: "val-1", RequestedRuns: 3, CompletedRuns: 3, Classification: "stable_pass", + Aggregates: []cli.ValidationAggregateResult{{Kind: "candidate", Requested: 3, Completed: 3, Passing: 3, Classification: "stable_pass", ResourceClassification: "available"}}, + }, comparisonResult: &cli.ValidationComparisonResult{ Classification: "fixed", Explanation: "base failed, candidate passed", @@ -252,6 +266,16 @@ func TestValidationDefineRunAndCompare(t *testing.T) { t.Fatalf("observation output missing: %q", stdout.String()) } + stdout.Reset() + err = c.Run(context.Background(), []string{"validation", "repeat", "val-1", "--kind", "candidate", "--runs", "3", "--concurrency", "2", "--execute"}) + requireNoErr(t, err) + if !svc.repeatValidationCalled || svc.lastRepeatValidationOpts.RunCount != 3 || svc.lastRepeatValidationOpts.Concurrency != 2 || !svc.lastRepeatValidationOpts.Execute { + t.Fatalf("repeat validation opts = %+v", svc.lastRepeatValidationOpts) + } + if !strings.Contains(stdout.String(), "stable_pass") { + t.Fatalf("repeat output = %q", stdout.String()) + } + stdout.Reset() err = c.Run(context.Background(), []string{"validation", "compare", "run-base", "run-candidate"}) requireNoErr(t, err) diff --git a/internal/cli/interfaces.go b/internal/cli/interfaces.go index 6c9a16a..68eabf3 100644 --- a/internal/cli/interfaces.go +++ b/internal/cli/interfaces.go @@ -445,6 +445,7 @@ type ValidationService interface { DefineValidation(ctx context.Context, investigationID string, opts DefineValidationOptions) (*ValidationResult, error) ShowValidation(ctx context.Context, id string) (*ValidationResult, error) RunValidation(ctx context.Context, id string, opts RunValidationOptions) (*ValidationRunResult, error) + RunValidationGroup(ctx context.Context, id string, opts RepeatValidationOptions) (*ValidationRunGroupResult, error) CompareValidation(ctx context.Context, baseRunID, candidateRunID string) (*ValidationComparisonResult, error) } @@ -454,6 +455,17 @@ type RunValidationOptions struct { Execute bool } +// RepeatValidationOptions bounds an explicitly authorized run group. +type RepeatValidationOptions struct { + Kinds []string + RunCount int + Concurrency int + PerRunTimeout time.Duration + OverallTimeout time.Duration + SampleInterval time.Duration + Execute bool +} + // DefineValidationOptions carries an explicit validation definition. type DefineValidationOptions struct { Kind string @@ -468,6 +480,8 @@ type DefineValidationOptions struct { Timeout time.Duration MaxOutputBytes int64 Observation *ValidationObservationContract + Protocol string + ReadinessTimeout time.Duration } // ValidationResult is a stored validation definition view. @@ -486,6 +500,8 @@ type ValidationResult struct { Timeout string `json:"timeout,omitempty"` MaxOutputBytes int64 `json:"max_output_bytes,omitempty"` Observation *ValidationObservationContract `json:"observation,omitempty"` + Protocol string `json:"protocol,omitempty"` + ReadinessTimeout string `json:"readiness_timeout,omitempty"` CreatedAt string `json:"created_at"` } @@ -509,6 +525,120 @@ type ValidationRunResult struct { WorkspaceSnapshotAfter string `json:"workspace_snapshot_after,omitempty"` WorkspaceBindingStatus string `json:"workspace_binding_status,omitempty"` WorkspaceBindingReason string `json:"workspace_binding_reason,omitempty"` + Process ValidationProcessIdentity `json:"process"` + Phases ValidationRunPhases `json:"phases"` + TimeoutPhase string `json:"timeout_phase,omitempty"` + FailurePhase string `json:"failure_phase,omitempty"` + Resources ValidationResourceTelemetry `json:"resources"` + Cleanup ValidationCleanupResult `json:"cleanup"` +} + +// ValidationProcessIdentity identifies a sampled process without conflating PID reuse. +type ValidationProcessIdentity struct { + PID int32 `json:"pid,omitempty"` + CreateTimeUnixMilli int64 `json:"create_time_unix_milli,omitempty"` +} + +// ValidationRunPhases exposes process and declared protocol milestones. +type ValidationRunPhases struct { + SpawnStartedAt string `json:"spawn_started_at,omitempty"` + ProcessStartedAt string `json:"process_started_at,omitempty"` + InitializedAt string `json:"initialized_at,omitempty"` + ToolsListedAt string `json:"tools_listed_at,omitempty"` + FirstResponseAt string `json:"first_response_at,omitempty"` + ExecutionEndedAt string `json:"execution_ended_at,omitempty"` + ShutdownStartedAt string `json:"shutdown_started_at,omitempty"` + ShutdownCheckedAt string `json:"shutdown_checked_at,omitempty"` +} + +// ValidationInt64Metric distinguishes an observed zero from unavailable data. +type ValidationInt64Metric struct { + Value *int64 `json:"value,omitempty"` + UnavailableReason string `json:"unavailable_reason,omitempty"` +} + +// ValidationUint64Metric distinguishes an observed zero from unavailable data. +type ValidationUint64Metric struct { + Value *uint64 `json:"value,omitempty"` + UnavailableReason string `json:"unavailable_reason,omitempty"` +} + +// ValidationResourceTelemetry reports bounded process-tree high-water marks. +type ValidationResourceTelemetry struct { + Provider string `json:"provider"` + Platform string `json:"platform"` + SampleInterval string `json:"sample_interval"` + SampleCount int `json:"sample_count"` + CPUTimeMillis ValidationInt64Metric `json:"cpu_time_millis"` + PeakRSSBytes ValidationUint64Metric `json:"peak_rss_bytes"` + PeakChildCount ValidationInt64Metric `json:"peak_child_count"` + SamplerOverheadNanoseconds int64 `json:"sampler_overhead_nanoseconds"` +} + +// ValidationCleanupResult reports sampled descendants after shutdown. +type ValidationCleanupResult struct { + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Survivors []ValidationProcessIdentity `json:"survivors,omitempty"` + CheckedAt string `json:"checked_at,omitempty"` +} + +// ValidationAttemptResult summarizes one independently timed attempt. +type ValidationAttemptResult struct { + Index int `json:"index"` + Kind string `json:"kind"` + RunID string `json:"run_id,omitempty"` + StartedAt string `json:"started_at"` + CompletedAt string `json:"completed_at"` + ExitCode int `json:"exit_code"` + Classification string `json:"classification"` + ObservationStatus string `json:"observation_status"` + TimeoutPhase string `json:"timeout_phase,omitempty"` + FailurePhase string `json:"failure_phase,omitempty"` + Error string `json:"error,omitempty"` + Process ValidationProcessIdentity `json:"process"` + Phases ValidationRunPhases `json:"phases"` + Resources ValidationResourceTelemetry `json:"resources"` + Cleanup ValidationCleanupResult `json:"cleanup"` +} + +// ValidationAggregateResult classifies comparable attempts for one run kind. +type ValidationAggregateResult struct { + Kind string `json:"kind"` + Requested int `json:"requested"` + Completed int `json:"completed"` + Passing int `json:"passing"` + Failing int `json:"failing"` + Inconclusive int `json:"inconclusive"` + Cancelled int `json:"cancelled"` + Classification string `json:"classification"` + ResourceClassification string `json:"resource_classification"` +} + +// ValidationGroupComparisonResult compares stable base and candidate aggregates. +type ValidationGroupComparisonResult struct { + Classification string `json:"classification"` + Explanation string `json:"explanation"` +} + +// ValidationRunGroupResult is the bounded repeat/stress result returned to clients. +type ValidationRunGroupResult struct { + ID string `json:"id"` + DefinitionID string `json:"definition_id"` + InvestigationID string `json:"investigation_id"` + ConfigurationSHA256 string `json:"configuration_sha256"` + RequestedRuns int `json:"requested_runs"` + CompletedRuns int `json:"completed_runs"` + Concurrency int `json:"concurrency"` + PerRunTimeout string `json:"per_run_timeout"` + OverallTimeout string `json:"overall_timeout"` + SampleInterval string `json:"sample_interval"` + Attempts []ValidationAttemptResult `json:"attempts"` + Aggregates []ValidationAggregateResult `json:"aggregates"` + Classification string `json:"classification"` + Comparison *ValidationGroupComparisonResult `json:"comparison,omitempty"` + StartedAt string `json:"started_at"` + CompletedAt string `json:"completed_at"` } // ValidationComparisonResult classifies a base run against a candidate run. diff --git a/internal/cli/output.go b/internal/cli/output.go index 97db72f..4b569f6 100644 --- a/internal/cli/output.go +++ b/internal/cli/output.go @@ -106,6 +106,8 @@ func humanOutput(v any) (string, error) { return validationHuman(r), nil case *ValidationRunResult: return validationRunHuman(r), nil + case *ValidationRunGroupResult: + return validationRunGroupHuman(r), nil case *ValidationComparisonResult: return validationComparisonHuman(r), nil case *EvidenceResult: diff --git a/internal/cli/validation_cli.go b/internal/cli/validation_cli.go index b997ef8..d5e5574 100644 --- a/internal/cli/validation_cli.go +++ b/internal/cli/validation_cli.go @@ -17,6 +17,8 @@ func (c *CLI) runValidation(ctx context.Context, command string, cmd *validation return c.defineValidation(ctx, service, &cmd.Define) case "validation run": return c.executeValidation(ctx, service, &cmd.Run) + case "validation repeat": + return c.executeRepeatedValidation(ctx, service, &cmd.Repeat) case "validation compare": return c.compareValidation(ctx, service, &cmd.Compare) default: @@ -24,6 +26,32 @@ func (c *CLI) runValidation(ctx context.Context, command string, cmd *validation } } +func (c *CLI) executeRepeatedValidation(ctx context.Context, service ValidationService, cmd *repeatValidationCmd) error { + definition, err := service.ShowValidation(ctx, cmd.ID) + if err != nil { + return c.mapError(err) + } + if !cmd.Execute { + return NewCLIError(ExitUsage, fmt.Errorf("host execution requires --execute; command: %s", formatCommand(definition.Command))) + } + kinds := []string{cmd.Kind} + if cmd.Kind == "both" { + kinds = []string{"base", "candidate"} + } + if _, err := fmt.Fprintf(c.stderr, "executing %d validation attempt(s) per target: %s\n", cmd.Runs, formatCommand(definition.Command)); err != nil { + return err + } + result, err := service.RunValidationGroup(ctx, cmd.ID, RepeatValidationOptions{ + Kinds: kinds, RunCount: cmd.Runs, Concurrency: cmd.Concurrency, + PerRunTimeout: cmd.PerRunTimeout, OverallTimeout: cmd.OverallTimeout, SampleInterval: cmd.SampleInterval, + Execute: true, + }) + if err != nil { + return c.mapError(err) + } + return c.render(cmd.JSON, result) +} + func (c *CLI) defineValidation(ctx context.Context, service ValidationService, cmd *defineValidationCmd) error { if _, err := fmt.Fprintf(c.stderr, "defining validation for investigation %s...\n", cmd.InvestigationID); err != nil { return err @@ -40,7 +68,7 @@ func (c *CLI) defineValidation(ctx context.Context, service ValidationService, c BaseWorkingDir: cmd.BaseWorkingDir, CandidateDir: cmd.CandidateDir, WorkspaceID: cmd.WorkspaceID, BaseWorkspaceID: cmd.BaseWorkspaceID, CandidateWorkspaceID: cmd.CandidateWorkspaceID, Env: cmd.Env, Timeout: cmd.Timeout, MaxOutputBytes: cmd.MaxOutput, - Observation: observation, + Observation: observation, Protocol: cmd.Protocol, ReadinessTimeout: cmd.ReadinessTimeout, }) if err != nil { return c.mapError(err) diff --git a/internal/cli/validation_output.go b/internal/cli/validation_output.go index 6f6ae39..f0a7357 100644 --- a/internal/cli/validation_output.go +++ b/internal/cli/validation_output.go @@ -65,6 +65,20 @@ func validationRunHuman(r *ValidationRunResult) string { return b.String() } +func validationRunGroupHuman(group *ValidationRunGroupResult) string { + var b strings.Builder + fmt.Fprintf(&b, "Validation group %s: %s (%d/%d runs)\n", group.ID, group.Classification, group.CompletedRuns, group.RequestedRuns) + for _, aggregate := range group.Aggregates { + fmt.Fprintf(&b, " %s: %s; pass=%d fail=%d inconclusive=%d cancelled=%d resources=%s\n", + aggregate.Kind, aggregate.Classification, aggregate.Passing, aggregate.Failing, + aggregate.Inconclusive, aggregate.Cancelled, aggregate.ResourceClassification) + } + if group.Comparison != nil { + fmt.Fprintf(&b, " comparison: %s — %s\n", group.Comparison.Classification, group.Comparison.Explanation) + } + return b.String() +} + func validationComparisonHuman(r *ValidationComparisonResult) string { var b strings.Builder fmt.Fprintf(&b, "Comparison: %s\n", r.Classification) diff --git a/internal/mcpserver/catalog.go b/internal/mcpserver/catalog.go index 2844a68..b4175dc 100644 --- a/internal/mcpserver/catalog.go +++ b/internal/mcpserver/catalog.go @@ -46,6 +46,7 @@ const ( ToolCreateWorkspace = "workspace.create" ToolDefineValidation = "validation.define" ToolRunValidation = "validation.run" + ToolRunRepeatedValidation = "validation.run_repeated" ToolStartInvestigation = "workflow.start_investigation" ToolRecordHypothesis = "workflow.record_hypothesis" ToolCheckDuplicates = "workflow.check_duplicates" @@ -95,6 +96,7 @@ var canonicalToolNames = []string{ ToolCreateWorkspace, ToolDefineValidation, ToolRunValidation, + ToolRunRepeatedValidation, ToolStartInvestigation, ToolRecordHypothesis, ToolCreateConcern, @@ -189,7 +191,7 @@ var toolsets = map[string][]string{ "code": { ToolSearchCode, ToolIndexRepositories, ToolCreateWorkspace, ToolCheckMergeConflicts, ToolInspectCommitChanges, ToolPlanSemanticCommits, - ToolDefineValidation, ToolRunValidation, ToolGetJob, ToolCancelJob, + ToolDefineValidation, ToolRunValidation, ToolRunRepeatedValidation, ToolGetJob, ToolCancelJob, }, "research": {ToolQueryDeepWiki}, "diagnostics": {ToolExplainMatch, ToolBuildRepositoryDossier, ToolGetJob}, diff --git a/internal/mcpserver/catalog_test.go b/internal/mcpserver/catalog_test.go index 628dc54..5d4fe63 100644 --- a/internal/mcpserver/catalog_test.go +++ b/internal/mcpserver/catalog_test.go @@ -253,6 +253,10 @@ func TestToolSchemasExposeMachineReadableContracts(t *testing.T) { assertSchemaValue(t, tools[ToolRankThreads].OutputSchema, []string{"properties", "total", "type"}, "integer") assertSchemaValue(t, tools[ToolRankThreads].OutputSchema, []string{"properties", "truncated", "type"}, "boolean") assertSchemaValue(t, tools[ToolRunValidation].InputSchema, []string{"properties", "execute", "const"}, true) + assertSchemaValue(t, tools[ToolDefineValidation].InputSchema, []string{"properties", "protocol", "enum"}, []any{"mcp_stdio"}) + assertSchemaValue(t, tools[ToolRunRepeatedValidation].InputSchema, []string{"properties", "run_count", "default"}, float64(3)) + assertSchemaValue(t, tools[ToolRunRepeatedValidation].InputSchema, []string{"properties", "run_count", "maximum"}, float64(100)) + assertSchemaValue(t, tools[ToolRunRepeatedValidation].InputSchema, []string{"properties", "execute", "const"}, true) assertSchemaValue(t, tools[ToolPromoteOpportunity].InputSchema, []string{"properties", "confidence", "maximum"}, float64(1)) validationSchema, err := json.Marshal(tools[ToolDefineValidation].InputSchema) if err != nil { @@ -347,6 +351,7 @@ func TestAgentToolSelectionProxy(t *testing.T) { {"Clone the remote and create a managed Git worktree", ToolCreateWorkspace}, {"Render and persist a pull request draft from a verified managed workspace diff", ToolPrepareContribution}, {"Execute the stored validation command against the candidate workspace", ToolRunValidation}, + {"Run a repeat stress validation group with concurrency and telemetry", ToolRunRepeatedValidation}, {"Stop a running durable job", ToolCancelJob}, {"Poll several durable jobs together with structured progress", ToolGetJob}, {"Read stored facet coverage for several exact threads", ToolGetCoverage}, @@ -395,6 +400,8 @@ func TestInvalidToolCallEvaluation(t *testing.T) { {ToolHydrateThreads, map[string]any{"threads": []any{map[string]any{"owner": "acme", "repo": "rocket", "number": 1}}, "facets": []string{"unknown"}}}, {ToolSyncThreads, map[string]any{"selection": "threads", "threads": []any{map[string]any{"owner": "acme", "repo": "rocket", "number": 1}}, "state": "open"}}, {ToolRunValidation, map[string]any{"id": "val-1", "kind": "candidate", "execute": false}}, + {ToolRunRepeatedValidation, map[string]any{"id": "val-1", "target": "both", "run_count": 3, "execute": false}}, + {ToolDefineValidation, map[string]any{"investigation_id": "inv-1", "kind": "test", "command": "server", "workspace_id": "ws-1", "readiness_timeout": "30s"}}, {ToolPromoteOpportunity, map[string]any{"hypothesis_id": "hyp-1", "problem_statement": "p", "scope": "s", "impact": "i", "expected_effort": "e", "confidence": 1.1}}, {ToolPrepareContribution, map[string]any{"opportunity_id": "opp-1", "kind": "pull_request", "approach": "focused"}}, } @@ -475,6 +482,10 @@ func TestSideEffectAuthorizationEvaluation(t *testing.T) { if run == nil || run.ReadOnlyHint || run.DestructiveHint == nil || !*run.DestructiveHint { t.Fatalf("validation annotations = %+v", run) } + repeat := tools[ToolRunRepeatedValidation].Annotations + if repeat == nil || repeat.ReadOnlyHint || repeat.DestructiveHint == nil || !*repeat.DestructiveHint { + t.Fatalf("repeated validation annotations = %+v", repeat) + } prepare := tools[ToolPrepareContribution] if prepare.Annotations == nil || prepare.Annotations.ReadOnlyHint || prepare.Annotations.OpenWorldHint == nil || *prepare.Annotations.OpenWorldHint { t.Fatalf("prepare contribution annotations = %+v", prepare.Annotations) diff --git a/internal/mcpserver/server.go b/internal/mcpserver/server.go index 0b28367..d2e9f7e 100644 --- a/internal/mcpserver/server.go +++ b/internal/mcpserver/server.go @@ -100,6 +100,7 @@ type Operator interface { CreateWorkspace(context.Context, CreateWorkspaceInput) (JobReference, error) DefineValidation(context.Context, DefineValidationInput) (ValidationOutput, error) RunValidation(context.Context, RunValidationInput) (JobReference, error) + RunRepeatedValidation(context.Context, RunRepeatedValidationInput) (JobReference, error) PrepareContribution(context.Context, PrepareContributionInput) (DraftOutput, error) ExportManifest(context.Context, ExportManifestInput) (ManifestOutput, error) CancelJobs(context.Context, CancelJobInput) (GetJobsOutput, error) diff --git a/internal/mcpserver/server_test.go b/internal/mcpserver/server_test.go index 1f8ac6b..e0af456 100644 --- a/internal/mcpserver/server_test.go +++ b/internal/mcpserver/server_test.go @@ -13,6 +13,7 @@ import ( type fakeReader struct { searchStarted chan struct{} + repeatInput RunRepeatedValidationInput } func (f *fakeReader) Search(ctx context.Context, in SearchInput) (SearchOutput, error) { @@ -278,6 +279,11 @@ func (*fakeReader) RunValidation(_ context.Context, in RunValidationInput) (JobR return JobReference{ID: "job-run-" + in.ID, Kind: "run_validation", Status: "queued"}, nil } +func (f *fakeReader) RunRepeatedValidation(_ context.Context, in RunRepeatedValidationInput) (JobReference, error) { + f.repeatInput = in + return JobReference{ID: "job-repeat-" + in.ID, Kind: "run_validation_group", Status: "queued"}, nil +} + func (*fakeReader) PrepareContribution(_ context.Context, in PrepareContributionInput) (DraftOutput, error) { return DraftOutput{OpportunityID: in.OpportunityID, Kind: in.Kind, Title: "draft", Body: "body"}, nil } @@ -702,6 +708,7 @@ func TestV1ParityToolsAndResources(t *testing.T) { ToolSearchRepositories, ToolSearchThreads, ToolGetRepositoryDossier, ToolExplainMatch, ToolGetJob, ToolGetReadiness, ToolBuildRepositoryDossier, ToolCreateWorkspace, ToolRunValidation, ToolStartInvestigation, ToolRecordHypothesis, + ToolRunRepeatedValidation, ToolCheckDuplicates, ToolFindCompetingWork, ToolPromoteOpportunity, ToolDefineValidation, ToolPrepareContribution, ToolCancelJob, } { @@ -738,6 +745,7 @@ func TestV1ParityToolsAndResources(t *testing.T) { {ToolBuildRepositoryDossier, map[string]any{"owner": "acme", "repo": "rocket"}}, {ToolCreateWorkspace, map[string]any{"investigation_id": "inv-1"}}, {ToolRunValidation, map[string]any{"id": "val-1", "kind": "base", "execute": true}}, + {ToolRunRepeatedValidation, map[string]any{"id": "val-1", "target": "both", "execute": true}}, {ToolStartInvestigation, map[string]any{"owner": "acme", "repo": "rocket", "commit_sha": "abc123"}}, {ToolRecordHypothesis, map[string]any{"investigation_id": "inv-1", "title": "leak", "description": "memory leak", "category": "bug"}}, {ToolCheckDuplicates, map[string]any{"target": "hypothesis", "id": "hyp-1"}}, @@ -775,3 +783,19 @@ func TestV1ParityToolsAndResources(t *testing.T) { } } } + +func TestRepeatedValidationAppliesDocumentedDefaults(t *testing.T) { + fake := &fakeReader{searchStarted: make(chan struct{})} + client, closeSessions := connect(t, fake) + defer closeSessions() + result, err := client.CallTool(context.Background(), &mcp.CallToolParams{ + Name: ToolRunRepeatedValidation, + Arguments: map[string]any{"id": "val-1", "target": "candidate", "execute": true}, + }) + if err != nil || result.IsError { + t.Fatalf("call repeated validation: err=%v result=%+v", err, result) + } + if fake.repeatInput.RunCount != 3 || fake.repeatInput.Concurrency != 1 || fake.repeatInput.SampleInterval != "100ms" { + t.Fatalf("repeat defaults = %+v", fake.repeatInput) + } +} diff --git a/internal/mcpserver/v1.go b/internal/mcpserver/v1.go index 92deaea..636352c 100644 --- a/internal/mcpserver/v1.go +++ b/internal/mcpserver/v1.go @@ -150,6 +150,18 @@ type RunValidationInput struct { Execute bool `json:"execute" jsonschema:"Must be true to authorize host execution"` } +// RunRepeatedValidationInput configures one bounded repeat/stress job. +type RunRepeatedValidationInput struct { + ID string `json:"id" jsonschema:"Validation definition ID"` + Target string `json:"target" jsonschema:"Run target: base, candidate, or both"` + RunCount int `json:"run_count,omitempty" jsonschema:"Attempts per target from 1 to 100"` + Concurrency int `json:"concurrency,omitempty" jsonschema:"Concurrent attempts from 1 to 16"` + PerRunTimeout string `json:"per_run_timeout,omitempty" jsonschema:"Optional Go duration per attempt"` + OverallTimeout string `json:"overall_timeout,omitempty" jsonschema:"Optional Go duration for the whole group"` + SampleInterval string `json:"sample_interval,omitempty" jsonschema:"Process telemetry interval from 10ms to 10s"` + Execute bool `json:"execute" jsonschema:"Must be true to authorize host execution"` +} + // StartInvestigationInput creates a local investigation for a repository revision. type StartInvestigationInput struct { Owner string `json:"owner" jsonschema:"GitHub repository owner"` @@ -239,6 +251,8 @@ type DefineValidationInput struct { Timeout string `json:"timeout,omitempty" jsonschema:"Positive Go duration; defaults to 30m"` MaxOutputBytes int64 `json:"max_output_bytes,omitempty" jsonschema:"Maximum captured bytes per output stream; defaults to 65536"` Observation *ValidationObservationContract `json:"observation,omitempty" jsonschema:"Expected bounded observations over captured base and candidate output"` + Protocol string `json:"protocol,omitempty" jsonschema:"Structured protocol adapter: mcp_stdio"` + ReadinessTimeout string `json:"readiness_timeout,omitempty" jsonschema:"Protocol initialization deadline; defaults to 30s"` } // ValidationExpectedObservation is one output assertion evaluated without a shell. @@ -274,6 +288,8 @@ type ValidationOutput struct { Timeout string `json:"timeout,omitempty"` MaxOutputBytes int64 `json:"max_output_bytes,omitempty"` Observation *ValidationObservationContract `json:"observation,omitempty"` + Protocol string `json:"protocol,omitempty" jsonschema:"Declared structured protocol adapter"` + ReadinessTimeout string `json:"readiness_timeout,omitempty" jsonschema:"Protocol initialization deadline"` CreatedAt string `json:"created_at"` } @@ -342,6 +358,19 @@ func (s *Server) registerV1() { setConst(schema, "execute", true) }), output: outputSchema[JobReference]("Reference to a newly queued validation execution job."), handler: s.runValidation, }) + addCatalogTool(s, catalogTool[RunRepeatedValidationInput, JobReference]{ + name: ToolRunRepeatedValidation, title: "Run repeated validation", + description: "Execute a stored shell-free validation repeatedly with bounded concurrency, independent per-attempt deadlines, process-tree resource telemetry, cleanup checks, and semantic aggregation. Requires execute=true and never contacts GitHub.", + annotations: executionAnnotations(), supportedBy: supports[Operator], input: inputSchema[RunRepeatedValidationInput](func(schema *schemaBuilder) { + setEnum(schema, "target", "base", "candidate", "both") + setRange(schema, "run_count", 1, 100) + setDefault(schema, "run_count", 3) + setRange(schema, "concurrency", 1, 16) + setDefault(schema, "concurrency", 1) + setDefault(schema, "sample_interval", "100ms") + setConst(schema, "execute", true) + }), output: outputSchema[JobReference]("Reference to a newly queued repeat validation job."), handler: s.runRepeatedValidation, + }) addCatalogTool(s, catalogTool[StartInvestigationInput, InvestigationOutput]{ name: ToolStartInvestigation, title: "Start local investigation", description: "Create a local investigation from a commit SHA, or atomically create its initial baseline hypothesis from a stored issue or pull-request number. This does not create a Git worktree or contact GitHub; use " + ToolCreateWorkspace + " separately when filesystem work is authorized.", @@ -382,11 +411,12 @@ func (s *Server) registerV1() { }) addCatalogTool(s, catalogTool[DefineValidationInput, ValidationOutput]{ name: ToolDefineValidation, title: "Define validation command", - description: "Parse and persist a shell-free validation command for managed workspace IDs belonging to the investigation, with an environment allowlist, timeout, and output bound. This does not execute the command; use " + ToolRunValidation + " separately with explicit authorization.", + description: "Parse and persist a shell-free validation command for managed workspace IDs belonging to the investigation, with an environment allowlist, timeout, output bound, and optional declared MCP stdio adapter. This does not execute the command; use " + ToolRunValidation + " separately with explicit authorization.", annotations: localWrite, supportedBy: supports[Operator], input: inputSchema[DefineValidationInput](func(schema *schemaBuilder) { setDefault(schema, "timeout", "30m") setRange(schema, "max_output_bytes", 1, 64*1024*1024) setDefault(schema, "max_output_bytes", 64*1024) + setEnum(schema, "protocol", "mcp_stdio") configureValidationObservationSchema(schema) }), output: outputSchema[ValidationOutput]("Persisted validation definition."), handler: s.defineValidation, }) @@ -556,6 +586,35 @@ func (s *Server) runValidation(ctx context.Context, _ *mcp.CallToolRequest, in R return nil, out, err } +func (s *Server) runRepeatedValidation(ctx context.Context, _ *mcp.CallToolRequest, in RunRepeatedValidationInput) (*mcp.CallToolResult, JobReference, error) { + id, err := normalizeID("id", in.ID) + if err != nil { + return nil, JobReference{}, err + } + in.ID = id + if in.RunCount == 0 { + in.RunCount = 3 + } + if in.Concurrency == 0 { + in.Concurrency = 1 + } + if in.SampleInterval == "" { + in.SampleInterval = "100ms" + } + if in.Target != "base" && in.Target != "candidate" && in.Target != "both" { + return nil, JobReference{}, errors.New("target must be base, candidate, or both") + } + if !in.Execute { + return nil, JobReference{}, errors.New("execute must be true to authorize host command execution") + } + operator, ok := s.reader.(Operator) + if !ok { + return nil, JobReference{}, errors.New("validation is not available") + } + out, err := operator.RunRepeatedValidation(ctx, in) + return nil, out, err +} + func (s *Server) startInvestigation(ctx context.Context, _ *mcp.CallToolRequest, in StartInvestigationInput) (*mcp.CallToolResult, InvestigationOutput, error) { if err := validateRepo(RepoInput{Owner: in.Owner, Repo: in.Repo}); err != nil { return nil, InvestigationOutput{}, err @@ -684,6 +743,14 @@ func (s *Server) defineValidation(ctx context.Context, _ *mcp.CallToolRequest, i return nil, ValidationOutput{}, InvalidArgument("timeout", "must be a positive Go duration", map[string]any{"timeout": "30m"}) } } + if in.ReadinessTimeout != "" { + if duration, err := time.ParseDuration(in.ReadinessTimeout); err != nil || duration <= 0 { + return nil, ValidationOutput{}, InvalidArgument("readiness_timeout", "must be a positive Go duration", map[string]any{"readiness_timeout": "30s"}) + } + if in.Protocol == "" { + return nil, ValidationOutput{}, InvalidArgument("protocol", "readiness_timeout requires a declared protocol adapter", map[string]any{"protocol": "mcp_stdio", "readiness_timeout": "30s"}) + } + } if in.MaxOutputBytes < 0 { return nil, ValidationOutput{}, InvalidArgument("max_output_bytes", "cannot be negative", map[string]any{"max_output_bytes": 65536}) } From 734354807d0ffec7a3b99bceb3500b4ce7025672 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 23 Jul 2026 04:37:11 +0000 Subject: [PATCH 3/7] docs(validation): document repeat and protocol telemetry --- README.md | 9 +++++++++ docs/architecture.md | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 730c955..ba9479b 100644 --- a/README.md +++ b/README.md @@ -202,9 +202,18 @@ gitcontribute validation define --kind=test --command="go test ./..." \ --working-dir=/path/to/workspace gitcontribute validation run --kind=base --execute gitcontribute validation run --kind=candidate --execute +gitcontribute validation repeat --kind=both --runs=5 \ + --concurrency=2 --execute gitcontribute validation compare ``` +Repeat validation stores bounded per-attempt results plus stable/flaky/ +inconclusive aggregates. CPU time, peak RSS, child count, process identity, and +post-shutdown cleanup are sampled through `gopsutil`; unsupported metrics stay +explicitly unavailable. To measure MCP readiness without parsing stdout, define +the validation with `--protocol=mcp_stdio --readiness-timeout=30s`. The official +MCP Go SDK then records initialize and tools/list milestones. + ### 4. Check readiness Run a deterministic local readiness gate before turning a candidate into a diff --git a/docs/architecture.md b/docs/architecture.md index 8aff112..6b79c71 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -59,12 +59,21 @@ application and domain packages expose product-owned values and interfaces. | DeepWiki external read | public repository structure, contents, questions | yes | no | no | no | | Git acquisition | acquire, workspace create | remote-dependent | yes | `git` only | no | | Local merge check | compare already-fetched revisions | no | no | `git` only | no | -| Validation | validation run with explicit execution | no by default | yes | yes | no | +| Validation | validation run/repeat with explicit execution | no by default | yes | yes | no | Version 1 has no GitHub mutation path. Adding one requires a separate application capability and protocol annotation; it must not be hidden behind a read operation. +Validation definitions remain shell-free and execution requires an explicit +authorization flag. Repeat groups bound attempts, concurrency, per-run and +overall timeouts, captured output, and sampler frequency. The generic process +runner records spawn/execution/shutdown boundaries and uses `gopsutil` for +PID-plus-start-time process-tree metrics. Declared `mcp_stdio` definitions use +the official MCP SDK for initialize and tools/list milestones; protocol state +is never inferred from arbitrary stdout text. Metric availability and cleanup +status are persisted separately from semantic pass/fail classification. + ## Corpus model The corpus separates source history from convenient current state: From f888f88cef06eb954c4e1cfbf41d1f8bc350d954 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 23 Jul 2026 05:55:29 +0000 Subject: [PATCH 4/7] refactor(adapters): isolate validation contracts --- internal/cli/cli.go | 51 ------- internal/cli/interfaces.go | 209 -------------------------- internal/cli/validation_commands.go | 54 +++++++ internal/cli/validation_contracts.go | 215 +++++++++++++++++++++++++++ internal/mcpserver/v1.go | 165 -------------------- internal/mcpserver/validation_v1.go | 174 ++++++++++++++++++++++ 6 files changed, 443 insertions(+), 425 deletions(-) create mode 100644 internal/cli/validation_commands.go create mode 100644 internal/cli/validation_contracts.go create mode 100644 internal/mcpserver/validation_v1.go diff --git a/internal/cli/cli.go b/internal/cli/cli.go index d4abc6a..ec518e4 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -389,57 +389,6 @@ type setStatusOpportunityCmd struct { JSON bool `name:"json" help:"Print the result as JSON"` } -type validationCmd struct { - Define defineValidationCmd `cmd:"" help:"Define a validation"` - Run runValidationCmd `cmd:"" help:"Run a validation definition"` - Repeat repeatValidationCmd `cmd:"" help:"Run a bounded repeat/stress validation"` - Compare compareValidationCmd `cmd:"" help:"Compare two validation runs"` -} - -type defineValidationCmd struct { - InvestigationID string `arg:"" help:"Investigation ID"` - Kind string `name:"kind" required:"" help:"Validation kind"` - Command string `name:"command" required:"" help:"Command argv as a single string"` - WorkingDir string `name:"working-dir" help:"Working directory for both runs"` - BaseWorkingDir string `name:"base-working-dir" help:"Base workspace directory"` - CandidateDir string `name:"candidate-dir" help:"Candidate workspace directory"` - WorkspaceID string `name:"workspace-id" help:"Managed workspace ID for both runs"` - BaseWorkspaceID string `name:"base-workspace-id" help:"Managed base workspace ID"` - CandidateWorkspaceID string `name:"candidate-workspace-id" help:"Managed candidate workspace ID"` - Env []string `name:"env" help:"Host environment variable names to pass through"` - Timeout time.Duration `name:"timeout" help:"Maximum execution time"` - MaxOutput int64 `name:"max-output" help:"Maximum captured output bytes per stream"` - Observation string `name:"observation-contract" help:"JSON observation contract for base and candidate output"` - Protocol string `name:"protocol" help:"Structured protocol adapter (mcp_stdio)"` - ReadinessTimeout time.Duration `name:"readiness-timeout" help:"Protocol initialization deadline"` - JSON bool `name:"json" help:"Print the result as JSON"` -} - -type runValidationCmd struct { - ID string `arg:"" help:"Validation definition ID"` - Kind string `name:"kind" required:"" enum:"base,candidate" help:"Run kind"` - Execute bool `name:"execute" help:"Authorize execution of the displayed command on the host"` - JSON bool `name:"json" help:"Print the result as JSON"` -} - -type repeatValidationCmd struct { - ID string `arg:"" help:"Validation definition ID"` - Kind string `name:"kind" default:"candidate" enum:"base,candidate,both" help:"Run target"` - Runs int `name:"runs" default:"3" help:"Attempts per target (1-100)"` - Concurrency int `name:"concurrency" default:"1" help:"Concurrent attempts (1-16)"` - PerRunTimeout time.Duration `name:"per-run-timeout" help:"Per-attempt timeout; defaults to the validation definition"` - OverallTimeout time.Duration `name:"overall-timeout" help:"Overall group timeout"` - SampleInterval time.Duration `name:"sample-interval" default:"100ms" help:"Process telemetry interval (10ms-10s)"` - Execute bool `name:"execute" help:"Authorize execution of the displayed command on the host"` - JSON bool `name:"json" help:"Print the result as JSON"` -} - -type compareValidationCmd struct { - BaseRunID string `arg:"" help:"Base run ID"` - CandidateRunID string `arg:"" help:"Candidate run ID"` - JSON bool `name:"json" help:"Print the result as JSON"` -} - type prepareCmd struct { Issue issueCmd `cmd:"" help:"Prepare an issue draft"` PR prCmd `cmd:"" name:"pr" help:"Prepare a pull request draft"` diff --git a/internal/cli/interfaces.go b/internal/cli/interfaces.go index 68eabf3..c0ce701 100644 --- a/internal/cli/interfaces.go +++ b/internal/cli/interfaces.go @@ -440,215 +440,6 @@ type WorkspaceResult struct { CreatedAt string `json:"created_at"` } -// ValidationService is the optional validation management capability used by the CLI. -type ValidationService interface { - DefineValidation(ctx context.Context, investigationID string, opts DefineValidationOptions) (*ValidationResult, error) - ShowValidation(ctx context.Context, id string) (*ValidationResult, error) - RunValidation(ctx context.Context, id string, opts RunValidationOptions) (*ValidationRunResult, error) - RunValidationGroup(ctx context.Context, id string, opts RepeatValidationOptions) (*ValidationRunGroupResult, error) - CompareValidation(ctx context.Context, baseRunID, candidateRunID string) (*ValidationComparisonResult, error) -} - -// RunValidationOptions carries the run target and explicit host-execution authorization. -type RunValidationOptions struct { - Kind string - Execute bool -} - -// RepeatValidationOptions bounds an explicitly authorized run group. -type RepeatValidationOptions struct { - Kinds []string - RunCount int - Concurrency int - PerRunTimeout time.Duration - OverallTimeout time.Duration - SampleInterval time.Duration - Execute bool -} - -// DefineValidationOptions carries an explicit validation definition. -type DefineValidationOptions struct { - Kind string - Command string - WorkingDir string - BaseWorkingDir string - CandidateDir string - WorkspaceID string - BaseWorkspaceID string - CandidateWorkspaceID string - Env []string - Timeout time.Duration - MaxOutputBytes int64 - Observation *ValidationObservationContract - Protocol string - ReadinessTimeout time.Duration -} - -// ValidationResult is a stored validation definition view. -type ValidationResult struct { - ID string `json:"id"` - InvestigationID string `json:"investigation_id"` - Kind string `json:"kind"` - Command []string `json:"command"` - WorkingDir string `json:"working_dir"` - BaseWorkingDir string `json:"base_working_dir,omitempty"` - CandidateDir string `json:"candidate_dir,omitempty"` - WorkspaceID string `json:"workspace_id,omitempty"` - BaseWorkspaceID string `json:"base_workspace_id,omitempty"` - CandidateWorkspaceID string `json:"candidate_workspace_id,omitempty"` - Env []string `json:"environment_allowlist,omitempty"` - Timeout string `json:"timeout,omitempty"` - MaxOutputBytes int64 `json:"max_output_bytes,omitempty"` - Observation *ValidationObservationContract `json:"observation,omitempty"` - Protocol string `json:"protocol,omitempty"` - ReadinessTimeout string `json:"readiness_timeout,omitempty"` - CreatedAt string `json:"created_at"` -} - -// ValidationRunResult is the captured outcome of one validation run. -type ValidationRunResult struct { - ID string `json:"id"` - DefinitionID string `json:"definition_id"` - InvestigationID string `json:"investigation_id"` - Kind string `json:"kind"` - ExitCode int `json:"exit_code"` - Stdout string `json:"stdout"` - Stderr string `json:"stderr"` - Truncated bool `json:"truncated"` - Error string `json:"error,omitempty"` - Classification string `json:"classification"` - ObservationStatus string `json:"observation_status"` - Observations []ValidationObservationResult `json:"observations,omitempty"` - StartedAt string `json:"started_at"` - CompletedAt string `json:"completed_at"` - WorkspaceSnapshotBefore string `json:"workspace_snapshot_before,omitempty"` - WorkspaceSnapshotAfter string `json:"workspace_snapshot_after,omitempty"` - WorkspaceBindingStatus string `json:"workspace_binding_status,omitempty"` - WorkspaceBindingReason string `json:"workspace_binding_reason,omitempty"` - Process ValidationProcessIdentity `json:"process"` - Phases ValidationRunPhases `json:"phases"` - TimeoutPhase string `json:"timeout_phase,omitempty"` - FailurePhase string `json:"failure_phase,omitempty"` - Resources ValidationResourceTelemetry `json:"resources"` - Cleanup ValidationCleanupResult `json:"cleanup"` -} - -// ValidationProcessIdentity identifies a sampled process without conflating PID reuse. -type ValidationProcessIdentity struct { - PID int32 `json:"pid,omitempty"` - CreateTimeUnixMilli int64 `json:"create_time_unix_milli,omitempty"` -} - -// ValidationRunPhases exposes process and declared protocol milestones. -type ValidationRunPhases struct { - SpawnStartedAt string `json:"spawn_started_at,omitempty"` - ProcessStartedAt string `json:"process_started_at,omitempty"` - InitializedAt string `json:"initialized_at,omitempty"` - ToolsListedAt string `json:"tools_listed_at,omitempty"` - FirstResponseAt string `json:"first_response_at,omitempty"` - ExecutionEndedAt string `json:"execution_ended_at,omitempty"` - ShutdownStartedAt string `json:"shutdown_started_at,omitempty"` - ShutdownCheckedAt string `json:"shutdown_checked_at,omitempty"` -} - -// ValidationInt64Metric distinguishes an observed zero from unavailable data. -type ValidationInt64Metric struct { - Value *int64 `json:"value,omitempty"` - UnavailableReason string `json:"unavailable_reason,omitempty"` -} - -// ValidationUint64Metric distinguishes an observed zero from unavailable data. -type ValidationUint64Metric struct { - Value *uint64 `json:"value,omitempty"` - UnavailableReason string `json:"unavailable_reason,omitempty"` -} - -// ValidationResourceTelemetry reports bounded process-tree high-water marks. -type ValidationResourceTelemetry struct { - Provider string `json:"provider"` - Platform string `json:"platform"` - SampleInterval string `json:"sample_interval"` - SampleCount int `json:"sample_count"` - CPUTimeMillis ValidationInt64Metric `json:"cpu_time_millis"` - PeakRSSBytes ValidationUint64Metric `json:"peak_rss_bytes"` - PeakChildCount ValidationInt64Metric `json:"peak_child_count"` - SamplerOverheadNanoseconds int64 `json:"sampler_overhead_nanoseconds"` -} - -// ValidationCleanupResult reports sampled descendants after shutdown. -type ValidationCleanupResult struct { - Status string `json:"status"` - Reason string `json:"reason,omitempty"` - Survivors []ValidationProcessIdentity `json:"survivors,omitempty"` - CheckedAt string `json:"checked_at,omitempty"` -} - -// ValidationAttemptResult summarizes one independently timed attempt. -type ValidationAttemptResult struct { - Index int `json:"index"` - Kind string `json:"kind"` - RunID string `json:"run_id,omitempty"` - StartedAt string `json:"started_at"` - CompletedAt string `json:"completed_at"` - ExitCode int `json:"exit_code"` - Classification string `json:"classification"` - ObservationStatus string `json:"observation_status"` - TimeoutPhase string `json:"timeout_phase,omitempty"` - FailurePhase string `json:"failure_phase,omitempty"` - Error string `json:"error,omitempty"` - Process ValidationProcessIdentity `json:"process"` - Phases ValidationRunPhases `json:"phases"` - Resources ValidationResourceTelemetry `json:"resources"` - Cleanup ValidationCleanupResult `json:"cleanup"` -} - -// ValidationAggregateResult classifies comparable attempts for one run kind. -type ValidationAggregateResult struct { - Kind string `json:"kind"` - Requested int `json:"requested"` - Completed int `json:"completed"` - Passing int `json:"passing"` - Failing int `json:"failing"` - Inconclusive int `json:"inconclusive"` - Cancelled int `json:"cancelled"` - Classification string `json:"classification"` - ResourceClassification string `json:"resource_classification"` -} - -// ValidationGroupComparisonResult compares stable base and candidate aggregates. -type ValidationGroupComparisonResult struct { - Classification string `json:"classification"` - Explanation string `json:"explanation"` -} - -// ValidationRunGroupResult is the bounded repeat/stress result returned to clients. -type ValidationRunGroupResult struct { - ID string `json:"id"` - DefinitionID string `json:"definition_id"` - InvestigationID string `json:"investigation_id"` - ConfigurationSHA256 string `json:"configuration_sha256"` - RequestedRuns int `json:"requested_runs"` - CompletedRuns int `json:"completed_runs"` - Concurrency int `json:"concurrency"` - PerRunTimeout string `json:"per_run_timeout"` - OverallTimeout string `json:"overall_timeout"` - SampleInterval string `json:"sample_interval"` - Attempts []ValidationAttemptResult `json:"attempts"` - Aggregates []ValidationAggregateResult `json:"aggregates"` - Classification string `json:"classification"` - Comparison *ValidationGroupComparisonResult `json:"comparison,omitempty"` - StartedAt string `json:"started_at"` - CompletedAt string `json:"completed_at"` -} - -// ValidationComparisonResult classifies a base run against a candidate run. -type ValidationComparisonResult struct { - Base *ValidationRunResult `json:"base"` - Candidate *ValidationRunResult `json:"candidate"` - Classification string `json:"classification"` - Explanation string `json:"explanation"` -} - // EvidenceService is the optional evidence reading capability used by the CLI. type EvidenceService interface { ShowEvidence(ctx context.Context, investigationID string) (*EvidenceResult, error) diff --git a/internal/cli/validation_commands.go b/internal/cli/validation_commands.go new file mode 100644 index 0000000..f129e67 --- /dev/null +++ b/internal/cli/validation_commands.go @@ -0,0 +1,54 @@ +package cli + +import "time" + +type validationCmd struct { + Define defineValidationCmd `cmd:"" help:"Define a validation"` + Run runValidationCmd `cmd:"" help:"Run a validation definition"` + Repeat repeatValidationCmd `cmd:"" help:"Run a bounded repeat/stress validation"` + Compare compareValidationCmd `cmd:"" help:"Compare two validation runs"` +} + +type defineValidationCmd struct { + InvestigationID string `arg:"" help:"Investigation ID"` + Kind string `name:"kind" required:"" help:"Validation kind"` + Command string `name:"command" required:"" help:"Command argv as a single string"` + WorkingDir string `name:"working-dir" help:"Working directory for both runs"` + BaseWorkingDir string `name:"base-working-dir" help:"Base workspace directory"` + CandidateDir string `name:"candidate-dir" help:"Candidate workspace directory"` + WorkspaceID string `name:"workspace-id" help:"Managed workspace ID for both runs"` + BaseWorkspaceID string `name:"base-workspace-id" help:"Managed base workspace ID"` + CandidateWorkspaceID string `name:"candidate-workspace-id" help:"Managed candidate workspace ID"` + Env []string `name:"env" help:"Host environment variable names to pass through"` + Timeout time.Duration `name:"timeout" help:"Maximum execution time"` + MaxOutput int64 `name:"max-output" help:"Maximum captured output bytes per stream"` + Observation string `name:"observation-contract" help:"JSON observation contract for base and candidate output"` + Protocol string `name:"protocol" help:"Structured protocol adapter (mcp_stdio)"` + ReadinessTimeout time.Duration `name:"readiness-timeout" help:"Protocol initialization deadline"` + JSON bool `name:"json" help:"Print the result as JSON"` +} + +type runValidationCmd struct { + ID string `arg:"" help:"Validation definition ID"` + Kind string `name:"kind" required:"" enum:"base,candidate" help:"Run kind"` + Execute bool `name:"execute" help:"Authorize execution of the displayed command on the host"` + JSON bool `name:"json" help:"Print the result as JSON"` +} + +type repeatValidationCmd struct { + ID string `arg:"" help:"Validation definition ID"` + Kind string `name:"kind" default:"candidate" enum:"base,candidate,both" help:"Run target"` + Runs int `name:"runs" default:"3" help:"Attempts per target (1-100)"` + Concurrency int `name:"concurrency" default:"1" help:"Concurrent attempts (1-16)"` + PerRunTimeout time.Duration `name:"per-run-timeout" help:"Per-attempt timeout; defaults to the validation definition"` + OverallTimeout time.Duration `name:"overall-timeout" help:"Overall group timeout"` + SampleInterval time.Duration `name:"sample-interval" default:"100ms" help:"Process telemetry interval (10ms-10s)"` + Execute bool `name:"execute" help:"Authorize execution of the displayed command on the host"` + JSON bool `name:"json" help:"Print the result as JSON"` +} + +type compareValidationCmd struct { + BaseRunID string `arg:"" help:"Base run ID"` + CandidateRunID string `arg:"" help:"Candidate run ID"` + JSON bool `name:"json" help:"Print the result as JSON"` +} diff --git a/internal/cli/validation_contracts.go b/internal/cli/validation_contracts.go new file mode 100644 index 0000000..2d778b4 --- /dev/null +++ b/internal/cli/validation_contracts.go @@ -0,0 +1,215 @@ +package cli + +import ( + "context" + "time" +) + +// ValidationService is the optional validation management capability used by the CLI. +type ValidationService interface { + DefineValidation(ctx context.Context, investigationID string, opts DefineValidationOptions) (*ValidationResult, error) + ShowValidation(ctx context.Context, id string) (*ValidationResult, error) + RunValidation(ctx context.Context, id string, opts RunValidationOptions) (*ValidationRunResult, error) + RunValidationGroup(ctx context.Context, id string, opts RepeatValidationOptions) (*ValidationRunGroupResult, error) + CompareValidation(ctx context.Context, baseRunID, candidateRunID string) (*ValidationComparisonResult, error) +} + +// RunValidationOptions carries the run target and explicit host-execution authorization. +type RunValidationOptions struct { + Kind string + Execute bool +} + +// RepeatValidationOptions bounds an explicitly authorized run group. +type RepeatValidationOptions struct { + Kinds []string + RunCount int + Concurrency int + PerRunTimeout time.Duration + OverallTimeout time.Duration + SampleInterval time.Duration + Execute bool +} + +// DefineValidationOptions carries an explicit validation definition. +type DefineValidationOptions struct { + Kind string + Command string + WorkingDir string + BaseWorkingDir string + CandidateDir string + WorkspaceID string + BaseWorkspaceID string + CandidateWorkspaceID string + Env []string + Timeout time.Duration + MaxOutputBytes int64 + Observation *ValidationObservationContract + Protocol string + ReadinessTimeout time.Duration +} + +// ValidationResult is a stored validation definition view. +type ValidationResult struct { + ID string `json:"id"` + InvestigationID string `json:"investigation_id"` + Kind string `json:"kind"` + Command []string `json:"command"` + WorkingDir string `json:"working_dir"` + BaseWorkingDir string `json:"base_working_dir,omitempty"` + CandidateDir string `json:"candidate_dir,omitempty"` + WorkspaceID string `json:"workspace_id,omitempty"` + BaseWorkspaceID string `json:"base_workspace_id,omitempty"` + CandidateWorkspaceID string `json:"candidate_workspace_id,omitempty"` + Env []string `json:"environment_allowlist,omitempty"` + Timeout string `json:"timeout,omitempty"` + MaxOutputBytes int64 `json:"max_output_bytes,omitempty"` + Observation *ValidationObservationContract `json:"observation,omitempty"` + Protocol string `json:"protocol,omitempty"` + ReadinessTimeout string `json:"readiness_timeout,omitempty"` + CreatedAt string `json:"created_at"` +} + +// ValidationRunResult is the captured outcome of one validation run. +type ValidationRunResult struct { + ID string `json:"id"` + DefinitionID string `json:"definition_id"` + InvestigationID string `json:"investigation_id"` + Kind string `json:"kind"` + ExitCode int `json:"exit_code"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + Truncated bool `json:"truncated"` + Error string `json:"error,omitempty"` + Classification string `json:"classification"` + ObservationStatus string `json:"observation_status"` + Observations []ValidationObservationResult `json:"observations,omitempty"` + StartedAt string `json:"started_at"` + CompletedAt string `json:"completed_at"` + WorkspaceSnapshotBefore string `json:"workspace_snapshot_before,omitempty"` + WorkspaceSnapshotAfter string `json:"workspace_snapshot_after,omitempty"` + WorkspaceBindingStatus string `json:"workspace_binding_status,omitempty"` + WorkspaceBindingReason string `json:"workspace_binding_reason,omitempty"` + Process ValidationProcessIdentity `json:"process"` + Phases ValidationRunPhases `json:"phases"` + TimeoutPhase string `json:"timeout_phase,omitempty"` + FailurePhase string `json:"failure_phase,omitempty"` + Resources ValidationResourceTelemetry `json:"resources"` + Cleanup ValidationCleanupResult `json:"cleanup"` +} + +// ValidationProcessIdentity identifies a sampled process without conflating PID reuse. +type ValidationProcessIdentity struct { + PID int32 `json:"pid,omitempty"` + CreateTimeUnixMilli int64 `json:"create_time_unix_milli,omitempty"` +} + +// ValidationRunPhases exposes process and declared protocol milestones. +type ValidationRunPhases struct { + SpawnStartedAt string `json:"spawn_started_at,omitempty"` + ProcessStartedAt string `json:"process_started_at,omitempty"` + InitializedAt string `json:"initialized_at,omitempty"` + ToolsListedAt string `json:"tools_listed_at,omitempty"` + FirstResponseAt string `json:"first_response_at,omitempty"` + ExecutionEndedAt string `json:"execution_ended_at,omitempty"` + ShutdownStartedAt string `json:"shutdown_started_at,omitempty"` + ShutdownCheckedAt string `json:"shutdown_checked_at,omitempty"` +} + +// ValidationInt64Metric distinguishes an observed zero from unavailable data. +type ValidationInt64Metric struct { + Value *int64 `json:"value,omitempty"` + UnavailableReason string `json:"unavailable_reason,omitempty"` +} + +// ValidationUint64Metric distinguishes an observed zero from unavailable data. +type ValidationUint64Metric struct { + Value *uint64 `json:"value,omitempty"` + UnavailableReason string `json:"unavailable_reason,omitempty"` +} + +// ValidationResourceTelemetry reports bounded process-tree high-water marks. +type ValidationResourceTelemetry struct { + Provider string `json:"provider"` + Platform string `json:"platform"` + SampleInterval string `json:"sample_interval"` + SampleCount int `json:"sample_count"` + CPUTimeMillis ValidationInt64Metric `json:"cpu_time_millis"` + PeakRSSBytes ValidationUint64Metric `json:"peak_rss_bytes"` + PeakChildCount ValidationInt64Metric `json:"peak_child_count"` + SamplerOverheadNanoseconds int64 `json:"sampler_overhead_nanoseconds"` +} + +// ValidationCleanupResult reports sampled descendants after shutdown. +type ValidationCleanupResult struct { + Status string `json:"status"` + Reason string `json:"reason,omitempty"` + Survivors []ValidationProcessIdentity `json:"survivors,omitempty"` + CheckedAt string `json:"checked_at,omitempty"` +} + +// ValidationAttemptResult summarizes one independently timed attempt. +type ValidationAttemptResult struct { + Index int `json:"index"` + Kind string `json:"kind"` + RunID string `json:"run_id,omitempty"` + StartedAt string `json:"started_at"` + CompletedAt string `json:"completed_at"` + ExitCode int `json:"exit_code"` + Classification string `json:"classification"` + ObservationStatus string `json:"observation_status"` + TimeoutPhase string `json:"timeout_phase,omitempty"` + FailurePhase string `json:"failure_phase,omitempty"` + Error string `json:"error,omitempty"` + Process ValidationProcessIdentity `json:"process"` + Phases ValidationRunPhases `json:"phases"` + Resources ValidationResourceTelemetry `json:"resources"` + Cleanup ValidationCleanupResult `json:"cleanup"` +} + +// ValidationAggregateResult classifies comparable attempts for one run kind. +type ValidationAggregateResult struct { + Kind string `json:"kind"` + Requested int `json:"requested"` + Completed int `json:"completed"` + Passing int `json:"passing"` + Failing int `json:"failing"` + Inconclusive int `json:"inconclusive"` + Cancelled int `json:"cancelled"` + Classification string `json:"classification"` + ResourceClassification string `json:"resource_classification"` +} + +// ValidationGroupComparisonResult compares stable base and candidate aggregates. +type ValidationGroupComparisonResult struct { + Classification string `json:"classification"` + Explanation string `json:"explanation"` +} + +// ValidationRunGroupResult is the bounded repeat/stress result returned to clients. +type ValidationRunGroupResult struct { + ID string `json:"id"` + DefinitionID string `json:"definition_id"` + InvestigationID string `json:"investigation_id"` + ConfigurationSHA256 string `json:"configuration_sha256"` + RequestedRuns int `json:"requested_runs"` + CompletedRuns int `json:"completed_runs"` + Concurrency int `json:"concurrency"` + PerRunTimeout string `json:"per_run_timeout"` + OverallTimeout string `json:"overall_timeout"` + SampleInterval string `json:"sample_interval"` + Attempts []ValidationAttemptResult `json:"attempts"` + Aggregates []ValidationAggregateResult `json:"aggregates"` + Classification string `json:"classification"` + Comparison *ValidationGroupComparisonResult `json:"comparison,omitempty"` + StartedAt string `json:"started_at"` + CompletedAt string `json:"completed_at"` +} + +// ValidationComparisonResult classifies a base run against a candidate run. +type ValidationComparisonResult struct { + Base *ValidationRunResult `json:"base"` + Candidate *ValidationRunResult `json:"candidate"` + Classification string `json:"classification"` + Explanation string `json:"explanation"` +} diff --git a/internal/mcpserver/v1.go b/internal/mcpserver/v1.go index 636352c..4982682 100644 --- a/internal/mcpserver/v1.go +++ b/internal/mcpserver/v1.go @@ -5,7 +5,6 @@ import ( "errors" "strconv" "strings" - "time" "github.com/modelcontextprotocol/go-sdk/mcp" ) @@ -143,25 +142,6 @@ type CreateWorkspaceInput struct { Name string `json:"name,omitempty" jsonschema:"Workspace name; defaults to a generated ID"` } -// RunValidationInput selects a validation definition and explicitly authorizes execution. -type RunValidationInput struct { - ID string `json:"id" jsonschema:"Validation definition ID"` - Kind string `json:"kind" jsonschema:"Run kind: base or candidate"` - Execute bool `json:"execute" jsonschema:"Must be true to authorize host execution"` -} - -// RunRepeatedValidationInput configures one bounded repeat/stress job. -type RunRepeatedValidationInput struct { - ID string `json:"id" jsonschema:"Validation definition ID"` - Target string `json:"target" jsonschema:"Run target: base, candidate, or both"` - RunCount int `json:"run_count,omitempty" jsonschema:"Attempts per target from 1 to 100"` - Concurrency int `json:"concurrency,omitempty" jsonschema:"Concurrent attempts from 1 to 16"` - PerRunTimeout string `json:"per_run_timeout,omitempty" jsonschema:"Optional Go duration per attempt"` - OverallTimeout string `json:"overall_timeout,omitempty" jsonschema:"Optional Go duration for the whole group"` - SampleInterval string `json:"sample_interval,omitempty" jsonschema:"Process telemetry interval from 10ms to 10s"` - Execute bool `json:"execute" jsonschema:"Must be true to authorize host execution"` -} - // StartInvestigationInput creates a local investigation for a repository revision. type StartInvestigationInput struct { Owner string `json:"owner" jsonschema:"GitHub repository owner"` @@ -239,60 +219,6 @@ type PromoteOpportunityInput struct { SourceRefs []SourceRef `json:"source_refs,omitempty" jsonschema:"Source references"` } -// DefineValidationInput records a bounded validation command without executing it. -type DefineValidationInput struct { - InvestigationID string `json:"investigation_id" jsonschema:"Investigation ID"` - Kind string `json:"kind" jsonschema:"Validation kind"` - Command string `json:"command" jsonschema:"Shell-free command to execute"` - WorkspaceID string `json:"workspace_id,omitempty" jsonschema:"Managed workspace ID used for both run kinds"` - BaseWorkspaceID string `json:"base_workspace_id,omitempty" jsonschema:"Managed base workspace ID; requires candidate_workspace_id"` - CandidateWorkspaceID string `json:"candidate_workspace_id,omitempty" jsonschema:"Managed candidate workspace ID; requires base_workspace_id"` - Env []string `json:"env,omitempty" jsonschema:"Allowed environment variable names"` - Timeout string `json:"timeout,omitempty" jsonschema:"Positive Go duration; defaults to 30m"` - MaxOutputBytes int64 `json:"max_output_bytes,omitempty" jsonschema:"Maximum captured bytes per output stream; defaults to 65536"` - Observation *ValidationObservationContract `json:"observation,omitempty" jsonschema:"Expected bounded observations over captured base and candidate output"` - Protocol string `json:"protocol,omitempty" jsonschema:"Structured protocol adapter: mcp_stdio"` - ReadinessTimeout string `json:"readiness_timeout,omitempty" jsonschema:"Protocol initialization deadline; defaults to 30s"` -} - -// ValidationExpectedObservation is one output assertion evaluated without a shell. -type ValidationExpectedObservation struct { - Run string `json:"run" jsonschema:"Run kind: base or candidate"` - Name string `json:"name" jsonschema:"Short observation name"` - Source string `json:"source" jsonschema:"Captured source: stdout, stderr, or artifact"` - Matcher string `json:"matcher" jsonschema:"Matcher: exact or regexp"` - Pattern string `json:"pattern" jsonschema:"Bounded exact string or Go regular expression"` - Occurrence string `json:"occurrence,omitempty" jsonschema:"Expected occurrence: present or absent; defaults to present"` - Path string `json:"path,omitempty" jsonschema:"Relative artifact path; valid only when source is artifact"` -} - -// ValidationObservationContract ties output assertions to the claimed behavior. -type ValidationObservationContract struct { - Intent string `json:"intent" jsonschema:"Short proof intent or invariant"` - Observations []ValidationExpectedObservation `json:"observations" jsonschema:"One to eight expected observations for each of base and candidate"` -} - -// ValidationOutput is the stable MCP representation of a validation definition. -type ValidationOutput struct { - ID string `json:"id"` - InvestigationID string `json:"investigation_id"` - Kind string `json:"kind"` - Command []string `json:"command"` - WorkingDir string `json:"working_dir"` - BaseWorkingDir string `json:"base_working_dir,omitempty"` - CandidateDir string `json:"candidate_dir,omitempty"` - WorkspaceID string `json:"workspace_id,omitempty" jsonschema:"Managed workspace ID used for both run kinds"` - BaseWorkspaceID string `json:"base_workspace_id,omitempty" jsonschema:"Managed base workspace ID"` - CandidateWorkspaceID string `json:"candidate_workspace_id,omitempty" jsonschema:"Managed candidate workspace ID"` - Env []string `json:"environment_allowlist,omitempty"` - Timeout string `json:"timeout,omitempty"` - MaxOutputBytes int64 `json:"max_output_bytes,omitempty"` - Observation *ValidationObservationContract `json:"observation,omitempty"` - Protocol string `json:"protocol,omitempty" jsonschema:"Declared structured protocol adapter"` - ReadinessTimeout string `json:"readiness_timeout,omitempty" jsonschema:"Protocol initialization deadline"` - CreatedAt string `json:"created_at"` -} - // CancelJobInput selects durable jobs for bounded, persisted cancellation. type CancelJobInput struct { IDs []string `json:"ids" jsonschema:"One to 100 durable job IDs"` @@ -566,55 +492,6 @@ func (s *Server) createWorkspace(ctx context.Context, _ *mcp.CallToolRequest, in return nil, out, err } -func (s *Server) runValidation(ctx context.Context, _ *mcp.CallToolRequest, in RunValidationInput) (*mcp.CallToolResult, JobReference, error) { - id, err := normalizeID("id", in.ID) - if err != nil { - return nil, JobReference{}, err - } - in.ID = id - if in.Kind != "base" && in.Kind != "candidate" { - return nil, JobReference{}, errors.New("kind must be base or candidate") - } - if !in.Execute { - return nil, JobReference{}, errors.New("execute must be true to authorize host command execution") - } - operator, ok := s.reader.(Operator) - if !ok { - return nil, JobReference{}, errors.New("validation is not available") - } - out, err := operator.RunValidation(ctx, in) - return nil, out, err -} - -func (s *Server) runRepeatedValidation(ctx context.Context, _ *mcp.CallToolRequest, in RunRepeatedValidationInput) (*mcp.CallToolResult, JobReference, error) { - id, err := normalizeID("id", in.ID) - if err != nil { - return nil, JobReference{}, err - } - in.ID = id - if in.RunCount == 0 { - in.RunCount = 3 - } - if in.Concurrency == 0 { - in.Concurrency = 1 - } - if in.SampleInterval == "" { - in.SampleInterval = "100ms" - } - if in.Target != "base" && in.Target != "candidate" && in.Target != "both" { - return nil, JobReference{}, errors.New("target must be base, candidate, or both") - } - if !in.Execute { - return nil, JobReference{}, errors.New("execute must be true to authorize host command execution") - } - operator, ok := s.reader.(Operator) - if !ok { - return nil, JobReference{}, errors.New("validation is not available") - } - out, err := operator.RunRepeatedValidation(ctx, in) - return nil, out, err -} - func (s *Server) startInvestigation(ctx context.Context, _ *mcp.CallToolRequest, in StartInvestigationInput) (*mcp.CallToolResult, InvestigationOutput, error) { if err := validateRepo(RepoInput{Owner: in.Owner, Repo: in.Repo}); err != nil { return nil, InvestigationOutput{}, err @@ -720,48 +597,6 @@ func (s *Server) promoteOpportunity(ctx context.Context, _ *mcp.CallToolRequest, return nil, out, err } -func (s *Server) defineValidation(ctx context.Context, _ *mcp.CallToolRequest, in DefineValidationInput) (*mcp.CallToolResult, ValidationOutput, error) { - if _, err := normalizeID("investigation_id", in.InvestigationID); err != nil { - return nil, ValidationOutput{}, err - } - in.Kind = strings.TrimSpace(in.Kind) - in.Command = strings.TrimSpace(in.Command) - in.WorkspaceID = strings.TrimSpace(in.WorkspaceID) - in.BaseWorkspaceID = strings.TrimSpace(in.BaseWorkspaceID) - in.CandidateWorkspaceID = strings.TrimSpace(in.CandidateWorkspaceID) - if in.Kind == "" || in.Command == "" { - return nil, ValidationOutput{}, InvalidArgument("command", "investigation_id, kind, and command are required", map[string]any{"investigation_id": in.InvestigationID, "kind": "regression", "command": "go test ./..."}) - } - if in.WorkspaceID != "" && (in.BaseWorkspaceID != "" || in.CandidateWorkspaceID != "") { - return nil, ValidationOutput{}, InvalidArgument("workspace_id", "cannot be combined with base_workspace_id or candidate_workspace_id", map[string]any{"workspace_id": in.WorkspaceID}) - } - if in.WorkspaceID == "" && (in.BaseWorkspaceID == "" || in.CandidateWorkspaceID == "") { - return nil, ValidationOutput{}, InvalidArgument("base_workspace_id", "base_workspace_id and candidate_workspace_id must be provided together", map[string]any{"base_workspace_id": "", "candidate_workspace_id": ""}) - } - if in.Timeout != "" { - if _, err := time.ParseDuration(in.Timeout); err != nil { - return nil, ValidationOutput{}, InvalidArgument("timeout", "must be a positive Go duration", map[string]any{"timeout": "30m"}) - } - } - if in.ReadinessTimeout != "" { - if duration, err := time.ParseDuration(in.ReadinessTimeout); err != nil || duration <= 0 { - return nil, ValidationOutput{}, InvalidArgument("readiness_timeout", "must be a positive Go duration", map[string]any{"readiness_timeout": "30s"}) - } - if in.Protocol == "" { - return nil, ValidationOutput{}, InvalidArgument("protocol", "readiness_timeout requires a declared protocol adapter", map[string]any{"protocol": "mcp_stdio", "readiness_timeout": "30s"}) - } - } - if in.MaxOutputBytes < 0 { - return nil, ValidationOutput{}, InvalidArgument("max_output_bytes", "cannot be negative", map[string]any{"max_output_bytes": 65536}) - } - operator, ok := s.reader.(Operator) - if !ok { - return nil, ValidationOutput{}, errors.New("validation definition is not available") - } - out, err := operator.DefineValidation(ctx, in) - return nil, out, err -} - func (s *Server) cancelJob(ctx context.Context, _ *mcp.CallToolRequest, in CancelJobInput) (*mcp.CallToolResult, GetJobsOutput, error) { if len(in.IDs) < 1 || len(in.IDs) > 100 { return nil, GetJobsOutput{}, errors.New("ids must contain 1 to 100 items") diff --git a/internal/mcpserver/validation_v1.go b/internal/mcpserver/validation_v1.go new file mode 100644 index 0000000..406c928 --- /dev/null +++ b/internal/mcpserver/validation_v1.go @@ -0,0 +1,174 @@ +package mcpserver + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// RunValidationInput selects a validation definition and explicitly authorizes execution. +type RunValidationInput struct { + ID string `json:"id" jsonschema:"Validation definition ID"` + Kind string `json:"kind" jsonschema:"Run kind: base or candidate"` + Execute bool `json:"execute" jsonschema:"Must be true to authorize host execution"` +} + +// RunRepeatedValidationInput configures one bounded repeat/stress job. +type RunRepeatedValidationInput struct { + ID string `json:"id" jsonschema:"Validation definition ID"` + Target string `json:"target" jsonschema:"Run target: base, candidate, or both"` + RunCount int `json:"run_count,omitempty" jsonschema:"Attempts per target from 1 to 100"` + Concurrency int `json:"concurrency,omitempty" jsonschema:"Concurrent attempts from 1 to 16"` + PerRunTimeout string `json:"per_run_timeout,omitempty" jsonschema:"Optional Go duration per attempt"` + OverallTimeout string `json:"overall_timeout,omitempty" jsonschema:"Optional Go duration for the whole group"` + SampleInterval string `json:"sample_interval,omitempty" jsonschema:"Process telemetry interval from 10ms to 10s"` + Execute bool `json:"execute" jsonschema:"Must be true to authorize host execution"` +} + +// DefineValidationInput records a bounded validation command without executing it. +type DefineValidationInput struct { + InvestigationID string `json:"investigation_id" jsonschema:"Investigation ID"` + Kind string `json:"kind" jsonschema:"Validation kind"` + Command string `json:"command" jsonschema:"Shell-free command to execute"` + WorkspaceID string `json:"workspace_id,omitempty" jsonschema:"Managed workspace ID used for both run kinds"` + BaseWorkspaceID string `json:"base_workspace_id,omitempty" jsonschema:"Managed base workspace ID; requires candidate_workspace_id"` + CandidateWorkspaceID string `json:"candidate_workspace_id,omitempty" jsonschema:"Managed candidate workspace ID; requires base_workspace_id"` + Env []string `json:"env,omitempty" jsonschema:"Allowed environment variable names"` + Timeout string `json:"timeout,omitempty" jsonschema:"Positive Go duration; defaults to 30m"` + MaxOutputBytes int64 `json:"max_output_bytes,omitempty" jsonschema:"Maximum captured bytes per output stream; defaults to 65536"` + Observation *ValidationObservationContract `json:"observation,omitempty" jsonschema:"Expected bounded observations over captured base and candidate output"` + Protocol string `json:"protocol,omitempty" jsonschema:"Structured protocol adapter: mcp_stdio"` + ReadinessTimeout string `json:"readiness_timeout,omitempty" jsonschema:"Protocol initialization deadline; defaults to 30s"` +} + +// ValidationExpectedObservation is one output assertion evaluated without a shell. +type ValidationExpectedObservation struct { + Run string `json:"run" jsonschema:"Run kind: base or candidate"` + Name string `json:"name" jsonschema:"Short observation name"` + Source string `json:"source" jsonschema:"Captured source: stdout, stderr, or artifact"` + Matcher string `json:"matcher" jsonschema:"Matcher: exact or regexp"` + Pattern string `json:"pattern" jsonschema:"Bounded exact string or Go regular expression"` + Occurrence string `json:"occurrence,omitempty" jsonschema:"Expected occurrence: present or absent; defaults to present"` + Path string `json:"path,omitempty" jsonschema:"Relative artifact path; valid only when source is artifact"` +} + +// ValidationObservationContract ties output assertions to the claimed behavior. +type ValidationObservationContract struct { + Intent string `json:"intent" jsonschema:"Short proof intent or invariant"` + Observations []ValidationExpectedObservation `json:"observations" jsonschema:"One to eight expected observations for each of base and candidate"` +} + +// ValidationOutput is the stable MCP representation of a validation definition. +type ValidationOutput struct { + ID string `json:"id"` + InvestigationID string `json:"investigation_id"` + Kind string `json:"kind"` + Command []string `json:"command"` + WorkingDir string `json:"working_dir"` + BaseWorkingDir string `json:"base_working_dir,omitempty"` + CandidateDir string `json:"candidate_dir,omitempty"` + WorkspaceID string `json:"workspace_id,omitempty" jsonschema:"Managed workspace ID used for both run kinds"` + BaseWorkspaceID string `json:"base_workspace_id,omitempty" jsonschema:"Managed base workspace ID"` + CandidateWorkspaceID string `json:"candidate_workspace_id,omitempty" jsonschema:"Managed candidate workspace ID"` + Env []string `json:"environment_allowlist,omitempty"` + Timeout string `json:"timeout,omitempty"` + MaxOutputBytes int64 `json:"max_output_bytes,omitempty"` + Observation *ValidationObservationContract `json:"observation,omitempty"` + Protocol string `json:"protocol,omitempty" jsonschema:"Declared structured protocol adapter"` + ReadinessTimeout string `json:"readiness_timeout,omitempty" jsonschema:"Protocol initialization deadline"` + CreatedAt string `json:"created_at"` +} + +func (s *Server) runValidation(ctx context.Context, _ *mcp.CallToolRequest, in RunValidationInput) (*mcp.CallToolResult, JobReference, error) { + id, err := normalizeID("id", in.ID) + if err != nil { + return nil, JobReference{}, err + } + in.ID = id + if in.Kind != "base" && in.Kind != "candidate" { + return nil, JobReference{}, errors.New("kind must be base or candidate") + } + if !in.Execute { + return nil, JobReference{}, errors.New("execute must be true to authorize host command execution") + } + operator, ok := s.reader.(Operator) + if !ok { + return nil, JobReference{}, errors.New("validation is not available") + } + out, err := operator.RunValidation(ctx, in) + return nil, out, err +} + +func (s *Server) runRepeatedValidation(ctx context.Context, _ *mcp.CallToolRequest, in RunRepeatedValidationInput) (*mcp.CallToolResult, JobReference, error) { + id, err := normalizeID("id", in.ID) + if err != nil { + return nil, JobReference{}, err + } + in.ID = id + if in.RunCount == 0 { + in.RunCount = 3 + } + if in.Concurrency == 0 { + in.Concurrency = 1 + } + if in.SampleInterval == "" { + in.SampleInterval = "100ms" + } + if in.Target != "base" && in.Target != "candidate" && in.Target != "both" { + return nil, JobReference{}, errors.New("target must be base, candidate, or both") + } + if !in.Execute { + return nil, JobReference{}, errors.New("execute must be true to authorize host command execution") + } + operator, ok := s.reader.(Operator) + if !ok { + return nil, JobReference{}, errors.New("validation is not available") + } + out, err := operator.RunRepeatedValidation(ctx, in) + return nil, out, err +} + +func (s *Server) defineValidation(ctx context.Context, _ *mcp.CallToolRequest, in DefineValidationInput) (*mcp.CallToolResult, ValidationOutput, error) { + if _, err := normalizeID("investigation_id", in.InvestigationID); err != nil { + return nil, ValidationOutput{}, err + } + in.Kind = strings.TrimSpace(in.Kind) + in.Command = strings.TrimSpace(in.Command) + in.WorkspaceID = strings.TrimSpace(in.WorkspaceID) + in.BaseWorkspaceID = strings.TrimSpace(in.BaseWorkspaceID) + in.CandidateWorkspaceID = strings.TrimSpace(in.CandidateWorkspaceID) + if in.Kind == "" || in.Command == "" { + return nil, ValidationOutput{}, InvalidArgument("command", "investigation_id, kind, and command are required", map[string]any{"investigation_id": in.InvestigationID, "kind": "regression", "command": "go test ./..."}) + } + if in.WorkspaceID != "" && (in.BaseWorkspaceID != "" || in.CandidateWorkspaceID != "") { + return nil, ValidationOutput{}, InvalidArgument("workspace_id", "cannot be combined with base_workspace_id or candidate_workspace_id", map[string]any{"workspace_id": in.WorkspaceID}) + } + if in.WorkspaceID == "" && (in.BaseWorkspaceID == "" || in.CandidateWorkspaceID == "") { + return nil, ValidationOutput{}, InvalidArgument("base_workspace_id", "base_workspace_id and candidate_workspace_id must be provided together", map[string]any{"base_workspace_id": "", "candidate_workspace_id": ""}) + } + if in.Timeout != "" { + if _, err := time.ParseDuration(in.Timeout); err != nil { + return nil, ValidationOutput{}, InvalidArgument("timeout", "must be a positive Go duration", map[string]any{"timeout": "30m"}) + } + } + if in.ReadinessTimeout != "" { + if duration, err := time.ParseDuration(in.ReadinessTimeout); err != nil || duration <= 0 { + return nil, ValidationOutput{}, InvalidArgument("readiness_timeout", "must be a positive Go duration", map[string]any{"readiness_timeout": "30s"}) + } + if in.Protocol == "" { + return nil, ValidationOutput{}, InvalidArgument("protocol", "readiness_timeout requires a declared protocol adapter", map[string]any{"protocol": "mcp_stdio", "readiness_timeout": "30s"}) + } + } + if in.MaxOutputBytes < 0 { + return nil, ValidationOutput{}, InvalidArgument("max_output_bytes", "cannot be negative", map[string]any{"max_output_bytes": 65536}) + } + operator, ok := s.reader.(Operator) + if !ok { + return nil, ValidationOutput{}, errors.New("validation definition is not available") + } + out, err := operator.DefineValidation(ctx, in) + return nil, out, err +} From 1a112556e5819baebfa851db5edb1d87ab5afb8f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 23 Jul 2026 06:12:40 +0000 Subject: [PATCH 5/7] refactor(app): isolate repeat validation adapter --- internal/app/mcp_v1.go | 60 --------------------- internal/app/repeated_validation_mcp.go | 72 +++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 60 deletions(-) create mode 100644 internal/app/repeated_validation_mcp.go diff --git a/internal/app/mcp_v1.go b/internal/app/mcp_v1.go index d438c19..ef9576e 100644 --- a/internal/app/mcp_v1.go +++ b/internal/app/mcp_v1.go @@ -351,66 +351,6 @@ func (r *MCPReader) RunValidation(ctx context.Context, in mcpserver.RunValidatio return queuedJobReference(id, "run_validation", "validation run started"), nil } -// RunRepeatedValidation submits a durable repeat/stress validation group. -func (r *MCPReader) RunRepeatedValidation(ctx context.Context, in mcpserver.RunRepeatedValidationInput) (mcpserver.JobReference, error) { - if !in.Execute { - return mcpserver.JobReference{}, errors.New("execute must be true to authorize host command execution") - } - if in.RunCount == 0 { - in.RunCount = 3 - } - if in.Concurrency == 0 { - in.Concurrency = 1 - } - if in.SampleInterval == "" { - in.SampleInterval = "100ms" - } - kinds := []string{in.Target} - if in.Target == "both" { - kinds = []string{"base", "candidate"} - } - perRunTimeout, err := parseOptionalDuration(in.PerRunTimeout) - if err != nil { - return mcpserver.JobReference{}, fmt.Errorf("per_run_timeout: %w", err) - } - overallTimeout, err := parseOptionalDuration(in.OverallTimeout) - if err != nil { - return mcpserver.JobReference{}, fmt.Errorf("overall_timeout: %w", err) - } - sampleInterval, err := parseOptionalDuration(in.SampleInterval) - if err != nil { - return mcpserver.JobReference{}, fmt.Errorf("sample_interval: %w", err) - } - opts := cli.RepeatValidationOptions{ - Kinds: kinds, RunCount: in.RunCount, Concurrency: in.Concurrency, - PerRunTimeout: perRunTimeout, OverallTimeout: overallTimeout, SampleInterval: sampleInterval, Execute: true, - } - id, err := r.submitJob(ctx, "run_validation_group", in, func(ctx context.Context, report func(progress, statistics string) error) (any, error) { - if err := report("validation", jobProgressCounts(0, in.RunCount*len(kinds))); err != nil { - return nil, err - } - result, err := r.RunValidationGroup(ctx, in.ID, opts) - if err != nil { - return nil, err - } - if err := report("validation", jobProgressCounts(result.CompletedRuns, result.RequestedRuns)); err != nil { - return nil, err - } - return result, nil - }) - if err != nil { - return mcpserver.JobReference{}, err - } - return queuedJobReference(id, "run_validation_group", "repeat validation job started"), nil -} - -func parseOptionalDuration(value string) (time.Duration, error) { - if strings.TrimSpace(value) == "" { - return 0, nil - } - return time.ParseDuration(value) -} - // StartInvestigation creates a new investigation workspace. func (r *MCPReader) StartInvestigation(ctx context.Context, in mcpserver.StartInvestigationInput) (mcpserver.InvestigationOutput, error) { if in.Number > 0 { diff --git a/internal/app/repeated_validation_mcp.go b/internal/app/repeated_validation_mcp.go new file mode 100644 index 0000000..cf49c4b --- /dev/null +++ b/internal/app/repeated_validation_mcp.go @@ -0,0 +1,72 @@ +package app + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/morluto/gitcontribute/internal/cli" + "github.com/morluto/gitcontribute/internal/mcpserver" +) + +// RunRepeatedValidation submits a durable repeat/stress validation group. +func (r *MCPReader) RunRepeatedValidation(ctx context.Context, in mcpserver.RunRepeatedValidationInput) (mcpserver.JobReference, error) { + if !in.Execute { + return mcpserver.JobReference{}, errors.New("execute must be true to authorize host command execution") + } + if in.RunCount == 0 { + in.RunCount = 3 + } + if in.Concurrency == 0 { + in.Concurrency = 1 + } + if in.SampleInterval == "" { + in.SampleInterval = "100ms" + } + kinds := []string{in.Target} + if in.Target == "both" { + kinds = []string{"base", "candidate"} + } + perRunTimeout, err := parseOptionalDuration(in.PerRunTimeout) + if err != nil { + return mcpserver.JobReference{}, fmt.Errorf("per_run_timeout: %w", err) + } + overallTimeout, err := parseOptionalDuration(in.OverallTimeout) + if err != nil { + return mcpserver.JobReference{}, fmt.Errorf("overall_timeout: %w", err) + } + sampleInterval, err := parseOptionalDuration(in.SampleInterval) + if err != nil { + return mcpserver.JobReference{}, fmt.Errorf("sample_interval: %w", err) + } + opts := cli.RepeatValidationOptions{ + Kinds: kinds, RunCount: in.RunCount, Concurrency: in.Concurrency, + PerRunTimeout: perRunTimeout, OverallTimeout: overallTimeout, SampleInterval: sampleInterval, Execute: true, + } + id, err := r.submitJob(ctx, "run_validation_group", in, func(ctx context.Context, report func(progress, statistics string) error) (any, error) { + if err := report("validation", jobProgressCounts(0, in.RunCount*len(kinds))); err != nil { + return nil, err + } + result, err := r.RunValidationGroup(ctx, in.ID, opts) + if err != nil { + return nil, err + } + if err := report("validation", jobProgressCounts(result.CompletedRuns, result.RequestedRuns)); err != nil { + return nil, err + } + return result, nil + }) + if err != nil { + return mcpserver.JobReference{}, err + } + return queuedJobReference(id, "run_validation_group", "repeat validation job started"), nil +} + +func parseOptionalDuration(value string) (time.Duration, error) { + if strings.TrimSpace(value) == "" { + return 0, nil + } + return time.ParseDuration(value) +} From 0a7cbf789d1b3c9d1a437245b5566cf0e2465c30 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 23 Jul 2026 06:49:20 +0000 Subject: [PATCH 6/7] chore(deps): normalize merged checksums --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 83bf473..f3f4694 100644 --- a/go.sum +++ b/go.sum @@ -140,10 +140,10 @@ github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfv github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= -github.com/sourcegraph/go-diff v0.8.0 h1:ipIyu4cTsLbIrln4l0qtHA3r0a7gyK4ntKjtQytHhvY= -github.com/sourcegraph/go-diff v0.8.0/go.mod h1:hWlcO7Al+UZStZAP8rBumHpCK5ZHQ5BXsMls8p4+F5E= github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sourcegraph/go-diff v0.8.0 h1:ipIyu4cTsLbIrln4l0qtHA3r0a7gyK4ntKjtQytHhvY= +github.com/sourcegraph/go-diff v0.8.0/go.mod h1:hWlcO7Al+UZStZAP8rBumHpCK5ZHQ5BXsMls8p4+F5E= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= From 24526f69fa088ce4a0bbc5a4cbba3cead986aa12 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 23 Jul 2026 06:55:15 +0000 Subject: [PATCH 7/7] test(mcp): isolate repeat validation coverage --- internal/mcpserver/server_test.go | 16 ---------------- internal/mcpserver/validation_v1_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 16 deletions(-) create mode 100644 internal/mcpserver/validation_v1_test.go diff --git a/internal/mcpserver/server_test.go b/internal/mcpserver/server_test.go index e0af456..cf928b6 100644 --- a/internal/mcpserver/server_test.go +++ b/internal/mcpserver/server_test.go @@ -783,19 +783,3 @@ func TestV1ParityToolsAndResources(t *testing.T) { } } } - -func TestRepeatedValidationAppliesDocumentedDefaults(t *testing.T) { - fake := &fakeReader{searchStarted: make(chan struct{})} - client, closeSessions := connect(t, fake) - defer closeSessions() - result, err := client.CallTool(context.Background(), &mcp.CallToolParams{ - Name: ToolRunRepeatedValidation, - Arguments: map[string]any{"id": "val-1", "target": "candidate", "execute": true}, - }) - if err != nil || result.IsError { - t.Fatalf("call repeated validation: err=%v result=%+v", err, result) - } - if fake.repeatInput.RunCount != 3 || fake.repeatInput.Concurrency != 1 || fake.repeatInput.SampleInterval != "100ms" { - t.Fatalf("repeat defaults = %+v", fake.repeatInput) - } -} diff --git a/internal/mcpserver/validation_v1_test.go b/internal/mcpserver/validation_v1_test.go new file mode 100644 index 0000000..4f92bd0 --- /dev/null +++ b/internal/mcpserver/validation_v1_test.go @@ -0,0 +1,24 @@ +package mcpserver + +import ( + "context" + "testing" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +func TestRepeatedValidationAppliesDocumentedDefaults(t *testing.T) { + fake := &fakeReader{searchStarted: make(chan struct{})} + client, closeSessions := connect(t, fake) + defer closeSessions() + result, err := client.CallTool(context.Background(), &mcp.CallToolParams{ + Name: ToolRunRepeatedValidation, + Arguments: map[string]any{"id": "val-1", "target": "candidate", "execute": true}, + }) + if err != nil || result.IsError { + t.Fatalf("call repeated validation: err=%v result=%+v", err, result) + } + if fake.repeatInput.RunCount != 3 || fake.repeatInput.Concurrency != 1 || fake.repeatInput.SampleInterval != "100ms" { + t.Fatalf("repeat defaults = %+v", fake.repeatInput) + } +}