diff --git a/cmd/gortex/daemon.go b/cmd/gortex/daemon.go index f5bdd892f..a1ce087dd 100644 --- a/cmd/gortex/daemon.go +++ b/cmd/gortex/daemon.go @@ -412,6 +412,16 @@ func runDaemonStart(cmd *cobra.Command, _ []string) error { if stateCapture.graph != nil { out["graph_nodes"] = stateCapture.graph.NodeCount() out["graph_edges"] = stateCapture.graph.EdgeCount() + if r, ok := stateCapture.graph.(graph.DBStatReporter); ok { + dbBytes, walBytes := r.DBStats() + if dbBytes > 0 || walBytes > 0 { + out["db_bytes"] = dbBytes + out["wal_bytes"] = walBytes + if dbBytes > 0 { + out["wal_db_ratio"] = float64(walBytes) / float64(dbBytes) + } + } + } } return out }) diff --git a/cmd/gortex/daemon_state.go b/cmd/gortex/daemon_state.go index 502e7b073..01636c922 100644 --- a/cmd/gortex/daemon_state.go +++ b/cmd/gortex/daemon_state.go @@ -423,9 +423,12 @@ func warmupDaemonState(state *daemonState, logger *zap.Logger) *indexer.MultiWat logger.Info("daemon: warmup phase done", zap.String("phase", "parallel_parse"), zap.Duration("elapsed", time.Since(phaseStart))) + parseStats := progress.Stats(string(progress.PhaseParse), phaseStart, len(repos), len(repos)) publishReadinessPhase(state, "parallel_parse_done", false, map[string]any{ "tracked_repos": len(repos), "elapsed_ms": time.Since(phaseStart).Milliseconds(), + "elapsed_human": parseStats.Elapsed, + "repos_per_sec": parseStats.ItemsPerSec, }) // Warm-restart fast path. When the reconcile loop above re-indexed diff --git a/cmd/gortex/githook.go b/cmd/gortex/githook.go index 26ba9da19..76aded907 100644 --- a/cmd/gortex/githook.go +++ b/cmd/gortex/githook.go @@ -91,10 +91,10 @@ func init() { // SupportedHooks list rather than importing it so the CLI surface // stays decoupled from the install package's internals. func supportedHook(name string) error { - if name == "post-commit" || name == "post-merge" { + if name == "post-commit" || name == "post-merge" || name == "post-checkout" { return nil } - return fmt.Errorf("unsupported hook %q (supported: post-commit, post-merge)", name) + return fmt.Errorf("unsupported hook %q (supported: post-commit, post-merge, post-checkout)", name) } func runGithookInstall(cmd *cobra.Command, args []string) error { diff --git a/internal/config/config.go b/internal/config/config.go index 28c12fdfc..b3e854745 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -355,6 +355,15 @@ type Config struct { // re-include something an outer layer excluded. Exclude []string `mapstructure:"exclude" yaml:"exclude,omitempty"` + // Include force-indexes paths an outer layer (notably the repo + // .gitignore) excludes — the dedicated, readable counterpart to + // hand-writing `!pattern` negations in Exclude. Each entry is appended + // last as a gitignore `!pattern` re-include, so it wins over the + // builtin / .gitignore / global / repo exclude layers. Use it for a + // vendored or generated tree you nonetheless want in the graph + // (e.g. Pods/, a per-customer customer/ directory). + Include []string `mapstructure:"include" yaml:"include,omitempty"` + // RuleFiles are paths (or globs) to TOML files of user-defined // domain-extractor rules. Each rule is a tree-sitter pattern that // becomes a registered detector surfaced through `analyze diff --git a/internal/config/gitignore.go b/internal/config/gitignore.go index e362c1b10..6d0a9b501 100644 --- a/internal/config/gitignore.go +++ b/internal/config/gitignore.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "strings" + "unicode/utf8" ) // loadRepoGitignore reads the `.gitignore` file at the repo root and @@ -35,12 +36,25 @@ func loadRepoGitignore(repoPath string) []string { var patterns []string scanner := bufio.NewScanner(f) + // Tolerate pathologically long lines (a DLP-encrypted or otherwise + // non-text .gitignore) instead of aborting the whole read on the + // default 64 KiB scanner limit. + scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, "#") { continue } + // A non-UTF-8 line is not a valid gitignore pattern — feeding it to + // the matcher would mis-scan. Skip it defensively so a corrupt or + // encrypted .gitignore can never poison exclusion. + if !utf8.ValidString(line) { + continue + } patterns = append(patterns, line) } + // A read error (a still-too-long line, a transient I/O fault) must not + // discard the patterns already read — gitignore loading is best-effort, + // never a hard failure. return patterns } diff --git a/internal/config/gitignore_test.go b/internal/config/gitignore_test.go new file mode 100644 index 000000000..aeb0acb82 --- /dev/null +++ b/internal/config/gitignore_test.go @@ -0,0 +1,42 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLoadRepoGitignore_SkipsBlanksCommentsAndNonUTF8(t *testing.T) { + dir := t.TempDir() + // A valid file interleaved with a blank line, a comment, and a + // non-UTF-8 (e.g. DLP-encrypted) line that must be skipped, not fed to + // the matcher and not aborting the read. + content := "# comment\n\nnode_modules/\n" + string([]byte{0xff, 0xfe, 0xfd}) + "\n*.log\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(content), 0o644)) + + got := loadRepoGitignore(dir) + require.Equal(t, []string{"node_modules/", "*.log"}, got) +} + +func TestLoadRepoGitignore_LongLineDoesNotAbort(t *testing.T) { + dir := t.TempDir() + // A pathologically long line (larger than the default 64 KiB scanner + // buffer) must not discard the patterns that follow it. + long := make([]byte, 200*1024) + for i := range long { + long[i] = 'a' + } + content := "*.tmp\n" + string(long) + "\nbuild/\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(content), 0o644)) + + got := loadRepoGitignore(dir) + require.Contains(t, got, "*.tmp") + require.Contains(t, got, "build/") +} + +func TestLoadRepoGitignore_MissingFileIsNil(t *testing.T) { + require.Nil(t, loadRepoGitignore(t.TempDir())) + require.Nil(t, loadRepoGitignore("")) +} diff --git a/internal/config/manager.go b/internal/config/manager.go index 0896fc376..9b7d467ea 100644 --- a/internal/config/manager.go +++ b/internal/config/manager.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "go.uber.org/zap" @@ -229,6 +230,23 @@ func (cm *ConfigManager) EffectiveExclude(repoPrefix string) []string { out = append(out, ws.Watch.Exclude...) } } + + // Force-include last so it wins: each Include entry becomes a gitignore + // `!pattern` re-include over every exclude layer above (builtin, + // .gitignore, global, repo). This is the readable form of hand-writing + // negations, for a vendored/generated tree you want indexed anyway. + if ws != nil { + for _, inc := range ws.Include { + inc = strings.TrimSpace(inc) + if inc == "" { + continue + } + if !strings.HasPrefix(inc, "!") { + inc = "!" + inc + } + out = append(out, inc) + } + } return out } diff --git a/internal/config/manager_test.go b/internal/config/manager_test.go index 51961e733..6526af9f9 100644 --- a/internal/config/manager_test.go +++ b/internal/config/manager_test.go @@ -126,6 +126,25 @@ index: assert.Contains(t, got, "legacy/**", "legacy index.exclude must still be honoured") } +func TestEffectiveExclude_IncludeForceReincludesGitignored(t *testing.T) { + cm, err := NewConfigManager("/tmp/nonexistent-gortex-test-cm/config.yaml") + require.NoError(t, err) + + repoDir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".gitignore"), []byte("Pods/\n"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".gortex.yaml"), + []byte("include:\n - \"Pods/\"\n"), 0644)) + cm.LoadWorkspaceConfig("repo-inc", repoDir) + + got := cm.EffectiveExclude("repo-inc") + assert.Contains(t, got, "Pods/", "the .gitignore exclude is present") + assert.Equal(t, "!Pods/", got[len(got)-1], "the force-include negation lands last so it wins") + + // The compiled matcher must NOT exclude the force-included tree. + assert.False(t, excludes.New(got).MatchRel("Pods/"), + "a force-included gitignored dir must be re-included") +} + func TestEffectiveExclude_FallsBackToBuiltin(t *testing.T) { cm, err := NewConfigManager("/tmp/nonexistent-gortex-test-cm/config.yaml") require.NoError(t, err) diff --git a/internal/githooks/install.go b/internal/githooks/install.go index ce02cb5b6..1ee6fb98a 100644 --- a/internal/githooks/install.go +++ b/internal/githooks/install.go @@ -31,7 +31,7 @@ const ( // SupportedHooks enumerates the hook names that InstallHook accepts. // Anything else returns an error so we don't silently scatter our // markers into hooks we haven't audited. -var SupportedHooks = []string{"post-commit", "post-merge"} +var SupportedHooks = []string{"post-commit", "post-merge", "post-checkout"} func isSupportedHook(name string) bool { for _, h := range SupportedHooks { @@ -103,6 +103,17 @@ func (o InstallOpts) withDefaults() InstallOpts { // marker block. The body is a `#!/bin/sh` snippet that runs every // enabled action and tolerates failures so the hook always completes. func hookCommands(hook string, opts InstallOpts) []string { + if hook == "post-checkout" { + // post-checkout fires on branch switch / clone / file checkout. + // Touch the notify file so a running gortex daemon reconciles the + // new working-tree state immediately (sub-second) instead of waiting + // out its poll interval. Harmless when no daemon is running. + return []string{ + "# Force a running gortex daemon to reconcile after a checkout.", + "mkdir -p .gortex 2>/dev/null || true", + "touch .gortex/reindex.notify 2>/dev/null || true", + } + } var cmds []string cmds = append(cmds, fmt.Sprintf("# Auto-regenerate gortex artefacts on %s.", hook)) cmds = append(cmds, "# Failures are tolerated so the hook always completes.") diff --git a/internal/graph/index_state.go b/internal/graph/index_state.go new file mode 100644 index 000000000..bc0c9b229 --- /dev/null +++ b/internal/graph/index_state.go @@ -0,0 +1,46 @@ +package graph + +// RepoIndexState is the per-repo freshness provenance recorded at the +// end of a (re)index: the git revision the graph reflects, whether the +// working tree was dirty at index time, the Merkle workspace +// fingerprint that gates global-pass short-circuiting, node/edge counts +// for the index-plausibility baseline, and a JSON map of the +// per-language extractor versions that produced the graph. +// +// It is the storage half of the FreshnessFact layer; the per-file half +// lives in the Merkle leaf (the salted content hash) and the file_mtimes +// ledger. +type RepoIndexState struct { + RepoPrefix string + IndexedSHA string + Dirty bool + IndexedAt int64 // unix seconds + WorkspaceFP string // Merkle root at index time + NodeCount int + EdgeCount int + ExtractorVersions string // JSON-encoded map[string]int +} + +// RepoIndexStateWriter persists the freshness provenance for one repo. +// Backends without durable state simply do not implement it — the +// indexer type-asserts and skips the write when absent, exactly like the +// FileMtime ledger. +type RepoIndexStateWriter interface { + SetRepoIndexState(state RepoIndexState) error +} + +// RepoIndexStateReader reads back the freshness provenance for one repo. +// The bool is false when no state has been recorded yet (a never-indexed +// or pre-feature repo), which callers treat as "freshness unknown" — they +// never block on it. +type RepoIndexStateReader interface { + GetRepoIndexState(repoPrefix string) (RepoIndexState, bool, error) +} + +// DBStatReporter is an optional capability: report the on-disk size of the +// backing database file and its write-ahead log, in bytes. Surfaced in +// daemon_health so a runaway WAL high-water mark is observable. In-memory +// backends do not implement it. +type DBStatReporter interface { + DBStats() (dbBytes, walBytes int64) +} diff --git a/internal/graph/store_sqlite/schema.go b/internal/graph/store_sqlite/schema.go index 8c7a794de..1c8d7fc22 100644 --- a/internal/graph/store_sqlite/schema.go +++ b/internal/graph/store_sqlite/schema.go @@ -86,6 +86,24 @@ CREATE TABLE IF NOT EXISTS file_mtimes ( PRIMARY KEY (repo_prefix, file_path) ) WITHOUT ROWID; +-- repo_index_state records per-repo freshness provenance written at the +-- end of a (re)index: the git revision + dirty flag the graph reflects, +-- the Merkle workspace fingerprint (Tree.Root) that gates global-pass +-- short-circuiting, node/edge counts for the index-plausibility baseline, +-- and the JSON per-language extractor versions that produced the graph. +-- One row per repo_prefix; WITHOUT ROWID — the PK index IS the table, +-- like file_mtimes / clone_shingles. +CREATE TABLE IF NOT EXISTS repo_index_state ( + repo_prefix TEXT PRIMARY KEY, + indexed_sha TEXT NOT NULL DEFAULT '', + dirty INTEGER NOT NULL DEFAULT 0, + indexed_at INTEGER NOT NULL DEFAULT 0, + workspace_fp TEXT NOT NULL DEFAULT '', + node_count INTEGER NOT NULL DEFAULT 0, + edge_count INTEGER NOT NULL DEFAULT 0, + extractor_versions TEXT NOT NULL DEFAULT '' +) WITHOUT ROWID; + -- clone_shingles is the per-symbol MinHash shingle-set sidecar. Each -- function/method node's []uint64 shingle set is stored as a little- -- endian BLOB (8 bytes/elem) keyed by node_id so the maintained clone- diff --git a/internal/graph/store_sqlite/store.go b/internal/graph/store_sqlite/store.go index 2fd62bb4a..01e483ecf 100644 --- a/internal/graph/store_sqlite/store.go +++ b/internal/graph/store_sqlite/store.go @@ -42,6 +42,11 @@ import ( type Store struct { db *sql.DB + // dbPath is the on-disk SQLite file path, retained for size + // telemetry — the WAL high-water mark surfaces in daemon_health so a + // runaway -wal is observable rather than silently filling the disk. + dbPath string + // writeMu serialises every mutation. SQLite serialises writers // internally; doing the same on the Go side turns SQLITE_BUSY // contention into clean lock-wait and keeps the conformance @@ -182,7 +187,7 @@ func Open(path string) (*Store, error) { return nil, fmt.Errorf("sqlite edges_external index: %w", err) } - s := &Store{db: db} + s := &Store{db: db, dbPath: path} // Initialise the bundle cache at construction so its pointer is // never written after Open — concurrent SearchSymbolBundles reads // and SetBundleFingerprints writes then race only on the cache's diff --git a/internal/graph/store_sqlite/store_dbstat.go b/internal/graph/store_sqlite/store_dbstat.go new file mode 100644 index 000000000..b7019fd30 --- /dev/null +++ b/internal/graph/store_sqlite/store_dbstat.go @@ -0,0 +1,21 @@ +package store_sqlite + +import "os" + +// DBStats returns the on-disk size of the SQLite database file and its +// write-ahead log, in bytes. A missing file (or a store opened without a +// path) reports 0 for that component. Surfaced in daemon_health so a +// runaway WAL high-water mark is observable instead of silently filling +// the disk. +func (s *Store) DBStats() (dbBytes, walBytes int64) { + if s == nil || s.dbPath == "" { + return 0, 0 + } + if fi, err := os.Stat(s.dbPath); err == nil { + dbBytes = fi.Size() + } + if fi, err := os.Stat(s.dbPath + "-wal"); err == nil { + walBytes = fi.Size() + } + return dbBytes, walBytes +} diff --git a/internal/graph/store_sqlite/store_dbstat_test.go b/internal/graph/store_sqlite/store_dbstat_test.go new file mode 100644 index 000000000..598c8f488 --- /dev/null +++ b/internal/graph/store_sqlite/store_dbstat_test.go @@ -0,0 +1,33 @@ +package store_sqlite_test + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func TestDBStats(t *testing.T) { + path := filepath.Join(t.TempDir(), "g.sqlite") + s, err := store_sqlite.Open(path) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + + // Force a write so the DB (and WAL, in WAL mode) carry real bytes. + s.AddBatch([]*graph.Node{ + {ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go"}, + }, nil) + + dbBytes, walBytes := s.DBStats() + require.Greater(t, dbBytes, int64(0), "the on-disk DB file must have nonzero size") + require.GreaterOrEqual(t, walBytes, int64(0), "WAL size is non-negative (0 after a checkpoint)") + + // A store with no path (the zero value) reports zero, never panics. + var empty store_sqlite.Store + db, wal := empty.DBStats() + require.Equal(t, int64(0), db) + require.Equal(t, int64(0), wal) +} diff --git a/internal/graph/store_sqlite/store_index_state.go b/internal/graph/store_sqlite/store_index_state.go new file mode 100644 index 000000000..c45ae29d3 --- /dev/null +++ b/internal/graph/store_sqlite/store_index_state.go @@ -0,0 +1,45 @@ +package store_sqlite + +import ( + "database/sql" + + "github.com/zzet/gortex/internal/graph" +) + +// SetRepoIndexState upserts the freshness-provenance row for one repo — +// written at the end of every (re)index. One row per repo_prefix. +func (s *Store) SetRepoIndexState(st graph.RepoIndexState) error { + s.writeMu.Lock() + defer s.writeMu.Unlock() + dirty := 0 + if st.Dirty { + dirty = 1 + } + _, err := s.db.Exec(` +INSERT OR REPLACE INTO repo_index_state + (repo_prefix, indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions) +VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + st.RepoPrefix, st.IndexedSHA, dirty, st.IndexedAt, st.WorkspaceFP, + st.NodeCount, st.EdgeCount, st.ExtractorVersions) + return err +} + +// GetRepoIndexState returns the recorded freshness provenance for a repo. +// The bool is false when no row exists yet (never-indexed / pre-feature). +func (s *Store) GetRepoIndexState(repoPrefix string) (graph.RepoIndexState, bool, error) { + row := s.db.QueryRow(` +SELECT indexed_sha, dirty, indexed_at, workspace_fp, node_count, edge_count, extractor_versions + FROM repo_index_state WHERE repo_prefix = ?`, repoPrefix) + st := graph.RepoIndexState{RepoPrefix: repoPrefix} + var dirty int + err := row.Scan(&st.IndexedSHA, &dirty, &st.IndexedAt, &st.WorkspaceFP, + &st.NodeCount, &st.EdgeCount, &st.ExtractorVersions) + if err == sql.ErrNoRows { + return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, nil + } + if err != nil { + return graph.RepoIndexState{RepoPrefix: repoPrefix}, false, err + } + st.Dirty = dirty != 0 + return st, true, nil +} diff --git a/internal/graph/store_sqlite/store_index_state_test.go b/internal/graph/store_sqlite/store_index_state_test.go new file mode 100644 index 000000000..b21737ae7 --- /dev/null +++ b/internal/graph/store_sqlite/store_index_state_test.go @@ -0,0 +1,61 @@ +package store_sqlite_test + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" +) + +func openIndexStateStore(t *testing.T) *store_sqlite.Store { + t.Helper() + s, err := store_sqlite.Open(filepath.Join(t.TempDir(), "is.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + return s +} + +func TestRepoIndexState_RoundTrip(t *testing.T) { + s := openIndexStateStore(t) + + // Absent state reads back as (zero, false, nil). + got, ok, err := s.GetRepoIndexState("gortex") + require.NoError(t, err) + require.False(t, ok) + require.Equal(t, "gortex", got.RepoPrefix) + + want := graph.RepoIndexState{ + RepoPrefix: "gortex", + IndexedSHA: "abc123", + Dirty: true, + IndexedAt: 1700000000, + WorkspaceFP: "deadbeef", + NodeCount: 42, + EdgeCount: 99, + ExtractorVersions: `{"go":2}`, + } + require.NoError(t, s.SetRepoIndexState(want)) + + got, ok, err = s.GetRepoIndexState("gortex") + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, want, got) + + // Upsert replaces in place (one row per repo_prefix). + want.IndexedSHA = "def456" + want.Dirty = false + require.NoError(t, s.SetRepoIndexState(want)) + got, ok, err = s.GetRepoIndexState("gortex") + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, "def456", got.IndexedSHA) + require.False(t, got.Dirty) + + // A different repo is isolated. + _, ok, err = s.GetRepoIndexState("other") + require.NoError(t, err) + require.False(t, ok) +} diff --git a/internal/indexer/affected_set.go b/internal/indexer/affected_set.go index 31f35930f..2b312ae54 100644 --- a/internal/indexer/affected_set.go +++ b/internal/indexer/affected_set.go @@ -56,6 +56,25 @@ func (idx *Indexer) affectedTypeSet(graphPaths []string) (types, ifaces map[stri return types, ifaces } +// staleFilesAffectDerivedEdges reports whether any stale file carries code +// structure (functions / methods / types / fields / …) that the capability +// and framework-dispatch synthesizers derive edges from. When every stale +// file is non-code — a README, a JSON/YAML config, a data file — those +// whole-graph passes cannot produce or change any edge, so the caller skips +// them (the doc/config-edit fast path). Sound by construction: a file with +// no structural nodes contributes no calls / reads / writes / dispatch +// sites, which is the only input those synthesizers read. +func (idx *Indexer) staleFilesAffectDerivedEdges(staleFiles []string) bool { + for _, p := range idx.graphFilePaths(staleFiles) { + for _, n := range idx.graph.GetFileNodes(p) { + if n != nil && isStructuralKind(n.Kind) { + return true + } + } + } + return false +} + // runScopedInferencePasses runs the implements/override inference passes scoped // to the types/interfaces a set of stale files can affect. Returns false when // scoping is disabled (caller should run the full passes). When nothing diff --git a/internal/indexer/affected_set_test.go b/internal/indexer/affected_set_test.go new file mode 100644 index 000000000..119613559 --- /dev/null +++ b/internal/indexer/affected_set_test.go @@ -0,0 +1,32 @@ +package indexer + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" +) + +// TestStaleFilesAffectDerivedEdges verifies the doc/config fast-path guard: +// a code file affects the capability/dispatch synthesizers, a doc-only file +// does not, so a README edit can skip those whole-graph passes. +func TestStaleFilesAffectDerivedEdges(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "a.go"), "package a\n\nfunc A() {}\n") + writeFile(t, filepath.Join(dir, "README.md"), "# Title\n\nsome prose\n") + + g := graph.New() + idx := newTestIndexer(g) + idx.SetRootPath(dir) + _, err := idx.Index(dir) + require.NoError(t, err) + + require.True(t, idx.staleFilesAffectDerivedEdges([]string{filepath.Join(dir, "a.go")}), + "a .go file with a function must affect derived edges") + require.False(t, idx.staleFilesAffectDerivedEdges([]string{filepath.Join(dir, "README.md")}), + "a doc-only change must not affect capability/dispatch edges") + require.False(t, idx.staleFilesAffectDerivedEdges(nil), + "an empty stale set affects nothing") +} diff --git a/internal/indexer/crash_isolation.go b/internal/indexer/crash_isolation.go index 7fe7011f2..34e98697d 100644 --- a/internal/indexer/crash_isolation.go +++ b/internal/indexer/crash_isolation.go @@ -249,6 +249,7 @@ func quarantineResult(relPath, lang, reason string) *parser.ExtractionResult { FilePath: relPath, Language: lang, Meta: map[string]any{ + "skip_reason": "parse_panic", "parse_error": reason, "quarantined": true, }, diff --git a/internal/indexer/extractor_version.go b/internal/indexer/extractor_version.go new file mode 100644 index 000000000..912dba7d5 --- /dev/null +++ b/internal/indexer/extractor_version.go @@ -0,0 +1,112 @@ +package indexer + +import ( + "path/filepath" + "strconv" + "strings" +) + +// extractorVersions records the logic version of each language's +// extractor. Bump a language's entry when its extraction logic changes +// in a way that should re-extract already-indexed files whose content +// did not change (a new edge kind, a corrected node shape, a fixed +// parser bug). The version is mixed into the Merkle leaf salt (see +// merkleSaltFor), so a bump re-flags exactly that language's files as +// stale on the next reconcile — without re-reading unchanged content +// and without disturbing other languages. +// +// A language absent here, or pinned at 1, carries no salt and therefore +// behaves exactly as before: the registry is dormant until a version is +// deliberately raised. This is the surgical alternative to the +// binary-wide snapshot invalidation (which restages the whole repo on +// any binary change): a Go-extractor fix re-extracts only `.go` files. +var extractorVersions = map[string]int{ + // Languages default to version 1 (no salt). Raise an entry here in + // the same change that alters a language's extraction logic, e.g. + // "go": 2, +} + +// extractorSaltExtLang maps a lower-case file extension to the language +// key used in extractorVersions. It need not be exhaustive: an unmapped +// extension simply carries no extractor-version salt (content-only +// staleness, the pre-existing behaviour). Extensions are grouped to the +// extractor that owns them. +var extractorSaltExtLang = map[string]string{ + ".go": "go", + ".py": "python", + ".pyi": "python", + ".js": "javascript", + ".jsx": "javascript", + ".mjs": "javascript", + ".cjs": "javascript", + ".ts": "typescript", + ".tsx": "typescript", + ".mts": "typescript", + ".cts": "typescript", + ".java": "java", + ".rb": "ruby", + ".rs": "rust", + ".c": "c", + ".h": "c", + ".cc": "cpp", + ".cpp": "cpp", + ".cxx": "cpp", + ".hpp": "cpp", + ".hh": "cpp", + ".cs": "csharp", + ".php": "php", + ".swift": "swift", + ".kt": "kotlin", + ".kts": "kotlin", + ".scala": "scala", + ".m": "objc", + ".mm": "objcpp", + ".lua": "lua", + ".dart": "dart", + ".ex": "elixir", + ".exs": "elixir", + ".sh": "bash", + ".bash": "bash", +} + +// extractorVersionForLang returns the registered extractor version for a +// language, defaulting to 1. +func extractorVersionForLang(lang string) int { + if v, ok := extractorVersions[lang]; ok && v > 0 { + return v + } + return 1 +} + +// merkleSaltFor returns the Merkle leaf salt for a repo-relative path: +// "" when the file's language extractor is at the baseline version 1 +// (so the leaf equals the content hash and nothing changes), or +// "lang@N" once a language's extractor version is bumped, so its files +// re-extract on the next reconcile even when their content is unchanged. +func merkleSaltFor(rel string) string { + lang := extractorSaltExtLang[strings.ToLower(filepath.Ext(rel))] + if lang == "" { + return "" + } + v := extractorVersionForLang(lang) + if v <= 1 { + return "" + } + return lang + "@" + strconv.Itoa(v) +} + +// extractorVersionsSnapshot returns a copy of the current per-language +// extractor versions for persistence in repo_index_state, so a future +// reconcile can tell which extractor produced the stored graph. +func extractorVersionsSnapshot() map[string]int { + out := make(map[string]int, len(extractorSaltExtLang)) + seen := map[string]bool{} + for _, lang := range extractorSaltExtLang { + if seen[lang] { + continue + } + seen[lang] = true + out[lang] = extractorVersionForLang(lang) + } + return out +} diff --git a/internal/indexer/index_state.go b/internal/indexer/index_state.go new file mode 100644 index 000000000..3e14798ff --- /dev/null +++ b/internal/indexer/index_state.go @@ -0,0 +1,62 @@ +package indexer + +import ( + "context" + "encoding/json" + "time" + + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/gitcmd" + "github.com/zzet/gortex/internal/graph" +) + +// persistRepoIndexState records the per-repo freshness provenance at the +// end of a (re)index. diskTarget is the durable store when indexing +// streams to disk; nil falls back to idx.graph. Backends without durable +// state (the in-memory graph) do not implement RepoIndexStateWriter, so +// the write is skipped — exactly like the file-mtime ledger. +func (idx *Indexer) persistRepoIndexState(diskTarget graph.Store, rootAbs, workspaceFP string, nodes, edges int) { + target := graph.Store(idx.graph) + if diskTarget != nil { + target = diskTarget + } + w, ok := target.(graph.RepoIndexStateWriter) + if !ok { + return + } + sha, dirty := repoHeadAndDirty(rootAbs) + vers, _ := json.Marshal(extractorVersionsSnapshot()) + st := graph.RepoIndexState{ + RepoPrefix: idx.repoPrefix, + IndexedSHA: sha, + Dirty: dirty, + IndexedAt: time.Now().Unix(), + WorkspaceFP: workspaceFP, + NodeCount: nodes, + EdgeCount: edges, + ExtractorVersions: string(vers), + } + if err := w.SetRepoIndexState(st); err != nil { + idx.logger.Warn("persist repo index state failed", + zap.String("repo", idx.repoPrefix), zap.Error(err)) + } +} + +// repoHeadAndDirty returns the working tree's current commit SHA and +// whether it has uncommitted changes. Best-effort: a non-git directory or +// any git error yields ("", false) — freshness provenance never blocks +// indexing. Git shell-outs route through the shared concurrency limiter. +func repoHeadAndDirty(rootAbs string) (sha string, dirty bool) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + sha, err := gitcmd.Output(ctx, rootAbs, "rev-parse", "HEAD") + if err != nil { + return "", false + } + status, err := gitcmd.Output(ctx, rootAbs, "status", "--porcelain") + if err != nil { + return sha, false + } + return sha, status != "" +} diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 51258f78f..e2af547a5 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -1814,6 +1814,7 @@ func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexRes var skippedLarge int var skippedBytes int64 var skippedBySize []skippedFile + var parseFailedFiles []skippedFile err = filepath.WalkDir(absRoot, func(path string, d os.DirEntry, err error) error { if err != nil { return nil @@ -2189,6 +2190,16 @@ func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexRes errMu.Unlock() } if result == nil { + // A full-index parse failure that produced no nodes: + // record it for a skip-node post-pass so the file + // stays visible instead of vanishing. (The live-modify + // path never reaches here — it keeps a file's prior + // nodes through a transient parse failure.) + if err != nil { + errMu.Lock() + parseFailedFiles = append(parseFailedFiles, skippedFile{relPath: relPath, lang: lang, cause: err.Error()}) + errMu.Unlock() + } continue } if skipped && len(result.Nodes) > 0 { @@ -2330,6 +2341,7 @@ func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexRes // they stay visible in the graph with skip telemetry attached // instead of vanishing silently. idx.emitSizeSkipNodes(skippedBySize) + idx.emitParseFailedSkipNodes(parseFailedFiles) // Populate fileMtimes for all detected files. Keyed through // relKey so the mtime map agrees with the graph's file-node keys @@ -2593,16 +2605,18 @@ func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexRes // Persist the Merkle baseline so the next incremental pass diffs // against content hashes rather than re-indexing the whole repo. + workspaceFP := "" if idx.merkleEnabled() { paths := make([]string, len(files)) for i, wf := range files { paths[i] = wf.path } - idx.saveMerkleBaseline(absRoot, paths) + workspaceFP = idx.saveMerkleBaseline(absRoot, paths) } idx.indexGen.Add(1) // invalidate the trigram search cache nodes, edges := idx.repoNodeEdgeCount() + idx.persistRepoIndexState(diskTarget, absRoot, workspaceFP, nodes, edges) result = &IndexResult{ NodeCount: nodes, EdgeCount: edges, @@ -4021,11 +4035,16 @@ func (idx *Indexer) IncrementalReindexPaths(root string, paths []string) (*Index idx.resolver.InferImplements() idx.resolver.InferOverrides() } - // Keep capability edges (reads_env / executes_process / - // accesses_field) fresh on incremental reindex — same idempotent - // re-derive RunGlobalGraphPasses runs at full index. - synthesizeCapabilityEdges(idx.graph) - resolver.RunFrameworkSynthesizers(idx.graph) + // Capability (reads_env / executes_process / accesses_field) and + // framework-dispatch synthesis derive from code structure; skip them + // when the reconcile touched only non-code files (docs/config) and + // removed nothing — they cannot change any edge in that case, and + // eviction already handled any deletion. Same idempotent re-derive + // RunGlobalGraphPasses runs at full index. + if len(deletedFiles) > 0 || idx.staleFilesAffectDerivedEdges(staleFiles) { + synthesizeCapabilityEdges(idx.graph) + resolver.RunFrameworkSynthesizers(idx.graph) + } // Incremental: synthesize external calls only for the reindexed // files (O(edited files)), not a full-graph recompute. resolver.SynthesizeExternalCallsForFiles(idx.graph, idx.externalCallSynthesisEnabled(), idx.graphFilePaths(staleFiles)) @@ -6287,3 +6306,26 @@ func (idx *Indexer) IsStale(relPath string) bool { return info.ModTime().UnixNano() != storedMtime } + +// IsTrackedStale reports whether a file that IS in the index has changed +// on disk since it was indexed. Unlike IsStale it returns false for an +// untracked path (a new file, a non-source path, or a path-form +// mismatch), so a freshness signal never false-positives on a file the +// index legitimately does not cover. +func (idx *Indexer) IsTrackedStale(relPath string) bool { + relPath = pathkey.Normalize(filepath.ToSlash(relPath)) + + idx.mtimeMu.RLock() + storedMtime, ok := idx.fileMtimes[relPath] + idx.mtimeMu.RUnlock() + if !ok { + return false + } + + absPath := filepath.Join(idx.rootPath, filepath.FromSlash(relPath)) + info, err := os.Stat(absPath) + if err != nil { + return false + } + return info.ModTime().UnixNano() != storedMtime +} diff --git a/internal/indexer/is_tracked_stale_test.go b/internal/indexer/is_tracked_stale_test.go new file mode 100644 index 000000000..274bfd91a --- /dev/null +++ b/internal/indexer/is_tracked_stale_test.go @@ -0,0 +1,35 @@ +package indexer + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" +) + +func TestIsTrackedStale(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "a.go"), "package a\n\nfunc A() {}\n") + + g := graph.New() + idx := newTestIndexer(g) + idx.SetRootPath(dir) + _, err := idx.Index(dir) + require.NoError(t, err) + + // Freshly indexed: not stale. Unknown / untracked paths are never + // tracked-stale (the key difference from IsStale, which treats an + // unknown file as stale). + require.False(t, idx.IsTrackedStale("a.go")) + require.False(t, idx.IsTrackedStale("does_not_exist.go")) + require.False(t, idx.IsTrackedStale("untracked.md")) + + // Touch the file with a later mtime: now tracked-stale. + future := time.Now().Add(2 * time.Second) + require.NoError(t, os.Chtimes(filepath.Join(dir, "a.go"), future, future)) + require.True(t, idx.IsTrackedStale("a.go")) +} diff --git a/internal/indexer/merkle/merkle.go b/internal/indexer/merkle/merkle.go index 48fec5685..4eec809f0 100644 --- a/internal/indexer/merkle/merkle.go +++ b/internal/indexer/merkle/merkle.go @@ -26,6 +26,24 @@ import ( type FileNode struct { Hash string `json:"hash"` Mtime int64 `json:"mtime"` + // Salt is an opaque per-file discriminator mixed into the leaf + // hash. It carries the language extractor version, so a file whose + // content is unchanged but whose extractor logic was upgraded still + // diffs as changed and is re-extracted. An empty Salt reproduces the + // pre-salt leaf exactly, so adopting salts is zero-cost and fully + // back-compatible until a language's extractor version is bumped. + Salt string `json:"salt,omitempty"` +} + +// leaf is the value mixed into the parent directory's hash for this +// file. An empty Salt yields exactly the content hash, so trees built +// before salting — and files whose extractor is at the baseline +// version — hash identically. +func (n FileNode) leaf() string { + if n.Salt == "" { + return n.Hash + } + return n.Hash + "\x01" + n.Salt } // Tree is a content-addressed Merkle tree of one repository snapshot. @@ -46,7 +64,20 @@ type Tree struct { // not re-read, so a rebuild reads only the mtime-changed files. A file // that cannot be read is recorded with an empty hash so a diff always // flags it. -func Build(rootAbs string, relPaths []string, prior *Tree) *Tree { +// +// saltFor, when non-nil, returns a per-file salt (e.g. the language +// extractor version) mixed into the leaf hash. A salt change re-flags a +// file as changed even when its content and mtime did not move, so an +// extractor-logic upgrade re-extracts the affected files — without a +// re-read, since the content hash is reused. A nil saltFor (or one +// returning "") leaves every leaf equal to its content hash. +func Build(rootAbs string, relPaths []string, prior *Tree, saltFor func(rel string) string) *Tree { + salt := func(rel string) string { + if saltFor == nil { + return "" + } + return saltFor(rel) + } t := &Tree{ Files: make(map[string]FileNode, len(relPaths)), Dirs: make(map[string]string), @@ -56,22 +87,25 @@ func Build(rootAbs string, relPaths []string, prior *Tree) *Tree { abs := filepath.Join(rootAbs, filepath.FromSlash(rel)) info, err := os.Stat(abs) if err != nil { - t.Files[rel] = FileNode{} // unreadable — always treated as changed + t.Files[rel] = FileNode{Salt: salt(rel)} // unreadable — always treated as changed continue } mtime := info.ModTime().UnixNano() if prior != nil { if pn, ok := prior.Files[rel]; ok && pn.Mtime == mtime && pn.Hash != "" { - t.Files[rel] = pn // mtime unchanged — reuse the hash, skip the read + // mtime unchanged — reuse the content hash, skip the read, + // but re-stamp the salt so an extractor-version bump still + // surfaces in the leaf. + t.Files[rel] = FileNode{Hash: pn.Hash, Mtime: mtime, Salt: salt(rel)} continue } } h, err := hashFile(abs) if err != nil { - t.Files[rel] = FileNode{Mtime: mtime} + t.Files[rel] = FileNode{Mtime: mtime, Salt: salt(rel)} continue } - t.Files[rel] = FileNode{Hash: h, Mtime: mtime} + t.Files[rel] = FileNode{Hash: h, Mtime: mtime, Salt: salt(rel)} } t.aggregate() return t @@ -114,7 +148,7 @@ func (t *Tree) aggregate() { for rel, node := range t.Files { d := parentDir(rel) markDir(d) - files[d] = append(files[d], baseName(rel)+"\x00f"+node.Hash) + files[d] = append(files[d], baseName(rel)+"\x00f"+node.leaf()) } // Deepest directories first, so a directory's child-dir hashes are @@ -161,7 +195,7 @@ func (t *Tree) Diff(prior *Tree) (changed, removed []string) { } for rel, node := range t.Files { pn, ok := prior.Files[rel] - if !ok || pn.Hash != node.Hash || node.Hash == "" { + if !ok || pn.leaf() != node.leaf() || node.Hash == "" { changed = append(changed, rel) } } diff --git a/internal/indexer/merkle/merkle_test.go b/internal/indexer/merkle/merkle_test.go index df882401d..3b29becc0 100644 --- a/internal/indexer/merkle/merkle_test.go +++ b/internal/indexer/merkle/merkle_test.go @@ -32,7 +32,7 @@ func TestBuildAndDiff_DetectsContentChange(t *testing.T) { write(t, root, "pkg/util.go", "package pkg\n") files := []string{"main.go", "pkg/util.go"} - t1 := Build(root, files, nil) + t1 := Build(root, files, nil, nil) if len(t1.Files) != 2 { t.Fatalf("expected 2 files, got %d", len(t1.Files)) } @@ -40,7 +40,7 @@ func TestBuildAndDiff_DetectsContentChange(t *testing.T) { write(t, root, "main.go", "package main\n\nfunc main() {}\n") bumpMtime(t, filepath.Join(root, "main.go")) - t2 := Build(root, files, t1) + t2 := Build(root, files, t1, nil) changed, removed := t2.Diff(t1) if len(removed) != 0 { t.Errorf("no removals expected, got %v", removed) @@ -55,10 +55,10 @@ func TestDiff_MtimeTouchIsNotAChange(t *testing.T) { write(t, root, "a.go", "package a\n") files := []string{"a.go"} - t1 := Build(root, files, nil) + t1 := Build(root, files, nil, nil) bumpMtime(t, filepath.Join(root, "a.go")) // touch: new mtime, same content - t2 := Build(root, files, t1) + t2 := Build(root, files, t1, nil) changed, _ := t2.Diff(t1) if len(changed) != 0 { t.Errorf("a touch with unchanged content must not be a change, got %v", changed) @@ -72,10 +72,10 @@ func TestDiff_AddAndRemove(t *testing.T) { root := t.TempDir() write(t, root, "keep.go", "package main\n") write(t, root, "gone.go", "package main\n") - t1 := Build(root, []string{"keep.go", "gone.go"}, nil) + t1 := Build(root, []string{"keep.go", "gone.go"}, nil, nil) write(t, root, "new.go", "package main\n") - t2 := Build(root, []string{"keep.go", "new.go"}, t1) + t2 := Build(root, []string{"keep.go", "new.go"}, t1, nil) changed, removed := t2.Diff(t1) if len(changed) != 1 || changed[0] != "new.go" { @@ -90,7 +90,7 @@ func TestDiff_NilPriorYieldsEverything(t *testing.T) { root := t.TempDir() write(t, root, "a.go", "x") write(t, root, "b.go", "y") - t1 := Build(root, []string{"a.go", "b.go"}, nil) + t1 := Build(root, []string{"a.go", "b.go"}, nil, nil) changed, removed := t1.Diff(nil) if len(changed) != 2 { @@ -106,11 +106,11 @@ func TestSubtreeChanged(t *testing.T) { write(t, root, "a/x.go", "package a\n") write(t, root, "b/y.go", "package b\n") files := []string{"a/x.go", "b/y.go"} - t1 := Build(root, files, nil) + t1 := Build(root, files, nil, nil) write(t, root, "a/x.go", "package a\n\nfunc X() {}\n") bumpMtime(t, filepath.Join(root, "a", "x.go")) - t2 := Build(root, files, t1) + t2 := Build(root, files, t1, nil) if !t2.SubtreeChanged("a", t1) { t.Error("subtree a changed and must report so") @@ -123,7 +123,7 @@ func TestSubtreeChanged(t *testing.T) { func TestSaveLoad_RoundTrip(t *testing.T) { root := t.TempDir() write(t, root, "main.go", "package main\n") - t1 := Build(root, []string{"main.go"}, nil) + t1 := Build(root, []string{"main.go"}, nil, nil) path := filepath.Join(t.TempDir(), "nested", "merkle.json") if err := t1.Save(path); err != nil { @@ -152,3 +152,67 @@ func TestLoad_MissingFile(t *testing.T) { t.Error("missing file must yield a nil tree") } } + +func TestDiff_SaltChangeReExtractsOnlyThatLanguage(t *testing.T) { + root := t.TempDir() + write(t, root, "a.go", "package a\n") + write(t, root, "b.py", "x = 1\n") + files := []string{"a.go", "b.py"} + + // Baseline: the .go file is salted at version 1, .py carries no salt. + saltV1 := func(rel string) string { + if filepath.Ext(rel) == ".go" { + return "go@1" + } + return "" + } + t1 := Build(root, files, nil, saltV1) + + // Bump only the .go extractor version; file content and mtime are + // untouched, so the only signal is the salt. + saltV2 := func(rel string) string { + if filepath.Ext(rel) == ".go" { + return "go@2" + } + return "" + } + t2 := Build(root, files, t1, saltV2) + + changed, removed := t2.Diff(t1) + if len(removed) != 0 { + t.Errorf("no removals expected, got %v", removed) + } + if len(changed) != 1 || changed[0] != "a.go" { + t.Errorf("a salt bump must re-flag only the salted language, changed = %v, want [a.go]", changed) + } + if t2.Root == t1.Root { + t.Error("root must move when a file's salt changes") + } + // The content hash was reused (no re-read) even though the leaf moved. + if t2.Files["a.go"].Hash != t1.Files["a.go"].Hash { + t.Error("content hash must be reused across a salt-only change") + } +} + +func TestSalt_EmptyEqualsLegacyContentOnly(t *testing.T) { + root := t.TempDir() + write(t, root, "a.go", "package a\n") + files := []string{"a.go"} + + // An empty-salt build, a nil-saltFor build, and the legacy + // content-only tree must all produce the same root — adopting salts + // costs nothing until a version is actually bumped. + emptySalt := func(string) string { return "" } + withEmpty := Build(root, files, nil, emptySalt) + withNil := Build(root, files, nil, nil) + if withEmpty.Root != withNil.Root { + t.Errorf("empty salt must equal nil saltFor: %s vs %s", withEmpty.Root, withNil.Root) + } + if withEmpty.Files["a.go"].Salt != "" { + t.Errorf("empty salt must be stored empty, got %q", withEmpty.Files["a.go"].Salt) + } + changed, _ := withEmpty.Diff(withNil) + if len(changed) != 0 { + t.Errorf("empty-salt and nil-salt trees must diff clean, got %v", changed) + } +} diff --git a/internal/indexer/merkle_reindex.go b/internal/indexer/merkle_reindex.go index 98604eef8..c9875d0ae 100644 --- a/internal/indexer/merkle_reindex.go +++ b/internal/indexer/merkle_reindex.go @@ -40,7 +40,7 @@ func (idx *Indexer) merkleStaleFiles(rootAbs string, diskFiles map[string]bool) } treePath := merkleTreeFile(rootAbs) prior, _ := merkle.Load(treePath) - tree := merkle.Build(rootAbs, rels, prior) + tree := merkle.Build(rootAbs, rels, prior, merkleSaltFor) changed, _ := tree.Diff(prior) if err := tree.Save(treePath); err != nil { idx.logger.Warn("indexer: merkle tree save failed", zap.Error(err)) @@ -54,16 +54,19 @@ func (idx *Indexer) merkleStaleFiles(rootAbs string, diskFiles map[string]bool) // saveMerkleBaseline builds and persists the Merkle tree after a full // index, so the next incremental pass diffs against a content-addressed -// baseline rather than treating every file as changed. -func (idx *Indexer) saveMerkleBaseline(rootAbs string, absFiles []string) { +// baseline rather than treating every file as changed. It returns the +// tree root — the workspace fingerprint recorded in repo_index_state and +// used to short-circuit the global derivation passes when nothing moved. +func (idx *Indexer) saveMerkleBaseline(rootAbs string, absFiles []string) string { rels := make([]string, 0, len(absFiles)) for _, f := range absFiles { if rel, err := filepath.Rel(rootAbs, f); err == nil { rels = append(rels, filepath.ToSlash(rel)) } } - tree := merkle.Build(rootAbs, rels, nil) + tree := merkle.Build(rootAbs, rels, nil, merkleSaltFor) if err := tree.Save(merkleTreeFile(rootAbs)); err != nil { idx.logger.Warn("indexer: merkle baseline save failed", zap.Error(err)) } + return tree.Root } diff --git a/internal/indexer/poller.go b/internal/indexer/poller.go index 2d9c618df..4f9e99bd5 100644 --- a/internal/indexer/poller.go +++ b/internal/indexer/poller.go @@ -4,6 +4,7 @@ import ( "context" "os" "path/filepath" + "strings" "sync" "time" @@ -36,7 +37,14 @@ type Poller struct { interval time.Duration done chan struct{} - stopped chan struct{} + wg sync.WaitGroup + + // notifyPath is the sentinel file an external trigger (a post-checkout + // git hook, or an agent that just wrote files) touches to force an + // immediate reconcile; notifyMtime is the last mtime the fast notify + // loop observed. Empty notifyPath disables the fast loop. + notifyPath string + notifyMtime int64 mu sync.Mutex lastSHA string @@ -115,10 +123,10 @@ func newPoller(w *Watcher, idx *Indexer, logger *zap.Logger) *Poller { indexer: idx, rootPath: root, logger: logger, - interval: pollInterval(nodeCount), - lastSHA: lastSHA, - done: make(chan struct{}), - stopped: make(chan struct{}), + interval: pollInterval(nodeCount), + lastSHA: lastSHA, + done: make(chan struct{}), + notifyPath: notifyFilePath(root), } } @@ -133,7 +141,19 @@ func (p *Poller) Start() { p.mu.Lock() p.loopStarted = true p.mu.Unlock() + p.wg.Add(1) go p.loop() + if p.notifyPath != "" { + // Seed the baseline mtime so the first observed change is a real + // touch, not the file's pre-existing state at startup. + if info, err := os.Stat(p.notifyPath); err == nil { + p.mu.Lock() + p.notifyMtime = info.ModTime().UnixNano() + p.mu.Unlock() + } + p.wg.Add(1) + go p.notifyLoop() + } } // Stop halts the poller. Idempotent — safe whether Start launched the @@ -151,12 +171,12 @@ func (p *Poller) Stop() { } close(p.done) if started { - <-p.stopped + p.wg.Wait() } } func (p *Poller) loop() { - defer close(p.stopped) + defer p.wg.Done() t := time.NewTicker(p.interval) defer t.Stop() if p.logger != nil { @@ -327,3 +347,58 @@ func pollerDiffNameStatus(ctx context.Context, repoPath, oldSHA, newSHA string) } return parseDiffNameStatus(out), nil } + +// notifyFileInterval is the fast poll cadence for the agent-triggered +// notify file — tight enough for sub-second re-index latency, cheap enough +// (one stat per tick) to run alongside the adaptive poller. +const notifyFileInterval = 250 * time.Millisecond + +// notifyFilePath resolves the sentinel file an external trigger (a +// post-checkout hook, or an agent that just wrote files) touches to force +// an immediate reconcile. GORTEX_NOTIFY_FILE overrides the per-repo default +// of /.gortex/reindex.notify. +func notifyFilePath(root string) string { + if env := strings.TrimSpace(os.Getenv("GORTEX_NOTIFY_FILE")); env != "" { + return env + } + if root == "" { + return "" + } + return filepath.Join(root, ".gortex", "reindex.notify") +} + +// notifyLoop watches the notify file's mtime on a tight cadence and runs a +// full poll cycle (HEAD + filesystem sweep) the instant it is touched, so an +// agent or git hook can force a sub-second reconcile instead of waiting out +// the adaptive interval. A missing file is a cheap no-op until it appears; +// the file's first sighting only seeds the baseline (it is not itself a +// trigger). +func (p *Poller) notifyLoop() { + defer p.wg.Done() + t := time.NewTicker(notifyFileInterval) + defer t.Stop() + for { + select { + case <-p.done: + return + case <-t.C: + info, err := os.Stat(p.notifyPath) + if err != nil { + continue + } + mt := info.ModTime().UnixNano() + p.mu.Lock() + prev := p.notifyMtime + p.notifyMtime = mt + p.mu.Unlock() + if prev == 0 || mt == prev { + continue + } + if p.logger != nil { + p.logger.Debug("watcher: notify file touched; forcing reconcile", + zap.String("notify", p.notifyPath)) + } + p.poll() + } + } +} diff --git a/internal/indexer/poller_test.go b/internal/indexer/poller_test.go index e04caf347..bfd232787 100644 --- a/internal/indexer/poller_test.go +++ b/internal/indexer/poller_test.go @@ -263,7 +263,7 @@ func TestPoller_StartedAndStoppedWithWatcher(t *testing.T) { // root), or Stop was already called once. func TestPoller_StopIdempotent(t *testing.T) { // Inert poller — no indexer, Start is a no-op. - inert := &Poller{done: make(chan struct{}), stopped: make(chan struct{})} + inert := &Poller{done: make(chan struct{})} inert.Start() inert.Stop() inert.Stop() // second call must not panic or block @@ -295,6 +295,44 @@ func TestPoller_StopIdempotent(t *testing.T) { } } +// TestPoller_NotifyFileTriggersReconcile verifies touching the notify file +// forces an immediate reconcile via the fast notify loop, independent of the +// (much longer) adaptive poll interval. +func TestPoller_NotifyFileTriggersReconcile(t *testing.T) { + dir := t.TempDir() + writeTestFile(t, filepath.Join(dir, "main.go"), "package main\n\nfunc Main() {}\n") + + g := graph.New() + idx := newTestIndexer(g) + idx.SetRootPath(dir) + _, err := idx.Index(dir) + require.NoError(t, err) + + w, err := NewWatcher(idx, config.WatchConfig{Enabled: true, DebounceMs: 10}, zap.NewNop()) + require.NoError(t, err) + p := newPoller(w, idx, zap.NewNop()) + require.NotEmpty(t, p.notifyPath, "notify path must default under .gortex") + require.NoError(t, os.MkdirAll(filepath.Dir(p.notifyPath), 0o755)) + require.NoError(t, os.WriteFile(p.notifyPath, []byte("x"), 0o644)) + + swept := make(chan int, 8) + p.swept = func(n int) { swept <- n } + p.Start() + defer p.Stop() + + // Touch the notify file with a strictly later mtime. + time.Sleep(20 * time.Millisecond) + future := time.Now().Add(2 * time.Second) + require.NoError(t, os.Chtimes(p.notifyPath, future, future)) + + select { + case <-swept: + // the notify loop forced a reconcile within the fast cadence + case <-time.After(3 * time.Second): + t.Fatal("notify-file touch did not trigger a reconcile within 3s") + } +} + // TestPoller_SweepHookReportsWork exercises the swept test hook and // confirms a poll cycle reports the number of files it re-dispatched // — the same count the production logger emits. diff --git a/internal/indexer/skip_telemetry.go b/internal/indexer/skip_telemetry.go index 28b2aaf9e..db51609f7 100644 --- a/internal/indexer/skip_telemetry.go +++ b/internal/indexer/skip_telemetry.go @@ -43,12 +43,15 @@ func safeExtract(ext parser.Extractor, relPath string, src []byte) (result *pars return ext.Extract(relPath, src) } -// skippedFile records a file dropped by the size cap, kept so a -// synthetic telemetry node can be emitted after the parse pass. +// skippedFile records a file dropped by the size cap or a full-index +// parse failure, kept so a synthetic telemetry node can be emitted after +// the parse pass. cause carries the extraction error for parse failures +// (empty for size skips). type skippedFile struct { relPath string lang string size int64 + cause string } // walkedFile records a file that survived the walk-time filters, @@ -104,6 +107,7 @@ func timeoutSkipResult(relPath, lang string, budgetMS int) *parser.ExtractionRes FilePath: relPath, Language: lang, Meta: map[string]any{ + "skip_reason": "timeout", "skipped_due_to_timeout": true, "extract_budget_ms": budgetMS, }, @@ -124,6 +128,7 @@ func minifiedSkipResult(relPath, lang, reason string) *parser.ExtractionResult { FilePath: relPath, Language: lang, Meta: map[string]any{ + "skip_reason": "minified", "skipped_due_to_minified": true, "minified_reason": reason, }, @@ -141,6 +146,7 @@ func sizeSkipNode(sf skippedFile, maxSize int64) *graph.Node { FilePath: sf.relPath, Language: sf.lang, Meta: map[string]any{ + "skip_reason": "size", "skipped_due_to_size": true, "file_size_bytes": sf.size, "max_file_size_bytes": maxSize, @@ -148,6 +154,34 @@ func sizeSkipNode(sf skippedFile, maxSize int64) *graph.Node { } } +// parseFailedSkipResult builds a synthetic single-node result for a file +// whose extractor returned an error that did not panic or time out — an +// ordinary parse failure that would otherwise be dropped silently. +// Keeping the file in the graph as a skip node makes "why is this symbol +// missing" answerable (index_health rolls the skip reasons up), and the +// Merkle reconcile retries it automatically once its content changes or +// its language's extractor version is bumped — so no separate retry +// ledger is needed. +func parseFailedSkipResult(relPath, lang string, cause error) *parser.ExtractionResult { + reason := "parse failed" + if cause != nil { + reason = cause.Error() + } + return &parser.ExtractionResult{ + Nodes: []*graph.Node{{ + ID: relPath, + Kind: graph.KindFile, + Name: filepath.Base(relPath), + FilePath: relPath, + Language: lang, + Meta: map[string]any{ + "skip_reason": "parse_failed", + "parse_error": reason, + }, + }}, + } +} + // emitSizeSkipNodes adds synthetic file nodes for every size-skipped // file so the file stays visible in the graph (queryable, // get_file_summary works) with skip telemetry instead of vanishing. @@ -163,3 +197,27 @@ func (idx *Indexer) emitSizeSkipNodes(skipped []skippedFile) { idx.applyRepoPrefix(nodes, nil) idx.graph.AddBatch(nodes, nil) } + +// emitParseFailedSkipNodes adds a synthetic file node for every file that +// failed extraction during a FULL index and produced no nodes, so the +// file stays visible (index_health rolls it up under skip_reason +// "parse_failed") instead of vanishing silently. Only the full-index path +// uses this: the live-modify path deliberately keeps a file's prior nodes +// through a transient mid-edit parse failure, so it must never be fed +// here. The Merkle reconcile retries a failed file when its content or +// extractor version changes. +func (idx *Indexer) emitParseFailedSkipNodes(failed []skippedFile) { + if len(failed) == 0 { + return + } + nodes := make([]*graph.Node, 0, len(failed)) + for _, sf := range failed { + var cause error + if sf.cause != "" { + cause = errors.New(sf.cause) + } + nodes = append(nodes, parseFailedSkipResult(sf.relPath, sf.lang, cause).Nodes...) + } + idx.applyRepoPrefix(nodes, nil) + idx.graph.AddBatch(nodes, nil) +} diff --git a/internal/indexer/skip_telemetry_test.go b/internal/indexer/skip_telemetry_test.go index 90d189dea..d5a3f139e 100644 --- a/internal/indexer/skip_telemetry_test.go +++ b/internal/indexer/skip_telemetry_test.go @@ -1,6 +1,7 @@ package indexer import ( + "errors" "path/filepath" "strings" "testing" @@ -130,3 +131,57 @@ func TestIndexFile_SizeSkip(t *testing.T) { require.NotNil(t, n) require.Equal(t, true, n.Meta["skipped_due_to_size"]) } + +// TestSkipNodes_CarryUnifiedReason verifies every skip-node shape stamps a +// uniform skip_reason so index_health can roll them up by reason. +func TestSkipNodes_CarryUnifiedReason(t *testing.T) { + cases := []struct { + name string + node *graph.Node + reason string + }{ + {"size", sizeSkipNode(skippedFile{relPath: "big.go", lang: "go", size: 1 << 20}, 1024), "size"}, + {"timeout", timeoutSkipResult("slow.go", "go", 500).Nodes[0], "timeout"}, + {"minified", minifiedSkipResult("bundle.js", "javascript", "long-lines").Nodes[0], "minified"}, + {"parse_failed", parseFailedSkipResult("bad.go", "go", errors.New("boom")).Nodes[0], "parse_failed"}, + {"parse_panic", quarantineResult("crash.go", "go", "panic").Nodes[0], "parse_panic"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + require.Equal(t, graph.KindFile, c.node.Kind) + require.Equal(t, c.reason, c.node.Meta["skip_reason"]) + }) + } +} + +func TestParseFailedSkipResult_RecordsError(t *testing.T) { + r := parseFailedSkipResult("bad.go", "go", errors.New("unexpected token")) + require.Len(t, r.Nodes, 1) + n := r.Nodes[0] + require.Equal(t, "bad.go", n.FilePath) + require.Equal(t, "bad.go", n.Name) + require.Equal(t, "parse_failed", n.Meta["skip_reason"]) + require.Equal(t, "unexpected token", n.Meta["parse_error"]) +} + +// TestIndex_ParseFailedSkipTelemetry verifies a file that fails to parse +// during a FULL index stays visible as a parse_failed skip node instead of +// vanishing — the safe counterpart to the live-modify path, which keeps a +// file's prior nodes through a transient failure (see +// TestPatchGraphModify_ParseFailureKeepsPriorNodes). +func TestIndex_ParseFailedSkipTelemetry(t *testing.T) { + idx, ext := newToggleIndexer(t) + ext.setFail(true) // every extraction returns an error + + dir := t.TempDir() + idx.SetRootPath(dir) + writeFile(t, filepath.Join(dir, "broken.fk"), "this does not parse") + + _, err := idx.Index(dir) + require.NoError(t, err) + + n := idx.graph.GetNode("broken.fk") + require.NotNil(t, n, "a full-index parse failure must leave a visible skip node") + require.Equal(t, graph.KindFile, n.Kind) + require.Equal(t, "parse_failed", n.Meta["skip_reason"]) +} diff --git a/internal/indexer/slow_mount_linux.go b/internal/indexer/slow_mount_linux.go new file mode 100644 index 000000000..738cccd1c --- /dev/null +++ b/internal/indexer/slow_mount_linux.go @@ -0,0 +1,45 @@ +//go:build linux + +package indexer + +import ( + "os" + "strings" + "syscall" +) + +// slowWatchMount reports whether path lives on a filesystem where native +// fsnotify is unreliable or prohibitively slow — notably a Windows drive +// surfaced into WSL2 via 9p/drvfs (inotify events arrive late or never), or +// an SMB/CIFS share. On such a mount the watcher disables fsnotify and +// relies on the adaptive poller + git hooks. GORTEX_FORCE_FSNOTIFY=1 forces +// native fsnotify on regardless. +func slowWatchMount(path string) bool { + if path == "" || os.Getenv("GORTEX_FORCE_FSNOTIFY") == "1" { + return false + } + if !runningUnderWSL() { + return false + } + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return false + } + switch int64(st.Type) { + case 0x01021997, // V9FS_MAGIC — 9p, WSL2's drvfs transport for Windows drives + 0xFF534D42: // CIFS_MAGIC — SMB/CIFS share + return true + } + return false +} + +// runningUnderWSL reports whether the process is inside the Windows +// Subsystem for Linux, probed from /proc/version's microsoft/WSL marker. +func runningUnderWSL() bool { + b, err := os.ReadFile("/proc/version") + if err != nil { + return false + } + v := strings.ToLower(string(b)) + return strings.Contains(v, "microsoft") || strings.Contains(v, "wsl") +} diff --git a/internal/indexer/slow_mount_other.go b/internal/indexer/slow_mount_other.go new file mode 100644 index 000000000..d07e2d8be --- /dev/null +++ b/internal/indexer/slow_mount_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package indexer + +// slowWatchMount is a Linux/WSL2 concern (9p/drvfs/SMB mounts where native +// fsnotify is unreliable). On other platforms native fsnotify is used as-is. +func slowWatchMount(string) bool { return false } diff --git a/internal/indexer/slow_mount_test.go b/internal/indexer/slow_mount_test.go new file mode 100644 index 000000000..d871a9e7b --- /dev/null +++ b/internal/indexer/slow_mount_test.go @@ -0,0 +1,15 @@ +package indexer + +import "testing" + +// TestSlowWatchMount_NormalMountNotDegraded guards the safe default: a +// normal local filesystem (the test temp dir) must never be flagged as a +// slow mount, so fsnotify is only disabled on a genuine WSL2 9p/SMB mount. +func TestSlowWatchMount_NormalMountNotDegraded(t *testing.T) { + if slowWatchMount(t.TempDir()) { + t.Error("a normal local mount must not be flagged slow (fsnotify must stay enabled)") + } + if slowWatchMount("") { + t.Error("an empty path must not be flagged slow") + } +} diff --git a/internal/indexer/watcher.go b/internal/indexer/watcher.go index 187b37a95..9b0eb8dfb 100644 --- a/internal/indexer/watcher.go +++ b/internal/indexer/watcher.go @@ -54,7 +54,11 @@ type Watcher struct { fsw fswatcher.Watcher fsCancel context.CancelFunc config config.WatchConfig - excludes *excludes.Matcher + // degradedNoFsnotify is set when Start detected a slow mount (a WSL2 + // 9p/drvfs Windows drive, an SMB share) and skipped the native fsnotify + // backend, relying on the adaptive poller + git hooks instead. + degradedNoFsnotify bool + excludes *excludes.Matcher events chan GraphChangeEvent history []GraphChangeEvent historyMu sync.Mutex @@ -180,6 +184,27 @@ func (w *Watcher) Start(paths []string) error { if len(paths) == 0 { return errors.New("watcher: no paths to watch") } + + // WSL2 / slow-mount degradation: on a 9p/drvfs mount (a Windows drive + // under WSL2, an SMB share) native fsnotify delivers events late or not + // at all, and confirmWatchActive would hang ~5s per path before timing + // out. Skip the fsnotify backend entirely and rely on the adaptive + // poller + git hooks, which are mount-agnostic. The downstream code + // already tolerates a nil fsw. GORTEX_FORCE_FSNOTIFY=1 overrides. + if w.config.Enabled { + probe := paths[0] + if abs, err := filepath.Abs(probe); err == nil { + probe = abs + } + if slowWatchMount(probe) { + w.degradedNoFsnotify = true + w.logger.Warn("watcher: slow mount detected — disabling native fsnotify, using adaptive poller fallback", + zap.String("path", probe)) + w.poller = newPoller(w, w.indexer, w.logger) + w.poller.Start() + return nil + } + } ready := make(chan struct{}) // Own the events/dropped channels so the library never closes them on // teardown. fswatcher's shutdown closes its events channel while its @@ -366,7 +391,11 @@ func (w *Watcher) Stop() error { if w.fsw != nil { w.fsw.Close() } - <-w.stopped + // In slow-mount degraded mode the fsnotify loop never ran, so its + // stopped channel is never closed — don't block on it. + if !w.degradedNoFsnotify { + <-w.stopped + } return nil } diff --git a/internal/mcp/auto_index.go b/internal/mcp/auto_index.go new file mode 100644 index 000000000..685c9d8c2 --- /dev/null +++ b/internal/mcp/auto_index.go @@ -0,0 +1,95 @@ +package mcp + +import ( + "context" + "os" + "strings" + "time" + + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/config" + "github.com/zzet/gortex/internal/gitcmd" +) + +// autoIndexFileLimit bounds zero-config auto-indexing: a cwd repo with more +// git-tracked files than this is left for an explicit `gortex track`, so a +// stray tool call never silently chews through an enormous tree. +const autoIndexFileLimit = 25000 + +// maybeAutoIndexCWD lazily background-indexes the current working directory +// the first time a tool is called in a session, when the cwd is an untracked +// git repo. It is OFF by default and opt-in via GORTEX_AUTOINDEX=1 — the +// consent guardrail: auto-indexing is expensive, so the user asks for it. +// The request path pays only one getenv + a sync.Once check; all real work +// (git probing, the size bound, the index) runs on a background goroutine. +func (s *Server) maybeAutoIndexCWD() { + if s == nil || s.multiIndexer == nil { + return + } + if os.Getenv("GORTEX_AUTOINDEX") != "1" { + return + } + s.autoIndexOnce.Do(func() { go s.autoIndexCWDBackground() }) +} + +// autoIndexCWDBackground performs the bounded, consented auto-index off the +// request path: resolve the git root of the cwd, skip it when already +// covered or oversized, and otherwise track it. +func (s *Server) autoIndexCWDBackground() { + cwd, err := os.Getwd() + if err != nil || cwd == "" { + return + } + // Already covered by a tracked repo? nothing to do. + if s.multiIndexer.RepoForFile(cwd) != "" { + return + } + root := gitRepoRoot(cwd) + if root == "" || s.multiIndexer.RepoForFile(root) != "" { + return + } + // Bounded: a tree larger than the limit is left for an explicit track. + if n, over := repoFileCountOverLimit(root, autoIndexFileLimit); over { + if s.logger != nil { + s.logger.Info("auto-index: cwd repo exceeds the file-count limit; run `gortex track` to index it", + zap.String("root", root), zap.Int("limit", autoIndexFileLimit), zap.Int("at_least", n)) + } + return + } + if s.logger != nil { + s.logger.Info("auto-index: background-indexing untracked cwd", zap.String("root", root)) + } + if _, err := s.multiIndexer.TrackRepoCtx(context.Background(), config.RepoEntry{Path: root}); err != nil && s.logger != nil { + s.logger.Warn("auto-index: track failed", zap.String("root", root), zap.Error(err)) + } +} + +// gitRepoRoot resolves the git working-tree root containing dir, or "" when +// dir is not in a git repo. +func gitRepoRoot(dir string) string { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + out, err := gitcmd.Output(ctx, dir, "rev-parse", "--show-toplevel") + if err != nil { + return "" + } + return strings.TrimSpace(out) +} + +// repoFileCountOverLimit counts git-tracked files under root, reporting +// whether the count exceeds limit. A git error returns (0, false) — when we +// cannot measure the size we do not block (the indexer's own caps apply). +func repoFileCountOverLimit(root string, limit int) (int, bool) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + out, err := gitcmd.Output(ctx, root, "ls-files") + if err != nil { + return 0, false + } + n := strings.Count(out, "\n") + if strings.TrimSpace(out) != "" && !strings.HasSuffix(out, "\n") { + n++ // last line without a trailing newline + } + return n, n > limit +} diff --git a/internal/mcp/auto_index_test.go b/internal/mcp/auto_index_test.go new file mode 100644 index 000000000..be29b7fa9 --- /dev/null +++ b/internal/mcp/auto_index_test.go @@ -0,0 +1,48 @@ +package mcp + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func runGitT(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + require.NoError(t, cmd.Run(), "git %v", args) +} + +func TestAutoIndexHelpers(t *testing.T) { + dir := t.TempDir() + + // Not a git repo: no root, and ls-files errors so the size bound never + // blocks (returns not-over). + require.Equal(t, "", gitRepoRoot(dir)) + if _, over := repoFileCountOverLimit(dir, 0); over { + t.Error("a non-git dir must not be flagged over the limit") + } + + // Make it a git repo with two tracked files. + runGitT(t, dir, "init") + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.go"), []byte("package a\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.go"), []byte("package b\n"), 0o644)) + runGitT(t, dir, "add", ".") + + root := gitRepoRoot(dir) + require.NotEmpty(t, root) + rootResolved, _ := filepath.EvalSymlinks(root) + dirResolved, _ := filepath.EvalSymlinks(dir) + require.Equal(t, dirResolved, rootResolved, "git root must resolve to the repo dir") + + n, over := repoFileCountOverLimit(dir, 100) + require.Equal(t, 2, n, "two tracked files") + require.False(t, over) + + if _, over := repoFileCountOverLimit(dir, 1); !over { + t.Error("two files must exceed a limit of one") + } +} diff --git a/internal/mcp/freshness_rider.go b/internal/mcp/freshness_rider.go new file mode 100644 index 000000000..1b8a12b02 --- /dev/null +++ b/internal/mcp/freshness_rider.go @@ -0,0 +1,173 @@ +package mcp + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/mark3labs/mcp-go/mcp" + + "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/indexer" +) + +// freshnessRiderFor returns a small structured freshness block for a +// file-reading tool whose target file has changed on disk since it was +// indexed — so an agent reading a just-edited file sees, inline, that the +// graph view may lag the working tree. It returns nil (no rider, zero +// extra tokens) for the overwhelmingly common fresh case, for non-file +// tools, and in multi-repo mode (where the legacy single-indexer's +// staleness signal is all-noise — per-repo watchers own freshness there). +// +// The check is O(1): one map lookup + one stat on the single file the +// tool targets, only for the handful of read/source tools. +func (s *Server) freshnessRiderFor(toolName string, req mcp.CallToolRequest) map[string]any { + if s.indexer == nil || s.multiIndexer != nil { + return nil + } + if os.Getenv("GORTEX_NO_FRESHNESS_RIDER") == "1" { + return nil + } + rel := targetRepoRelFile(toolName, req, s.indexer.RepoPrefix()) + if rel == "" { + return nil + } + stale := s.indexer.IsTrackedStale(rel) + mismatch := s.detectWorktreeMismatch() + if !stale && !mismatch { + return nil + } + out := map[string]any{"file": rel} + if stale { + out["stale"] = true + out["hint"] = "this file changed on disk since it was last indexed; the graph view may lag the working tree" + if r, ok := graph.Store(s.graph).(graph.RepoIndexStateReader); ok { + if st, found, _ := r.GetRepoIndexState(s.indexer.RepoPrefix()); found { + if st.IndexedSHA != "" { + out["indexed_sha"] = shortFreshSHA(st.IndexedSHA) + } + if st.Dirty { + out["working_tree_dirty_at_index"] = true + } + } + } + } + if mismatch { + out["worktree_mismatch"] = true + out["worktree_hint"] = "the working directory is a linked git worktree the indexed graph does not cover — results reflect another checkout" + } + return out +} + +// detectWorktreeMismatch reports (once per server, cached) whether the +// current working directory is a linked git worktree that the indexed graph +// does not cover — i.e. the agent is working in a worktree but the graph +// reflects a different checkout, so its results may not match the files on +// disk. Single-repo only; multi-repo routing owns its own worktree scoping. +func (s *Server) detectWorktreeMismatch() bool { + s.worktreeMismatchOnce.Do(func() { + if s.indexer == nil || s.multiIndexer != nil { + return + } + cwd, err := os.Getwd() + if err != nil || cwd == "" { + return + } + if !indexer.ResolveWorktree(cwd).IsWorktree { + return // not a linked worktree — nothing to warn about + } + root := s.indexer.RootPath() + if root == "" { + return + } + // The cwd is a linked worktree. If the indexed root does not contain + // it, the graph reflects another checkout. + cwdResolved, _ := filepath.EvalSymlinks(cwd) + rootResolved, _ := filepath.EvalSymlinks(root) + if cwdResolved == "" { + cwdResolved = cwd + } + if rootResolved == "" { + rootResolved = root + } + if !pathWithin(cwdResolved, rootResolved) { + s.worktreeMismatch = true + } + }) + return s.worktreeMismatch +} + +// pathWithin reports whether child is equal to or nested under parent, on a +// slash-segment boundary (so /a/bc is not "within" /a/b). +func pathWithin(child, parent string) bool { + child = filepath.Clean(child) + parent = filepath.Clean(parent) + if child == parent { + return true + } + return strings.HasPrefix(child, parent+string(filepath.Separator)) +} + +// targetRepoRelFile extracts the repo-relative path of the single file a +// read tool targets, or "" when the tool is not file-scoped. A leading +// repo prefix is stripped so the result matches the indexer's mtime keys; +// a path that does not match is simply reported not-stale (IsTrackedStale +// returns false for an unknown key), so imperfect normalization is safe. +func targetRepoRelFile(toolName string, req mcp.CallToolRequest, prefix string) string { + var raw string + switch toolName { + case "read_file", "get_file_summary", "get_editing_context": + raw = req.GetString("path", "") + case "get_symbol_source", "get_symbol": + id := req.GetString("id", "") + if i := strings.Index(id, "::"); i >= 0 { + raw = id[:i] + } + default: + return "" + } + if raw == "" { + return "" + } + raw = filepath.ToSlash(raw) + if prefix != "" { + raw = strings.TrimPrefix(raw, prefix+"/") + } + return raw +} + +// decorateResultWithFreshness attaches the freshness rider to a JSON-object +// tool response under the "freshness" key. Non-JSON-object payloads +// (GCX / TOON / arrays) are left untouched — a best-effort hint must never +// reshape a compact wire format the caller opted into. +func decorateResultWithFreshness(res *mcp.CallToolResult, rider map[string]any) *mcp.CallToolResult { + if len(rider) == 0 { + return res + } + text, ok := singleTextContent(res) + if !ok || text == "" { + return res + } + var asObj map[string]any + if json.Unmarshal([]byte(text), &asObj) != nil { + return res + } + if _, exists := asObj["freshness"]; exists { + return res + } + asObj["freshness"] = rider + body, err := json.Marshal(asObj) + if err != nil { + return res + } + return rebuildTextResult(res, string(body)) +} + +// shortFreshSHA trims a git SHA to 12 chars for the rider. +func shortFreshSHA(sha string) string { + if len(sha) > 12 { + return sha[:12] + } + return sha +} diff --git a/internal/mcp/freshness_rider_test.go b/internal/mcp/freshness_rider_test.go new file mode 100644 index 000000000..546fc7f23 --- /dev/null +++ b/internal/mcp/freshness_rider_test.go @@ -0,0 +1,69 @@ +package mcp + +import ( + "encoding/json" + "testing" + + "github.com/mark3labs/mcp-go/mcp" + "github.com/stretchr/testify/require" +) + +func freshReq(args map[string]any) mcp.CallToolRequest { + var req mcp.CallToolRequest + req.Params.Arguments = args + return req +} + +func TestTargetRepoRelFile(t *testing.T) { + require.Equal(t, "internal/x.go", + targetRepoRelFile("read_file", freshReq(map[string]any{"path": "internal/x.go"}), "")) + require.Equal(t, "internal/x.go", + targetRepoRelFile("read_file", freshReq(map[string]any{"path": "gortex/internal/x.go"}), "gortex")) + require.Equal(t, "a.go", + targetRepoRelFile("get_symbol_source", freshReq(map[string]any{"id": "a.go::Foo"}), "")) + // Non-file tools yield no target. + require.Equal(t, "", + targetRepoRelFile("search_symbols", freshReq(map[string]any{"query": "x"}), "")) + // Empty args yield no target. + require.Equal(t, "", + targetRepoRelFile("read_file", freshReq(map[string]any{}), "")) +} + +func TestDecorateResultWithFreshness(t *testing.T) { + rider := map[string]any{"file": "a.go", "stale": true} + + // JSON object: rider attached under "freshness", original keys kept. + got := decorateResultWithFreshness(mcp.NewToolResultText(`{"x":1}`), rider) + text, ok := singleTextContent(got) + require.True(t, ok) + var obj map[string]any + require.NoError(t, json.Unmarshal([]byte(text), &obj)) + require.Equal(t, float64(1), obj["x"]) + require.NotNil(t, obj["freshness"]) + + // Non-JSON-object payload (GCX/TOON) is left untouched. + got2 := decorateResultWithFreshness(mcp.NewToolResultText("GCX1 tool=foo\nrow1"), rider) + text2, _ := singleTextContent(got2) + require.Equal(t, "GCX1 tool=foo\nrow1", text2) + + // Empty rider is a no-op. + got3 := decorateResultWithFreshness(mcp.NewToolResultText(`{"x":1}`), nil) + text3, _ := singleTextContent(got3) + require.Equal(t, `{"x":1}`, text3) + + // A worktree-mismatch-only rider still attaches. + got4 := decorateResultWithFreshness(mcp.NewToolResultText(`{"x":1}`), + map[string]any{"worktree_mismatch": true}) + text4, _ := singleTextContent(got4) + var obj4 map[string]any + require.NoError(t, json.Unmarshal([]byte(text4), &obj4)) + require.Equal(t, true, obj4["freshness"].(map[string]any)["worktree_mismatch"]) +} + +func TestPathWithin(t *testing.T) { + require.True(t, pathWithin("/a/b/c", "/a/b")) + require.True(t, pathWithin("/a/b", "/a/b")) + require.False(t, pathWithin("/a/bc", "/a/b"), "must respect segment boundaries") + require.False(t, pathWithin("/a", "/a/b")) + require.False(t, pathWithin("/x/y", "/a/b")) +} diff --git a/internal/mcp/index_health_skip_test.go b/internal/mcp/index_health_skip_test.go new file mode 100644 index 000000000..ea7eff557 --- /dev/null +++ b/internal/mcp/index_health_skip_test.go @@ -0,0 +1,30 @@ +package mcp + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/zzet/gortex/internal/graph" +) + +// TestIndexHealth_SkipRollupAndDensity verifies index_health rolls up +// synthetic skip nodes by reason and reports a nodes_per_file density. +func TestIndexHealth_SkipRollupAndDensity(t *testing.T) { + srv, _ := setupTestServer(t) + + srv.graph.AddBatch([]*graph.Node{ + {ID: "skip_a.go", Kind: graph.KindFile, Name: "skip_a.go", FilePath: "skip_a.go", Meta: map[string]any{"skip_reason": "size"}}, + {ID: "skip_b.js", Kind: graph.KindFile, Name: "skip_b.js", FilePath: "skip_b.js", Meta: map[string]any{"skip_reason": "parse_failed"}}, + {ID: "skip_c.ts", Kind: graph.KindFile, Name: "skip_c.ts", FilePath: "skip_c.ts", Meta: map[string]any{"skip_reason": "parse_failed"}}, + }, nil) + + payload := srv.buildIndexHealthPayload() + require.NotNil(t, payload) + require.Contains(t, payload, "nodes_per_file") + + skipped, ok := payload["skipped"].(map[string]int) + require.True(t, ok, "index_health must roll up skip_reason counts") + require.Equal(t, 1, skipped["size"]) + require.Equal(t, 2, skipped["parse_failed"]) +} diff --git a/internal/mcp/overlay.go b/internal/mcp/overlay.go index 6b29e8313..6666ce271 100644 --- a/internal/mcp/overlay.go +++ b/internal/mcp/overlay.go @@ -105,6 +105,10 @@ func (s *Server) wrapToolHandler(h mcpserver.ToolHandlerFunc) mcpserver.ToolHand if blocked := s.checkToolGate(ctx, req.Params.Name); blocked != nil { return blocked, nil } + // Opt-in zero-config: background-index an untracked cwd on the first + // tool call (GORTEX_AUTOINDEX=1). Cheap getenv + sync.Once on the + // request path; all real work runs on a background goroutine. + s.maybeAutoIndexCWD() view, err := s.buildOverlayViewForCtx(ctx) if err != nil { // Drift surfaces as a structured tool error result so the @@ -141,6 +145,15 @@ func (s *Server) wrapToolHandler(h mcpserver.ToolHandlerFunc) mcpserver.ToolHand if warming && hErr == nil { res = decorateResultWithWarming(res, env) } + // Inline freshness: when a file-reading tool returns content for a + // file that has changed on disk since it was indexed, attach a + // small `freshness` block so the agent knows the graph view may lag + // the working tree. Omitted (zero cost) for the common fresh case. + if hErr == nil { + if rider := s.freshnessRiderFor(req.Params.Name, req); rider != nil { + res = decorateResultWithFreshness(res, rider) + } + } // Capture large successful responses into the session ring so // the post-filter tools can re-cut them without re-querying. if hErr == nil { diff --git a/internal/mcp/server.go b/internal/mcp/server.go index ebab39906..11f392932 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -165,6 +165,17 @@ type Server struct { cochangeByFile map[string]map[string]float64 cochangeCount map[string]map[string]int + // autoIndexOnce guards the opt-in (GORTEX_AUTOINDEX=1) zero-config + // background index of an untracked cwd, fired at most once per session + // from the first tool call. See auto_index.go. + autoIndexOnce sync.Once + + // worktreeMismatch is computed once per server: true when the working + // directory is a linked git worktree the indexed graph does not cover, + // so read tools can warn that results reflect another checkout. + worktreeMismatchOnce sync.Once + worktreeMismatch bool + // artifacts caches the materialised `.gortex.yaml::artifacts` // manifest. artifactEntries is the configured manifest (installed // via SetArtifacts); artifactList is the result of materialising diff --git a/internal/mcp/tools_enhancements.go b/internal/mcp/tools_enhancements.go index f73a17d3c..7377d155a 100644 --- a/internal/mcp/tools_enhancements.go +++ b/internal/mcp/tools_enhancements.go @@ -2744,6 +2744,31 @@ func (s *Server) buildIndexHealthPayload() map[string]any { langCoverage[lang] = true } + // Skip rollup: count synthetic file nodes by skip_reason so an agent + // can see WHY a file is missing from the graph (size / timeout / + // minified / parse_failed / parse_panic) instead of guessing. + skipped := map[string]int{} + for n := range s.graph.NodesByKind(graph.KindFile) { + if n == nil || n.Meta == nil { + continue + } + if reason, _ := n.Meta["skip_reason"].(string); reason != "" { + skipped[reason]++ + } + } + + // Density plausibility: even a trivial source file yields a file node + // plus at least one symbol, so a populated graph averaging barely more + // than one node per file means extraction produced little beyond the + // file shells — a broken grammar or an aborted reindex. A soft warning, + // never a reject. + var nodesPerFile float64 + fileNodes := stats.ByKind[string(graph.KindFile)] + if fileNodes > 0 { + nodesPerFile = math.Round(float64(stats.TotalNodes)/float64(fileNodes)*100) / 100 + } + densityDegenerate := fileNodes >= 5 && nodesPerFile > 0 && nodesPerFile < 1.2 + lastIndexTime := s.indexer.LastIndexTime() lastIndexStr := "" if !lastIndexTime.IsZero() { @@ -2769,6 +2794,14 @@ func (s *Server) buildIndexHealthPayload() map[string]any { recommendation = msg + " " + recommendation } } + if densityDegenerate { + msg := "Index has files but almost no symbol nodes (nodes_per_file < 1.2) — extraction produced little beyond file shells. Re-index with index_repository path \".\"; if it persists the language grammar may be broken or the files unsupported." + if recommendation == "" { + recommendation = msg + } else { + recommendation = msg + " " + recommendation + } + } result := map[string]any{ "health_score": healthScore, @@ -2779,6 +2812,10 @@ func (s *Server) buildIndexHealthPayload() map[string]any { "node_count": stats.TotalNodes, "edge_count": stats.TotalEdges, "edges_ok": edgesOK, + "nodes_per_file": nodesPerFile, + } + if len(skipped) > 0 { + result["skipped"] = skipped } if len(parseErrors) > 0 { result["parse_failures"] = parseErrors diff --git a/internal/progress/eta.go b/internal/progress/eta.go new file mode 100644 index 000000000..333cff399 --- /dev/null +++ b/internal/progress/eta.go @@ -0,0 +1,88 @@ +package progress + +import ( + "fmt" + "time" +) + +// Phase enumerates the typed sub-phases of an index / enrich run, so +// progress labels are consistent across reporters and surfaces instead of +// ad-hoc strings. They double as human-readable stage labels. +type Phase string + +const ( + PhaseDiscover Phase = "discovering files" + PhaseParse Phase = "parsing" + PhaseResolve Phase = "resolving references" + PhaseInfer Phase = "linking symbols" + PhasePersist Phase = "persisting" +) + +// FormatElapsed renders a duration as a compact human string: "45s", +// "5m 12s", "1h 20m". +func FormatElapsed(d time.Duration) string { + if d < 0 { + d = 0 + } + switch { + case d < time.Minute: + return fmt.Sprintf("%ds", int(d.Seconds()+0.5)) + case d < time.Hour: + m := int(d / time.Minute) + s := int((d % time.Minute) / time.Second) + return fmt.Sprintf("%dm %02ds", m, s) + default: + h := int(d / time.Hour) + m := int((d % time.Hour) / time.Minute) + return fmt.Sprintf("%dh %02dm", h, m) + } +} + +// ETA returns the throughput (items/sec) and estimated time remaining given +// a start time and within-phase counters. A zero/unknown total yields a +// zero eta. Rate is 0 until some time and work have elapsed. +func ETA(start time.Time, current, total int) (itemsPerSec float64, eta time.Duration) { + elapsed := time.Since(start) + if elapsed <= 0 || current <= 0 { + return 0, 0 + } + itemsPerSec = float64(current) / elapsed.Seconds() + if total > current && itemsPerSec > 0 { + remaining := float64(total - current) + eta = time.Duration(remaining/itemsPerSec) * time.Second + } + return itemsPerSec, eta +} + +// ProgressStats bundles the derived progress metrics for a phase, ready to +// drop into a readiness / health payload. +type ProgressStats struct { + Phase string `json:"phase"` + Current int `json:"current"` + Total int `json:"total"` + Percent int `json:"percent"` + ItemsPerSec float64 `json:"items_per_sec"` + Elapsed string `json:"elapsed"` + ETA string `json:"eta,omitempty"` +} + +// Stats computes the derived progress metrics for a phase from its start +// time and counters — percent, throughput, human-formatted elapsed, and a +// human-formatted ETA (omitted when the total is unknown). +func Stats(phase string, start time.Time, current, total int) ProgressStats { + rate, eta := ETA(start, current, total) + ps := ProgressStats{ + Phase: phase, + Current: current, + Total: total, + ItemsPerSec: rate, + Elapsed: FormatElapsed(time.Since(start)), + } + if total > 0 { + ps.Percent = int(float64(current) / float64(total) * 100) + } + if eta > 0 { + ps.ETA = FormatElapsed(eta) + } + return ps +} diff --git a/internal/progress/eta_test.go b/internal/progress/eta_test.go new file mode 100644 index 000000000..1f04a3487 --- /dev/null +++ b/internal/progress/eta_test.go @@ -0,0 +1,51 @@ +package progress + +import ( + "testing" + "time" +) + +func TestFormatElapsed(t *testing.T) { + cases := []struct { + d time.Duration + want string + }{ + {0, "0s"}, + {30 * time.Second, "30s"}, + {90 * time.Second, "1m 30s"}, + {3700 * time.Second, "1h 01m"}, + } + for _, c := range cases { + if got := FormatElapsed(c.d); got != c.want { + t.Errorf("FormatElapsed(%v) = %q, want %q", c.d, got, c.want) + } + } +} + +func TestETAAndStats(t *testing.T) { + start := time.Now().Add(-10 * time.Second) + + rate, eta := ETA(start, 100, 200) + if rate < 5 || rate > 20 { // ~10/s + t.Errorf("rate = %v, want ~10/s", rate) + } + if eta <= 0 { + t.Error("eta should be positive when total > current") + } + + // Unknown total → no eta. + if _, eta2 := ETA(start, 100, 0); eta2 != 0 { + t.Errorf("unknown total must yield zero eta, got %v", eta2) + } + + ps := Stats(string(PhaseParse), start, 100, 200) + if ps.Percent != 50 { + t.Errorf("percent = %d, want 50", ps.Percent) + } + if ps.ETA == "" { + t.Error("Stats must include an ETA when total known") + } + if ps.Phase != "parsing" { + t.Errorf("phase = %q", ps.Phase) + } +}