diff --git a/cmd/gortex/eval_parity.go b/cmd/gortex/eval_parity.go index c79dea7a7..5eb665c96 100644 --- a/cmd/gortex/eval_parity.go +++ b/cmd/gortex/eval_parity.go @@ -64,12 +64,14 @@ func runEvalParity(cmd *cobra.Command, args []string) error { fmt.Fprintf(cmd.ErrOrStderr(), " clone failed: %v — skipping\n", err) continue } - g, err := indexRepoForInit(ctx, dir, zap.NewNop()) + g, cleanup, err := indexRepoForInit(ctx, dir, zap.NewNop()) if err != nil { fmt.Fprintf(cmd.ErrOrStderr(), " index failed: %v — skipping\n", err) continue } - for _, c := range parity.CoverageOf(g) { + covs := parity.CoverageOf(g) + cleanup() // release the temp store before the next repo + for _, c := range covs { if c.Language != repo.Language { continue // a Go repo may carry a few yaml/json files; measure its own language } diff --git a/cmd/gortex/init.go b/cmd/gortex/init.go index 79313fa01..bcd6f58bc 100644 --- a/cmd/gortex/init.go +++ b/cmd/gortex/init.go @@ -33,6 +33,7 @@ import ( "github.com/zzet/gortex/internal/claudemd" "github.com/zzet/gortex/internal/config" "github.com/zzet/gortex/internal/graph" + "github.com/zzet/gortex/internal/graph/store_sqlite" "github.com/zzet/gortex/internal/indexer" "github.com/zzet/gortex/internal/parser" "github.com/zzet/gortex/internal/parser/languages" @@ -266,10 +267,11 @@ func runInit(cmd *cobra.Command, args []string) (err error) { if prog.Enabled() { idxLogger = zap.NewNop() } - g, idxErr := indexRepoForInit(ctx, absRoot, idxLogger) + g, cleanup, idxErr := indexRepoForInit(ctx, absRoot, idxLogger) if idxErr != nil { fmt.Fprintf(cmd.ErrOrStderr(), "[gortex init] indexing failed: %v — proceeding without analysis/skills\n", idxErr) } else { + defer cleanup() prog.StageDone(stageIndex, "") if initAnalyze { prog.Stage(stageAnalyze, "") @@ -365,7 +367,16 @@ func toEnvSkills(src []genskills.GeneratedSkill) []agents.GeneratedSkill { // stage transitions ("walking files", "parsing", …) as sub-status. // Pass a Nop logger when running under an animated spinner so structured // info logs don't duplicate the mesh frame. -func indexRepoForInit(ctx context.Context, root string, logger *zap.Logger) (*graph.Graph, error) { +// +// It indexes into a temporary on-disk sqlite store rather than an +// all-in-memory graph: nodes persist per file and the content sink leans +// document / section text to disk, so a content-heavy repo (a RAG corpus +// of decks, spreadsheets, and dataset shards) can't pin the whole +// post-parse graph in RAM and OOM `gortex init` (#120). The store inherits +// the indexer's shadow / byte-budget guards. The returned cleanup closes +// the store and removes the temp dir; callers MUST call it once they are +// done reading the returned graph. +func indexRepoForInit(ctx context.Context, root string, logger *zap.Logger) (graph.Store, func(), error) { if logger == nil { logger = newLogger() } @@ -376,15 +387,29 @@ func indexRepoForInit(ctx context.Context, root string, logger *zap.Logger) (*gr cfg = &config.Config{} } - g := graph.New() + tmpDir, err := os.MkdirTemp("", "gortex-init-store-*") + if err != nil { + return nil, nil, err + } + st, err := store_sqlite.Open(filepath.Join(tmpDir, "init.sqlite")) + if err != nil { + _ = os.RemoveAll(tmpDir) + return nil, nil, err + } + cleanup := func() { + _ = st.Close() + _ = os.RemoveAll(tmpDir) + } + reg := parser.NewRegistry() languages.RegisterAll(reg) - idx := indexer.New(g, reg, cfg.Index, logger) - if _, err := idx.IndexCtx(ctx, root); err != nil { - return nil, err + idx := indexer.New(st, reg, cfg.Index, logger) + if _, ierr := idx.IndexCtx(ctx, root); ierr != nil { + cleanup() + return nil, nil, ierr } - return g, nil + return st, cleanup, nil } // emitJSONReport writes a single JSON object to w. Shape kept diff --git a/internal/config/config.go b/internal/config/config.go index eb7c46541..0fcc6f39b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -752,6 +752,21 @@ type IndexConfig struct { // listable without being read or extracted. See // ContentAdmissionConfig. Configured under `index.content`. Content ContentAdmissionConfig `mapstructure:"content" yaml:"content,omitempty"` + + // SkipUntrackedAssets drops document / data / image assets that git + // does not track (uncommitted working-tree files) during a full index. + // `.gitignore` is already honoured (respect_gitignore), but it can't + // catch files that are merely untracked — never `git add`ed and not in + // .gitignore — which is exactly how RAG corpora, downloaded datasets, + // and build outputs end up admitted (#120). When on, an untracked + // asset-class file is skipped at the walk with an `untracked_asset` + // telemetry node; untracked CODE is still indexed, so new / unsaved + // source keeps working. Off by default. Inert on a non-git repo or when + // `git ls-files` is unavailable (everything is admitted as before). + // Applies to the cold full-index walk; the incremental watcher path + // keeps the size / class caps only. Configured under + // `index.skip_untracked_assets`. + SkipUntrackedAssets bool `mapstructure:"skip_untracked_assets" yaml:"skip_untracked_assets,omitempty"` } // ContentAdmissionConfig gates which large non-source artifacts enter the diff --git a/internal/indexer/content_admission.go b/internal/indexer/content_admission.go index 685194a6d..cd3b193fc 100644 --- a/internal/indexer/content_admission.go +++ b/internal/indexer/content_admission.go @@ -1,8 +1,13 @@ package indexer import ( + "context" "path/filepath" + "strings" + "go.uber.org/zap" + + "github.com/zzet/gortex/internal/gitcmd" "github.com/zzet/gortex/internal/graph" "github.com/zzet/gortex/internal/parser" ) @@ -11,9 +16,10 @@ import ( // Meta["skip_reason"] so a dropped asset stays listable and index_health // rolls it up, mirroring the size / timeout / minified skip telemetry. const ( - skipReasonLargeDocument = "large_document" // document over the per-file cap - skipReasonVectorData = "vector_data" // data asset, data indexing off - skipReasonLargeData = "large_data_asset" // data asset over the per-file cap + skipReasonLargeDocument = "large_document" // document over the per-file cap + skipReasonVectorData = "vector_data" // data asset, data indexing off + skipReasonLargeData = "large_data_asset" // data asset over the per-file cap + skipReasonUntrackedAsset = "untracked_asset" // asset-class file git does not track ) // contentAdmissionGate decides, by asset class, whether a non-source artifact @@ -73,6 +79,67 @@ func (g *contentAdmissionGate) skip(lang string, size int64) (string, bool) { return "", false } +// untrackedAssetGate skips asset-class files (document / data / image) that +// git does not track, when index.skip_untracked_assets is on. It is built +// once per cold walk from the registry's asset-class map and the repo's +// `git ls-files` set. A nil gate is inert — the flag is off, the repo is not +// a git repo, or `git ls-files` failed (admit everything as before). +type untrackedAssetGate struct { + classes map[string]parser.AssetClass + tracked map[string]struct{} // absolute paths git tracks +} + +// newUntrackedAssetGate builds the gate, returning nil (inert) when the flag +// is off, no asset extractors are registered, or the tracked set can't be +// resolved. +func (idx *Indexer) newUntrackedAssetGate(ctx context.Context, absRoot string) *untrackedAssetGate { + if !idx.config.SkipUntrackedAssets { + return nil + } + classes := idx.registry.AssetClasses() + if len(classes) == 0 { + return nil + } + tracked, ok := gitTrackedSet(ctx, absRoot) + if !ok { + idx.logger.Info("indexer: skip_untracked_assets is on but the git tracked-set is unavailable; admitting all assets", + zap.String("root", absRoot)) + return nil + } + return &untrackedAssetGate{classes: classes, tracked: tracked} +} + +// skip reports whether an asset-class file at absPath should be dropped +// because git does not track it. Non-asset languages (untracked code) and +// tracked assets are never skipped here. +func (g *untrackedAssetGate) skip(lang, absPath string) (string, bool) { + if g == nil || g.classes[lang] == "" { + return "", false + } + if _, ok := g.tracked[absPath]; ok { + return "", false + } + return skipReasonUntrackedAsset, true +} + +// gitTrackedSet returns the set of absolute paths git tracks under root, or +// (nil, false) when root is not a git repo or `git ls-files` fails. The -z +// form is NUL-delimited so paths with spaces / newlines are handled exactly. +func gitTrackedSet(ctx context.Context, root string) (map[string]struct{}, bool) { + out, err := gitcmd.Output(ctx, root, "ls-files", "-z") + if err != nil { + return nil, false + } + set := make(map[string]struct{}) + for rel := range strings.SplitSeq(out, "\x00") { + if rel == "" { + continue + } + set[filepath.Join(root, filepath.FromSlash(rel))] = struct{}{} + } + return set, true +} + // contentSkipNode builds a synthetic file node for a content / data asset // dropped by the admission gate, carrying the skip reason and size so the // file stays visible (queryable, index_health rollup) without being read or diff --git a/internal/indexer/content_admission_test.go b/internal/indexer/content_admission_test.go index ee500eecc..901d0a13a 100644 --- a/internal/indexer/content_admission_test.go +++ b/internal/indexer/content_admission_test.go @@ -1,6 +1,7 @@ package indexer import ( + "os/exec" "path/filepath" "strings" "testing" @@ -170,6 +171,71 @@ func TestIndex_DataAssetSkippedByDefault(t *testing.T) { require.Equal(t, "data", n2.Meta["data_class"]) } +// TestIndex_SkipUntrackedAssets verifies that, with the opt-in flag on, +// untracked document/data assets are dropped while tracked assets and +// untracked CODE are still indexed; and that the default (flag off) admits +// untracked documents. +func TestIndex_SkipUntrackedAssets(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git binary not available in PATH") + } + dir := t.TempDir() + runGit(t, dir, "init", "-q", "-b", "main") + runGit(t, dir, "config", "user.email", "t@example.com") + runGit(t, dir, "config", "user.name", "T") + runGit(t, dir, "config", "commit.gpgsign", "false") + + // Tracked: a code file and a small document. + writeFile(t, filepath.Join(dir, "code.go"), "package main\n\nfunc A() {}\n") + writeFile(t, filepath.Join(dir, "tracked.txt"), "committed note") + runGit(t, dir, "add", ".") + runGit(t, dir, "commit", "-q", "-m", "init") + + // Untracked working-tree files: a document, a data asset, and code. + writeFile(t, filepath.Join(dir, "untracked.txt"), "scratch rag asset") + writeFile(t, filepath.Join(dir, "vectors.npy"), "NUMPY-placeholder-bytes") + writeFile(t, filepath.Join(dir, "new.go"), "package main\n\nfunc B() {}\n") + + // Flag ON: untracked assets dropped, tracked asset + untracked code kept. + g := graph.New() + idx := newAssetTestIndexer(g) + idx.config.SkipUntrackedAssets = true + if _, err := idx.IndexCtx(testCtx(), dir); err != nil { + t.Fatalf("index: %v", err) + } + for _, p := range []string{"untracked.txt", "vectors.npy"} { + n := g.GetNode(p) + require.NotNil(t, n, p) + require.Equal(t, skipReasonUntrackedAsset, n.Meta["skip_reason"], p) + } + tracked := g.GetNode("tracked.txt") + require.NotNil(t, tracked) + require.Nil(t, tracked.Meta["skip_reason"], "tracked document must be admitted") + newGo := g.GetNode("new.go") + require.NotNil(t, newGo, "untracked CODE must still be indexed") + require.Nil(t, newGo.Meta["skip_reason"]) + + // Flag OFF (default): the untracked document is admitted as content. + g2 := graph.New() + idx2 := newAssetTestIndexer(g2) + if _, err := idx2.IndexCtx(testCtx(), dir); err != nil { + t.Fatalf("index: %v", err) + } + un := g2.GetNode("untracked.txt") + require.NotNil(t, un) + require.Nil(t, un.Meta["skip_reason"], "with the flag off, an untracked document is admitted") +} + +// TestGitTrackedSet_NonGitDirInert verifies gitTrackedSet reports failure +// (gate inert) outside a git repo. +func TestGitTrackedSet_NonGitDirInert(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git binary not available in PATH") + } + _, ok := gitTrackedSet(testCtx(), t.TempDir()) + require.False(t, ok) +} + // TestIndexFile_ContentSkip verifies the incremental (watcher) path applies // the same gate, so a document the cold walk would skip doesn't get parsed // back in on re-index. diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 9763bce86..a4653e7b4 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -1953,6 +1953,11 @@ func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexRes // non-source files into the parse pipeline and OOM (#120). Inert for // all-code repos. contentGate := idx.newContentAdmissionGate() + // Git-aware admission (opt-in): when index.skip_untracked_assets is on, + // drop asset-class files git does not track — uncommitted RAG corpora / + // datasets / build outputs that .gitignore can't catch (#120). Inert + // when off, on a non-git repo, or when git is unavailable. + untrackedGate := idx.newUntrackedAssetGate(ctx, absRoot) var files []walkedFile var skippedLarge int var skippedBytes int64 @@ -1992,6 +1997,14 @@ func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexRes }) return nil } + if reason, skip := untrackedGate.skip(lang, path); skip { + skippedContentBytes += info.Size() + rel, _ := filepath.Rel(absRoot, path) + skippedByContent = append(skippedByContent, skippedFile{ + relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(), reason: reason, + }) + return nil + } if reason, skip := contentGate.skip(lang, info.Size()); skip { skippedContentBytes += info.Size() rel, _ := filepath.Rel(absRoot, path)