feat(indexer): git-aware untracked-asset skip + bounded init store

cmd/gortex/eval_parity.go

@@ -64,12 +64,14 @@ func runEvalParity(cmd *cobra.Command, args []string) error {
 			fmt.Fprintf(cmd.ErrOrStderr(), "  clone failed: %v — skipping\n", err)
 			continue
 		}
-		g, err := indexRepoForInit(ctx, dir, zap.NewNop())
+		g, cleanup, err := indexRepoForInit(ctx, dir, zap.NewNop())
 		if err != nil {
 			fmt.Fprintf(cmd.ErrOrStderr(), "  index failed: %v — skipping\n", err)
 			continue
 		}
-		for _, c := range parity.CoverageOf(g) {
+		covs := parity.CoverageOf(g)
+		cleanup() // release the temp store before the next repo
+		for _, c := range covs {
 			if c.Language != repo.Language {
 				continue // a Go repo may carry a few yaml/json files; measure its own language
 			}

cmd/gortex/init.go

@@ -33,6 +33,7 @@ import (
 	"github.com/zzet/gortex/internal/claudemd"
 	"github.com/zzet/gortex/internal/config"
 	"github.com/zzet/gortex/internal/graph"
+	"github.com/zzet/gortex/internal/graph/store_sqlite"
 	"github.com/zzet/gortex/internal/indexer"
 	"github.com/zzet/gortex/internal/parser"
 	"github.com/zzet/gortex/internal/parser/languages"
@@ -266,10 +267,11 @@ func runInit(cmd *cobra.Command, args []string) (err error) {
 		if prog.Enabled() {
 			idxLogger = zap.NewNop()
 		}
-		g, idxErr := indexRepoForInit(ctx, absRoot, idxLogger)
+		g, cleanup, idxErr := indexRepoForInit(ctx, absRoot, idxLogger)
 		if idxErr != nil {
 			fmt.Fprintf(cmd.ErrOrStderr(), "[gortex init] indexing failed: %v — proceeding without analysis/skills\n", idxErr)
 		} else {
+			defer cleanup()
 			prog.StageDone(stageIndex, "")
 			if initAnalyze {
 				prog.Stage(stageAnalyze, "")
@@ -365,7 +367,16 @@ func toEnvSkills(src []genskills.GeneratedSkill) []agents.GeneratedSkill {
 // stage transitions ("walking files", "parsing", …) as sub-status.
 // Pass a Nop logger when running under an animated spinner so structured
 // info logs don't duplicate the mesh frame.
-func indexRepoForInit(ctx context.Context, root string, logger *zap.Logger) (*graph.Graph, error) {
+//
+// It indexes into a temporary on-disk sqlite store rather than an
+// all-in-memory graph: nodes persist per file and the content sink leans
+// document / section text to disk, so a content-heavy repo (a RAG corpus
+// of decks, spreadsheets, and dataset shards) can't pin the whole
+// post-parse graph in RAM and OOM `gortex init` (#120). The store inherits
+// the indexer's shadow / byte-budget guards. The returned cleanup closes
+// the store and removes the temp dir; callers MUST call it once they are
+// done reading the returned graph.
+func indexRepoForInit(ctx context.Context, root string, logger *zap.Logger) (graph.Store, func(), error) {
 	if logger == nil {
 		logger = newLogger()
 	}
@@ -376,15 +387,29 @@ func indexRepoForInit(ctx context.Context, root string, logger *zap.Logger) (*gr
 		cfg = &config.Config{}
 	}
 
-	g := graph.New()
+	tmpDir, err := os.MkdirTemp("", "gortex-init-store-*")
+	if err != nil {
+		return nil, nil, err
+	}
+	st, err := store_sqlite.Open(filepath.Join(tmpDir, "init.sqlite"))
+	if err != nil {
+		_ = os.RemoveAll(tmpDir)
+		return nil, nil, err
+	}
+	cleanup := func() {
+		_ = st.Close()
+		_ = os.RemoveAll(tmpDir)
+	}
+
 	reg := parser.NewRegistry()
 	languages.RegisterAll(reg)
 
-	idx := indexer.New(g, reg, cfg.Index, logger)
-	if _, err := idx.IndexCtx(ctx, root); err != nil {
-		return nil, err
+	idx := indexer.New(st, reg, cfg.Index, logger)
+	if _, ierr := idx.IndexCtx(ctx, root); ierr != nil {
+		cleanup()
+		return nil, nil, ierr
 	}
-	return g, nil
+	return st, cleanup, nil
 }
 
 // emitJSONReport writes a single JSON object to w. Shape kept

internal/config/config.go

@@ -752,6 +752,21 @@ type IndexConfig struct {
 	// listable without being read or extracted. See
 	// ContentAdmissionConfig. Configured under `index.content`.
 	Content ContentAdmissionConfig `mapstructure:"content" yaml:"content,omitempty"`
+
+	// SkipUntrackedAssets drops document / data / image assets that git
+	// does not track (uncommitted working-tree files) during a full index.
+	// `.gitignore` is already honoured (respect_gitignore), but it can't
+	// catch files that are merely untracked — never `git add`ed and not in
+	// .gitignore — which is exactly how RAG corpora, downloaded datasets,
+	// and build outputs end up admitted (#120). When on, an untracked
+	// asset-class file is skipped at the walk with an `untracked_asset`
+	// telemetry node; untracked CODE is still indexed, so new / unsaved
+	// source keeps working. Off by default. Inert on a non-git repo or when
+	// `git ls-files` is unavailable (everything is admitted as before).
+	// Applies to the cold full-index walk; the incremental watcher path
+	// keeps the size / class caps only. Configured under
+	// `index.skip_untracked_assets`.
+	SkipUntrackedAssets bool `mapstructure:"skip_untracked_assets" yaml:"skip_untracked_assets,omitempty"`
 }
 
 // ContentAdmissionConfig gates which large non-source artifacts enter the

internal/indexer/content_admission.go

@@ -1,8 +1,13 @@
 package indexer
 
 import (
+	"context"
 	"path/filepath"
+	"strings"
 
+	"go.uber.org/zap"
+
+	"github.com/zzet/gortex/internal/gitcmd"
 	"github.com/zzet/gortex/internal/graph"
 	"github.com/zzet/gortex/internal/parser"
 )
@@ -11,9 +16,10 @@ import (
 // Meta["skip_reason"] so a dropped asset stays listable and index_health
 // rolls it up, mirroring the size / timeout / minified skip telemetry.
 const (
-	skipReasonLargeDocument = "large_document"   // document over the per-file cap
-	skipReasonVectorData    = "vector_data"      // data asset, data indexing off
-	skipReasonLargeData     = "large_data_asset" // data asset over the per-file cap
+	skipReasonLargeDocument  = "large_document"   // document over the per-file cap
+	skipReasonVectorData     = "vector_data"      // data asset, data indexing off
+	skipReasonLargeData      = "large_data_asset" // data asset over the per-file cap
+	skipReasonUntrackedAsset = "untracked_asset"  // asset-class file git does not track
 )
 
 // contentAdmissionGate decides, by asset class, whether a non-source artifact
@@ -73,6 +79,67 @@ func (g *contentAdmissionGate) skip(lang string, size int64) (string, bool) {
 	return "", false
 }
 
+// untrackedAssetGate skips asset-class files (document / data / image) that
+// git does not track, when index.skip_untracked_assets is on. It is built
+// once per cold walk from the registry's asset-class map and the repo's
+// `git ls-files` set. A nil gate is inert — the flag is off, the repo is not
+// a git repo, or `git ls-files` failed (admit everything as before).
+type untrackedAssetGate struct {
+	classes map[string]parser.AssetClass
+	tracked map[string]struct{} // absolute paths git tracks
+}
+
+// newUntrackedAssetGate builds the gate, returning nil (inert) when the flag
+// is off, no asset extractors are registered, or the tracked set can't be
+// resolved.
+func (idx *Indexer) newUntrackedAssetGate(ctx context.Context, absRoot string) *untrackedAssetGate {
+	if !idx.config.SkipUntrackedAssets {
+		return nil
+	}
+	classes := idx.registry.AssetClasses()
+	if len(classes) == 0 {
+		return nil
+	}
+	tracked, ok := gitTrackedSet(ctx, absRoot)
+	if !ok {
+		idx.logger.Info("indexer: skip_untracked_assets is on but the git tracked-set is unavailable; admitting all assets",
+			zap.String("root", absRoot))
+		return nil
+	}
+	return &untrackedAssetGate{classes: classes, tracked: tracked}
+}
+
+// skip reports whether an asset-class file at absPath should be dropped
+// because git does not track it. Non-asset languages (untracked code) and
+// tracked assets are never skipped here.
+func (g *untrackedAssetGate) skip(lang, absPath string) (string, bool) {
+	if g == nil || g.classes[lang] == "" {
+		return "", false
+	}
+	if _, ok := g.tracked[absPath]; ok {
+		return "", false
+	}
+	return skipReasonUntrackedAsset, true
+}
+
+// gitTrackedSet returns the set of absolute paths git tracks under root, or
+// (nil, false) when root is not a git repo or `git ls-files` fails. The -z
+// form is NUL-delimited so paths with spaces / newlines are handled exactly.
+func gitTrackedSet(ctx context.Context, root string) (map[string]struct{}, bool) {
+	out, err := gitcmd.Output(ctx, root, "ls-files", "-z")
+	if err != nil {
+		return nil, false
+	}
+	set := make(map[string]struct{})
+	for rel := range strings.SplitSeq(out, "\x00") {
+		if rel == "" {
+			continue
+		}
+		set[filepath.Join(root, filepath.FromSlash(rel))] = struct{}{}
+	}
+	return set, true
+}
+
 // contentSkipNode builds a synthetic file node for a content / data asset
 // dropped by the admission gate, carrying the skip reason and size so the
 // file stays visible (queryable, index_health rollup) without being read or

internal/indexer/content_admission_test.go

@@ -1,6 +1,7 @@
 package indexer
 
 import (
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -170,6 +171,71 @@ func TestIndex_DataAssetSkippedByDefault(t *testing.T) {
 	require.Equal(t, "data", n2.Meta["data_class"])
 }
 
+// TestIndex_SkipUntrackedAssets verifies that, with the opt-in flag on,
+// untracked document/data assets are dropped while tracked assets and
+// untracked CODE are still indexed; and that the default (flag off) admits
+// untracked documents.
+func TestIndex_SkipUntrackedAssets(t *testing.T) {
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git binary not available in PATH")
+	}
+	dir := t.TempDir()
+	runGit(t, dir, "init", "-q", "-b", "main")
+	runGit(t, dir, "config", "user.email", "t@example.com")
+	runGit(t, dir, "config", "user.name", "T")
+	runGit(t, dir, "config", "commit.gpgsign", "false")
+
+	// Tracked: a code file and a small document.
+	writeFile(t, filepath.Join(dir, "code.go"), "package main\n\nfunc A() {}\n")
+	writeFile(t, filepath.Join(dir, "tracked.txt"), "committed note")
+	runGit(t, dir, "add", ".")
+	runGit(t, dir, "commit", "-q", "-m", "init")
+
+	// Untracked working-tree files: a document, a data asset, and code.
+	writeFile(t, filepath.Join(dir, "untracked.txt"), "scratch rag asset")
+	writeFile(t, filepath.Join(dir, "vectors.npy"), "NUMPY-placeholder-bytes")
+	writeFile(t, filepath.Join(dir, "new.go"), "package main\n\nfunc B() {}\n")
+
+	// Flag ON: untracked assets dropped, tracked asset + untracked code kept.
+	g := graph.New()
+	idx := newAssetTestIndexer(g)
+	idx.config.SkipUntrackedAssets = true
+	if _, err := idx.IndexCtx(testCtx(), dir); err != nil {
+		t.Fatalf("index: %v", err)
+	}
+	for _, p := range []string{"untracked.txt", "vectors.npy"} {
+		n := g.GetNode(p)
+		require.NotNil(t, n, p)
+		require.Equal(t, skipReasonUntrackedAsset, n.Meta["skip_reason"], p)
+	}
+	tracked := g.GetNode("tracked.txt")
+	require.NotNil(t, tracked)
+	require.Nil(t, tracked.Meta["skip_reason"], "tracked document must be admitted")
+	newGo := g.GetNode("new.go")
+	require.NotNil(t, newGo, "untracked CODE must still be indexed")
+	require.Nil(t, newGo.Meta["skip_reason"])
+
+	// Flag OFF (default): the untracked document is admitted as content.
+	g2 := graph.New()
+	idx2 := newAssetTestIndexer(g2)
+	if _, err := idx2.IndexCtx(testCtx(), dir); err != nil {
+		t.Fatalf("index: %v", err)
+	}
+	un := g2.GetNode("untracked.txt")
+	require.NotNil(t, un)
+	require.Nil(t, un.Meta["skip_reason"], "with the flag off, an untracked document is admitted")
+}
+
+// TestGitTrackedSet_NonGitDirInert verifies gitTrackedSet reports failure
+// (gate inert) outside a git repo.
+func TestGitTrackedSet_NonGitDirInert(t *testing.T) {
+	if _, err := exec.LookPath("git"); err != nil {
+		t.Skip("git binary not available in PATH")
+	}
+	_, ok := gitTrackedSet(testCtx(), t.TempDir())
+	require.False(t, ok)
+}
+
 // TestIndexFile_ContentSkip verifies the incremental (watcher) path applies
 // the same gate, so a document the cold walk would skip doesn't get parsed
 // back in on re-index.

internal/indexer/indexer.go

@@ -1953,6 +1953,11 @@ func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexRes
 	// non-source files into the parse pipeline and OOM (#120). Inert for
 	// all-code repos.
 	contentGate := idx.newContentAdmissionGate()
+	// Git-aware admission (opt-in): when index.skip_untracked_assets is on,
+	// drop asset-class files git does not track — uncommitted RAG corpora /
+	// datasets / build outputs that .gitignore can't catch (#120). Inert
+	// when off, on a non-git repo, or when git is unavailable.
+	untrackedGate := idx.newUntrackedAssetGate(ctx, absRoot)
 	var files []walkedFile
 	var skippedLarge int
 	var skippedBytes int64
@@ -1992,6 +1997,14 @@ func (idx *Indexer) IndexCtx(ctx context.Context, root string) (result *IndexRes
 			})
 			return nil
 		}
+		if reason, skip := untrackedGate.skip(lang, path); skip {
+			skippedContentBytes += info.Size()
+			rel, _ := filepath.Rel(absRoot, path)
+			skippedByContent = append(skippedByContent, skippedFile{
+				relPath: filepath.ToSlash(rel), lang: lang, size: info.Size(), reason: reason,
+			})
+			return nil
+		}
 		if reason, skip := contentGate.skip(lang, info.Size()); skip {
 			skippedContentBytes += info.Size()
 			rel, _ := filepath.Rel(absRoot, path)