feat: index freshness, staleness & watch robustness

cmd/gortex/daemon.go

@@ -412,6 +412,16 @@ func runDaemonStart(cmd *cobra.Command, _ []string) error {
 			if stateCapture.graph != nil {
 				out["graph_nodes"] = stateCapture.graph.NodeCount()
 				out["graph_edges"] = stateCapture.graph.EdgeCount()
+				if r, ok := stateCapture.graph.(graph.DBStatReporter); ok {
+					dbBytes, walBytes := r.DBStats()
+					if dbBytes > 0 || walBytes > 0 {
+						out["db_bytes"] = dbBytes
+						out["wal_bytes"] = walBytes
+						if dbBytes > 0 {
+							out["wal_db_ratio"] = float64(walBytes) / float64(dbBytes)
+						}
+					}
+				}
 			}
 			return out
 		})

cmd/gortex/daemon_state.go

@@ -423,9 +423,12 @@ func warmupDaemonState(state *daemonState, logger *zap.Logger) *indexer.MultiWat
 	logger.Info("daemon: warmup phase done",
 		zap.String("phase", "parallel_parse"),
 		zap.Duration("elapsed", time.Since(phaseStart)))
+	parseStats := progress.Stats(string(progress.PhaseParse), phaseStart, len(repos), len(repos))
 	publishReadinessPhase(state, "parallel_parse_done", false, map[string]any{
 		"tracked_repos": len(repos),
 		"elapsed_ms":    time.Since(phaseStart).Milliseconds(),
+		"elapsed_human": parseStats.Elapsed,
+		"repos_per_sec": parseStats.ItemsPerSec,
 	})
 
 	// Warm-restart fast path. When the reconcile loop above re-indexed

cmd/gortex/githook.go

@@ -91,10 +91,10 @@ func init() {
 // SupportedHooks list rather than importing it so the CLI surface
 // stays decoupled from the install package's internals.
 func supportedHook(name string) error {
-	if name == "post-commit" || name == "post-merge" {
+	if name == "post-commit" || name == "post-merge" || name == "post-checkout" {
 		return nil
 	}
-	return fmt.Errorf("unsupported hook %q (supported: post-commit, post-merge)", name)
+	return fmt.Errorf("unsupported hook %q (supported: post-commit, post-merge, post-checkout)", name)
 }
 
 func runGithookInstall(cmd *cobra.Command, args []string) error {

internal/config/config.go

@@ -355,6 +355,15 @@ type Config struct {
 	// re-include something an outer layer excluded.
 	Exclude []string `mapstructure:"exclude" yaml:"exclude,omitempty"`
 
+	// Include force-indexes paths an outer layer (notably the repo
+	// .gitignore) excludes — the dedicated, readable counterpart to
+	// hand-writing `!pattern` negations in Exclude. Each entry is appended
+	// last as a gitignore `!pattern` re-include, so it wins over the
+	// builtin / .gitignore / global / repo exclude layers. Use it for a
+	// vendored or generated tree you nonetheless want in the graph
+	// (e.g. Pods/, a per-customer customer/ directory).
+	Include []string `mapstructure:"include" yaml:"include,omitempty"`
+
 	// RuleFiles are paths (or globs) to TOML files of user-defined
 	// domain-extractor rules. Each rule is a tree-sitter pattern that
 	// becomes a registered detector surfaced through `analyze

internal/config/gitignore.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 )
 
 // loadRepoGitignore reads the `.gitignore` file at the repo root and
@@ -35,12 +36,25 @@ func loadRepoGitignore(repoPath string) []string {
 
 	var patterns []string
 	scanner := bufio.NewScanner(f)
+	// Tolerate pathologically long lines (a DLP-encrypted or otherwise
+	// non-text .gitignore) instead of aborting the whole read on the
+	// default 64 KiB scanner limit.
+	scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
 	for scanner.Scan() {
 		line := strings.TrimSpace(scanner.Text())
 		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}
+		// A non-UTF-8 line is not a valid gitignore pattern — feeding it to
+		// the matcher would mis-scan. Skip it defensively so a corrupt or
+		// encrypted .gitignore can never poison exclusion.
+		if !utf8.ValidString(line) {
+			continue
+		}
 		patterns = append(patterns, line)
 	}
+	// A read error (a still-too-long line, a transient I/O fault) must not
+	// discard the patterns already read — gitignore loading is best-effort,
+	// never a hard failure.
 	return patterns
 }

internal/config/gitignore_test.go

@@ -0,0 +1,42 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLoadRepoGitignore_SkipsBlanksCommentsAndNonUTF8(t *testing.T) {
+	dir := t.TempDir()
+	// A valid file interleaved with a blank line, a comment, and a
+	// non-UTF-8 (e.g. DLP-encrypted) line that must be skipped, not fed to
+	// the matcher and not aborting the read.
+	content := "# comment\n\nnode_modules/\n" + string([]byte{0xff, 0xfe, 0xfd}) + "\n*.log\n"
+	require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(content), 0o644))
+
+	got := loadRepoGitignore(dir)
+	require.Equal(t, []string{"node_modules/", "*.log"}, got)
+}
+
+func TestLoadRepoGitignore_LongLineDoesNotAbort(t *testing.T) {
+	dir := t.TempDir()
+	// A pathologically long line (larger than the default 64 KiB scanner
+	// buffer) must not discard the patterns that follow it.
+	long := make([]byte, 200*1024)
+	for i := range long {
+		long[i] = 'a'
+	}
+	content := "*.tmp\n" + string(long) + "\nbuild/\n"
+	require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte(content), 0o644))
+
+	got := loadRepoGitignore(dir)
+	require.Contains(t, got, "*.tmp")
+	require.Contains(t, got, "build/")
+}
+
+func TestLoadRepoGitignore_MissingFileIsNil(t *testing.T) {
+	require.Nil(t, loadRepoGitignore(t.TempDir()))
+	require.Nil(t, loadRepoGitignore(""))
+}

internal/config/manager.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	"go.uber.org/zap"
@@ -229,6 +230,23 @@ func (cm *ConfigManager) EffectiveExclude(repoPrefix string) []string {
 			out = append(out, ws.Watch.Exclude...)
 		}
 	}
+
+	// Force-include last so it wins: each Include entry becomes a gitignore
+	// `!pattern` re-include over every exclude layer above (builtin,
+	// .gitignore, global, repo). This is the readable form of hand-writing
+	// negations, for a vendored/generated tree you want indexed anyway.
+	if ws != nil {
+		for _, inc := range ws.Include {
+			inc = strings.TrimSpace(inc)
+			if inc == "" {
+				continue
+			}
+			if !strings.HasPrefix(inc, "!") {
+				inc = "!" + inc
+			}
+			out = append(out, inc)
+		}
+	}
 	return out
 }
 

internal/config/manager_test.go

@@ -126,6 +126,25 @@ index:
 	assert.Contains(t, got, "legacy/**", "legacy index.exclude must still be honoured")
 }
 
+func TestEffectiveExclude_IncludeForceReincludesGitignored(t *testing.T) {
+	cm, err := NewConfigManager("/tmp/nonexistent-gortex-test-cm/config.yaml")
+	require.NoError(t, err)
+
+	repoDir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".gitignore"), []byte("Pods/\n"), 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(repoDir, ".gortex.yaml"),
+		[]byte("include:\n  - \"Pods/\"\n"), 0644))
+	cm.LoadWorkspaceConfig("repo-inc", repoDir)
+
+	got := cm.EffectiveExclude("repo-inc")
+	assert.Contains(t, got, "Pods/", "the .gitignore exclude is present")
+	assert.Equal(t, "!Pods/", got[len(got)-1], "the force-include negation lands last so it wins")
+
+	// The compiled matcher must NOT exclude the force-included tree.
+	assert.False(t, excludes.New(got).MatchRel("Pods/"),
+		"a force-included gitignored dir must be re-included")
+}
+
 func TestEffectiveExclude_FallsBackToBuiltin(t *testing.T) {
 	cm, err := NewConfigManager("/tmp/nonexistent-gortex-test-cm/config.yaml")
 	require.NoError(t, err)

internal/githooks/install.go

@@ -31,7 +31,7 @@ const (
 // SupportedHooks enumerates the hook names that InstallHook accepts.
 // Anything else returns an error so we don't silently scatter our
 // markers into hooks we haven't audited.
-var SupportedHooks = []string{"post-commit", "post-merge"}
+var SupportedHooks = []string{"post-commit", "post-merge", "post-checkout"}
 
 func isSupportedHook(name string) bool {
 	for _, h := range SupportedHooks {
@@ -103,6 +103,17 @@ func (o InstallOpts) withDefaults() InstallOpts {
 // marker block. The body is a `#!/bin/sh` snippet that runs every
 // enabled action and tolerates failures so the hook always completes.
 func hookCommands(hook string, opts InstallOpts) []string {
+	if hook == "post-checkout" {
+		// post-checkout fires on branch switch / clone / file checkout.
+		// Touch the notify file so a running gortex daemon reconciles the
+		// new working-tree state immediately (sub-second) instead of waiting
+		// out its poll interval. Harmless when no daemon is running.
+		return []string{
+			"# Force a running gortex daemon to reconcile after a checkout.",
+			"mkdir -p .gortex 2>/dev/null || true",
+			"touch .gortex/reindex.notify 2>/dev/null || true",
+		}
+	}
 	var cmds []string
 	cmds = append(cmds, fmt.Sprintf("# Auto-regenerate gortex artefacts on %s.", hook))
 	cmds = append(cmds, "# Failures are tolerated so the hook always completes.")

internal/graph/index_state.go

@@ -0,0 +1,46 @@
+package graph
+
+// RepoIndexState is the per-repo freshness provenance recorded at the
+// end of a (re)index: the git revision the graph reflects, whether the
+// working tree was dirty at index time, the Merkle workspace
+// fingerprint that gates global-pass short-circuiting, node/edge counts
+// for the index-plausibility baseline, and a JSON map of the
+// per-language extractor versions that produced the graph.
+//
+// It is the storage half of the FreshnessFact layer; the per-file half
+// lives in the Merkle leaf (the salted content hash) and the file_mtimes
+// ledger.
+type RepoIndexState struct {
+	RepoPrefix        string
+	IndexedSHA        string
+	Dirty             bool
+	IndexedAt         int64  // unix seconds
+	WorkspaceFP       string // Merkle root at index time
+	NodeCount         int
+	EdgeCount         int
+	ExtractorVersions string // JSON-encoded map[string]int
+}
+
+// RepoIndexStateWriter persists the freshness provenance for one repo.
+// Backends without durable state simply do not implement it — the
+// indexer type-asserts and skips the write when absent, exactly like the
+// FileMtime ledger.
+type RepoIndexStateWriter interface {
+	SetRepoIndexState(state RepoIndexState) error
+}
+
+// RepoIndexStateReader reads back the freshness provenance for one repo.
+// The bool is false when no state has been recorded yet (a never-indexed
+// or pre-feature repo), which callers treat as "freshness unknown" — they
+// never block on it.
+type RepoIndexStateReader interface {
+	GetRepoIndexState(repoPrefix string) (RepoIndexState, bool, error)
+}
+
+// DBStatReporter is an optional capability: report the on-disk size of the
+// backing database file and its write-ahead log, in bytes. Surfaced in
+// daemon_health so a runaway WAL high-water mark is observable. In-memory
+// backends do not implement it.
+type DBStatReporter interface {
+	DBStats() (dbBytes, walBytes int64)
+}

internal/graph/store_sqlite/schema.go

@@ -86,6 +86,24 @@ CREATE TABLE IF NOT EXISTS file_mtimes (
     PRIMARY KEY (repo_prefix, file_path)
 ) WITHOUT ROWID;
 
+-- repo_index_state records per-repo freshness provenance written at the
+-- end of a (re)index: the git revision + dirty flag the graph reflects,
+-- the Merkle workspace fingerprint (Tree.Root) that gates global-pass
+-- short-circuiting, node/edge counts for the index-plausibility baseline,
+-- and the JSON per-language extractor versions that produced the graph.
+-- One row per repo_prefix; WITHOUT ROWID — the PK index IS the table,
+-- like file_mtimes / clone_shingles.
+CREATE TABLE IF NOT EXISTS repo_index_state (
+    repo_prefix        TEXT PRIMARY KEY,
+    indexed_sha        TEXT NOT NULL DEFAULT '',
+    dirty              INTEGER NOT NULL DEFAULT 0,
+    indexed_at         INTEGER NOT NULL DEFAULT 0,
+    workspace_fp       TEXT NOT NULL DEFAULT '',
+    node_count         INTEGER NOT NULL DEFAULT 0,
+    edge_count         INTEGER NOT NULL DEFAULT 0,
+    extractor_versions TEXT NOT NULL DEFAULT ''
+) WITHOUT ROWID;
+
 -- clone_shingles is the per-symbol MinHash shingle-set sidecar. Each
 -- function/method node's []uint64 shingle set is stored as a little-
 -- endian BLOB (8 bytes/elem) keyed by node_id so the maintained clone-

internal/graph/store_sqlite/store.go

@@ -42,6 +42,11 @@ import (
 type Store struct {
 	db *sql.DB
 
+	// dbPath is the on-disk SQLite file path, retained for size
+	// telemetry — the WAL high-water mark surfaces in daemon_health so a
+	// runaway -wal is observable rather than silently filling the disk.
+	dbPath string
+
 	// writeMu serialises every mutation. SQLite serialises writers
 	// internally; doing the same on the Go side turns SQLITE_BUSY
 	// contention into clean lock-wait and keeps the conformance
@@ -182,7 +187,7 @@ func Open(path string) (*Store, error) {
 		return nil, fmt.Errorf("sqlite edges_external index: %w", err)
 	}
 
-	s := &Store{db: db}
+	s := &Store{db: db, dbPath: path}
 	// Initialise the bundle cache at construction so its pointer is
 	// never written after Open — concurrent SearchSymbolBundles reads
 	// and SetBundleFingerprints writes then race only on the cache's

internal/graph/store_sqlite/store_dbstat.go

@@ -0,0 +1,21 @@
+package store_sqlite
+
+import "os"
+
+// DBStats returns the on-disk size of the SQLite database file and its
+// write-ahead log, in bytes. A missing file (or a store opened without a
+// path) reports 0 for that component. Surfaced in daemon_health so a
+// runaway WAL high-water mark is observable instead of silently filling
+// the disk.
+func (s *Store) DBStats() (dbBytes, walBytes int64) {
+	if s == nil || s.dbPath == "" {
+		return 0, 0
+	}
+	if fi, err := os.Stat(s.dbPath); err == nil {
+		dbBytes = fi.Size()
+	}
+	if fi, err := os.Stat(s.dbPath + "-wal"); err == nil {
+		walBytes = fi.Size()
+	}
+	return dbBytes, walBytes
+}

internal/graph/store_sqlite/store_dbstat_test.go

@@ -0,0 +1,33 @@
+package store_sqlite_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zzet/gortex/internal/graph"
+	"github.com/zzet/gortex/internal/graph/store_sqlite"
+)
+
+func TestDBStats(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "g.sqlite")
+	s, err := store_sqlite.Open(path)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = s.Close() })
+
+	// Force a write so the DB (and WAL, in WAL mode) carry real bytes.
+	s.AddBatch([]*graph.Node{
+		{ID: "a.go::Foo", Kind: graph.KindFunction, Name: "Foo", FilePath: "a.go"},
+	}, nil)
+
+	dbBytes, walBytes := s.DBStats()
+	require.Greater(t, dbBytes, int64(0), "the on-disk DB file must have nonzero size")
+	require.GreaterOrEqual(t, walBytes, int64(0), "WAL size is non-negative (0 after a checkpoint)")
+
+	// A store with no path (the zero value) reports zero, never panics.
+	var empty store_sqlite.Store
+	db, wal := empty.DBStats()
+	require.Equal(t, int64(0), db)
+	require.Equal(t, int64(0), wal)
+}