fix(#5915 J2 slice-3): deadref/cache/tier/cleanup segment-set gates + restore EmbeddingRef on FB load

internal/cli/branches.go

@@ -233,19 +233,23 @@ func repoBaseForSlug(storeRoot string, repo registry.Repo) string {
 }
 
 // inferTierFromDisk makes a best-effort tier determination from file mtimes
-// without requiring a live daemon. It reads the graph.fb mtime and maps it
-// to HOT/WARM/COLD/EXPIRED using the default TTL windows.
+// without requiring a live daemon. It reads the active graph's mtime and maps
+// it to HOT/WARM/COLD/EXPIRED using the default TTL windows.
+//
+// #5915 J2 slice-3: graph.CurrentGraphMtime is segment-set aware — the prior
+// os.Stat over [graph.CurrentGraphPath(stateDir), graph.json] only ever named
+// a flat .fb path, which is ABSENT for a segment-set ref (graph.<gen>/ dir +
+// manifest.json, no flat .fb). That silently inferred mtime-zero (tier "cold"
+// with a zero lastSeen) for a freshly-indexed segment-set ref.
 func inferTierFromDisk(stateDir string) (tierStr string, lastSeen time.Time) {
-	fbPath := graph.CurrentGraphPath(stateDir) // #5891: resolve active gen
 	jsonPath := filepath.Join(stateDir, "graph.json")
 
 	var mtime time.Time
-	for _, p := range []string{fbPath, jsonPath} {
-		if fi, err := os.Stat(p); err == nil {
-			if fi.ModTime().After(mtime) {
-				mtime = fi.ModTime()
-			}
-		}
+	if mt, ok := graph.CurrentGraphMtime(stateDir); ok {
+		mtime = mt
+	}
+	if fi, err := os.Stat(jsonPath); err == nil && fi.ModTime().After(mtime) {
+		mtime = fi.ModTime()
 	}
 
 	if mtime.IsZero() {

internal/cli/branches_segmentset_test.go

@@ -0,0 +1,102 @@
+package cli
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/cajasmota/grafel/internal/graph"
+	"github.com/cajasmota/grafel/internal/graph/fbwriter"
+)
+
+// writeSegmentSetFixtureForTierTest hand-builds a 1-segment gen dir
+// (graph.<gen>/ with seg-0000.fb + manifest.json) under stateDir and points
+// `current` at it, then stamps the manifest.json mtime — same fixture shape
+// used elsewhere in this slice (internal/daemon/state_path_graphfbexists_test.go,
+// internal/cli/status_stats_segmentset_test.go).
+func writeSegmentSetFixtureForTierTest(t *testing.T, stateDir string, gen uint64, mtime time.Time) {
+	t.Helper()
+	doc := &graph.Document{Repo: "seg", Entities: []graph.Entity{
+		{ID: "aa1", QualifiedName: "p.A", Kind: "function", Name: "A"},
+	}}
+	genDir := filepath.Join(stateDir, graph.GenDirName(gen))
+	name := graph.SegmentFileName(0)
+	if err := fbwriter.WriteAtomic(filepath.Join(genDir, name), doc); err != nil {
+		t.Fatalf("write %s: %v", name, err)
+	}
+	m := &graph.Manifest{FormatVersion: graph.ManifestFormatVersion, Segments: []graph.SegmentMeta{{
+		File: name, Kind: graph.SegmentEntities, EntityCount: 1,
+		MinKey: "aa1", MaxKey: "aa1",
+	}}}
+	if err := graph.WriteManifest(genDir, m); err != nil {
+		t.Fatalf("write manifest: %v", err)
+	}
+	manifestPath := filepath.Join(genDir, graph.ManifestFileName)
+	if err := os.Chtimes(manifestPath, mtime, mtime); err != nil {
+		t.Fatal(err)
+	}
+	if err := graph.WriteCurrentPointerRaw(stateDir, graph.GenDirName(gen)); err != nil {
+		t.Fatalf("write current: %v", err)
+	}
+}
+
+// TestInferTierFromDisk_SegmentSet is the RED test for #5915 J2 slice-3: a
+// segment-set state dir (graph.<gen>/ dir + manifest.json, no flat graph.fb)
+// indexed a minute ago must infer tier "hot" with a non-zero lastSeen. Before
+// the fix, inferTierFromDisk stat'd only [graph.CurrentGraphPath(stateDir),
+// graph.json] — both absent for a segment-set dir — so it silently inferred
+// tier "cold" with a zero lastSeen (mtime never found).
+func TestInferTierFromDisk_SegmentSet(t *testing.T) {
+	stateDir := t.TempDir()
+	mtime := time.Now().Add(-1 * time.Minute)
+	writeSegmentSetFixtureForTierTest(t, stateDir, 5, mtime)
+
+	tierStr, lastSeen := inferTierFromDisk(stateDir)
+
+	if tierStr != "hot" {
+		t.Errorf("tierStr = %q, want %q", tierStr, "hot")
+	}
+	if lastSeen.IsZero() {
+		t.Fatal("lastSeen is zero, want non-zero (segment-set manifest.json mtime)")
+	}
+	if diff := lastSeen.Sub(mtime); diff < -time.Second || diff > time.Second {
+		t.Errorf("lastSeen = %v, want ~%v", lastSeen, mtime)
+	}
+}
+
+// TestInferTierFromDisk_SingleFileParity guards the single-gen-file (and
+// legacy flat) resolution is byte-identical to the pre-fix behavior.
+func TestInferTierFromDisk_SingleFileParity(t *testing.T) {
+	stateDir := t.TempDir()
+	flat := filepath.Join(stateDir, "graph.fb")
+	mtime := time.Now().Add(-1 * time.Minute)
+	if err := os.WriteFile(flat, []byte("legacy"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Chtimes(flat, mtime, mtime); err != nil {
+		t.Fatal(err)
+	}
+
+	tierStr, lastSeen := inferTierFromDisk(stateDir)
+
+	if tierStr != "hot" {
+		t.Errorf("tierStr = %q, want %q", tierStr, "hot")
+	}
+	if diff := lastSeen.Sub(mtime); diff < -time.Second || diff > time.Second {
+		t.Errorf("lastSeen = %v, want ~%v", lastSeen, mtime)
+	}
+}
+
+// TestInferTierFromDisk_NeverIndexed guards a never-indexed state dir infers
+// tier "cold" with a zero lastSeen.
+func TestInferTierFromDisk_NeverIndexed(t *testing.T) {
+	stateDir := t.TempDir()
+	tierStr, lastSeen := inferTierFromDisk(stateDir)
+	if tierStr != "cold" {
+		t.Errorf("tierStr = %q, want %q", tierStr, "cold")
+	}
+	if !lastSeen.IsZero() {
+		t.Errorf("lastSeen = %v, want zero", lastSeen)
+	}
+}

internal/cli/cleanup.go

@@ -120,6 +120,18 @@ func runCleanup(w io.Writer, dryRun bool, ttlDays int) error {
 // plus (legacy) <repo>/.grafel/graph.fb for repos indexed before PH1a.
 // We walk the daemon store directory; .grafel sidecar graphs are not
 // swept (they are repo-local, managed by the repo owner).
+//
+// #5915 J2 slice-3: a segment-set ref (graph.<gen>/ dir + seg-NNNN.fb +
+// manifest.json, no flat .fb) has NO file matching graph.IsGraphFileName —
+// seg-NNNN.fb matches neither the legacy flat name nor the graph.<gen>.fb
+// pattern — so the walk skipped every segment-set ref's state dir entirely.
+// Its entities' embedding refs were never added to `active`, so the sweep
+// below (embed cache TTL GC) could reap embedding cache entries still
+// referenced by a live segment-set graph. The walk now ALSO recognises
+// manifest.json as a segment-set's canonical marker file and, when it is the
+// active generation's manifest, loads the ref's STATE dir (the manifest's
+// gen dir's parent) exactly once — mirroring the single-file branch's
+// dedup-to-the-active-generation guard.
 func collectActiveEmbeddingHashes() (map[string]bool, error) {
 	h, err := registry.HomeDir()
 	if err != nil {
@@ -136,17 +148,42 @@ func collectActiveEmbeddingHashes() (map[string]bool, error) {
 		if d.IsDir() {
 			return nil
 		}
-		// #5891: recognise generation files (graph.<gen>.fb), and process each
-		// state dir exactly once — only when this path is the ACTIVE generation
-		// resolved by the pointer (or the legacy flat graph.fb) — so a dir with
-		// current+previous gens isn't walked twice.
-		if !graph.IsGraphFileName(filepath.Base(path)) {
-			return nil
-		}
-		if path != graph.CurrentGraphPath(filepath.Dir(path)) {
+
+		var stateDir string
+		switch {
+		case filepath.Base(path) == graph.ManifestFileName:
+			// Candidate segment-set marker: path = <stateDir>/graph.<gen>/manifest.json.
+			genDir := filepath.Dir(path)
+			candidate := filepath.Dir(genDir)
+			desc, dErr := graph.CurrentGraphDescriptor(candidate)
+			if dErr != nil || desc.Kind != graph.GraphSegmentSet {
+				return nil
+			}
+			// Only the ACTIVE generation's manifest counts, so a stale
+			// previous-gen manifest (kept around by GCStaleGens' retention
+			// window) isn't double-processed.
+			if filepath.Join(desc.GenDir, graph.ManifestFileName) != path {
+				return nil
+			}
+			stateDir = candidate
+
+		case graph.IsGraphFileName(filepath.Base(path)):
+			// #5891: recognise generation files (graph.<gen>.fb), and process
+			// each state dir exactly once — only when this path is the ACTIVE
+			// generation resolved by the pointer (or the legacy flat
+			// graph.fb) — so a dir with current+previous gens isn't walked
+			// twice.
+			candidate := filepath.Dir(path)
+			if path != graph.CurrentGraphPath(candidate) {
+				return nil
+			}
+			stateDir = candidate
+
+		default:
 			return nil
 		}
-		doc, loadErr := graph.LoadGraphFromDir(filepath.Dir(path))
+
+		doc, loadErr := graph.LoadGraphFromDir(stateDir)
 		if loadErr != nil {
 			return nil // corrupt/missing — skip
 		}

internal/cli/cleanup_segmentset_test.go

@@ -0,0 +1,94 @@
+package cli
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/cajasmota/grafel/internal/graph"
+	"github.com/cajasmota/grafel/internal/graph/fbwriter"
+)
+
+// writeSegmentSetRefForEmbeddingWalk hand-builds a 1-segment gen dir
+// (graph.<gen>/ with seg-0000.fb + manifest.json) under refDir, points
+// `current` at it, and stamps the single entity with embeddingRef — mirroring
+// the fixture builders used elsewhere in this slice
+// (internal/daemon/state_path_graphfbexists_test.go). No flat graph.fb is
+// ever written, matching a real segment-set ref.
+func writeSegmentSetRefForEmbeddingWalk(t *testing.T, refDir string, gen uint64, embeddingRef string) {
+	t.Helper()
+	doc := &graph.Document{Repo: "seg", Entities: []graph.Entity{
+		{ID: "aa1", QualifiedName: "p.A", Kind: "function", Name: "A", EmbeddingRef: embeddingRef},
+	}}
+	genDir := filepath.Join(refDir, graph.GenDirName(gen))
+	name := graph.SegmentFileName(0)
+	if err := fbwriter.WriteAtomic(filepath.Join(genDir, name), doc); err != nil {
+		t.Fatalf("write %s: %v", name, err)
+	}
+	m := &graph.Manifest{FormatVersion: graph.ManifestFormatVersion, Segments: []graph.SegmentMeta{{
+		File: name, Kind: graph.SegmentEntities, EntityCount: 1,
+		MinKey: "aa1", MaxKey: "aa1",
+	}}}
+	if err := graph.WriteManifest(genDir, m); err != nil {
+		t.Fatalf("write manifest: %v", err)
+	}
+	if err := graph.WriteCurrentPointerRaw(refDir, graph.GenDirName(gen)); err != nil {
+		t.Fatalf("write current: %v", err)
+	}
+}
+
+// TestCollectActiveEmbeddingHashes_SegmentSet is the RED test for #5915 J2
+// slice-3: a live segment-set ref's embedding_ref values must be reported
+// active. Before the fix, seg-NNNN.fb matches neither graph.IsGraphFileName's
+// legacy-flat nor graph.<gen>.fb pattern, so the walk skipped a segment-set
+// ref's state dir entirely — its embedding refs were never collected as
+// active, so a subsequent TTL-based embedding-cache sweep could reap a
+// .vec cache entry that a live segment-set graph still references.
+func TestCollectActiveEmbeddingHashes_SegmentSet(t *testing.T) {
+	home := t.TempDir()
+	t.Setenv("GRAFEL_HOME", home)
+
+	refDir := filepath.Join(home, "store", "myrepo-abc123", "refs", "main")
+	if err := os.MkdirAll(refDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	writeSegmentSetRefForEmbeddingWalk(t, refDir, 4, "sha256:seg-live-hash")
+
+	active, err := collectActiveEmbeddingHashes()
+	if err != nil {
+		t.Fatalf("collectActiveEmbeddingHashes: %v", err)
+	}
+	if !active["sha256:seg-live-hash"] {
+		t.Fatalf("active = %v, want it to include the segment-set ref's embedding hash", active)
+	}
+}
+
+// TestCollectActiveEmbeddingHashes_SingleFileParity guards the single-gen-file
+// (and legacy flat) resolution is unchanged: exactly one state dir's
+// embedding refs are collected, not double-counted across generations.
+func TestCollectActiveEmbeddingHashes_SingleFileParity(t *testing.T) {
+	home := t.TempDir()
+	t.Setenv("GRAFEL_HOME", home)
+
+	refDir := filepath.Join(home, "store", "myrepo-abc123", "refs", "main")
+	if err := os.MkdirAll(refDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	doc := &graph.Document{Repo: "single", Entities: []graph.Entity{
+		{ID: "aaaa0001", QualifiedName: "p.A", Kind: "function", Name: "A", EmbeddingRef: "sha256:single-live-hash"},
+	}}
+	if err := fbwriter.WriteAtomic(filepath.Join(refDir, graph.GenFileName(1)), doc); err != nil {
+		t.Fatal(err)
+	}
+	if err := graph.WriteCurrentPointer(refDir, graph.GenFileName(1)); err != nil {
+		t.Fatal(err)
+	}
+
+	active, err := collectActiveEmbeddingHashes()
+	if err != nil {
+		t.Fatalf("collectActiveEmbeddingHashes: %v", err)
+	}
+	if !active["sha256:single-live-hash"] {
+		t.Fatalf("active = %v, want it to include the single-file ref's embedding hash (parity)", active)
+	}
+}

internal/daemon/algo/cache.go

@@ -160,21 +160,27 @@ func cacheKey(stateDir string) string { return stateDir }
 // staleness.
 func readFromDisk(stateDir string) (*Results, bool) {
 	cachePath := filepath.Join(stateDir, cacheFileName)
-	graphPath := graph.CurrentGraphPath(stateDir) // #5891: resolve active gen
+	// #5915 J2 slice-3: graph.CurrentGraphMtime is segment-set aware — the
+	// prior os.Stat(graph.CurrentGraphPath(stateDir)) only ever named a flat
+	// .fb path, which is ABSENT for a segment-set stateDir (graph.<gen>/ dir +
+	// manifest.json, no flat .fb). That made the disk cache permanently
+	// unvalidatable for a segment-set repo: readFromDisk always missed
+	// (forcing recompute every call) and writeToDisk always errored (the
+	// cache was never persisted).
+	graphMtime, ok := graph.CurrentGraphMtime(stateDir)
+	if !ok {
+		return nil, false // no active graph found; nothing to validate against
+	}
 
 	cacheInfo, err := os.Stat(cachePath)
 	if err != nil {
 		return nil, false // cache file does not exist
 	}
-	graphInfo, err := os.Stat(graphPath)
-	if err != nil {
-		return nil, false // graph.fb not found; nothing to validate against
-	}
 
-	// Staleness check: graph.fb must not be newer than the cache.
+	// Staleness check: the active graph must not be newer than the cache.
 	// Use a 1-second tolerance to absorb filesystem timestamp granularity.
-	if graphInfo.ModTime().After(cacheInfo.ModTime().Add(time.Second)) {
-		return nil, false // graph.fb was updated after we cached
+	if graphMtime.After(cacheInfo.ModTime().Add(time.Second)) {
+		return nil, false // graph was updated after we cached
 	}
 
 	data, err := os.ReadFile(cachePath)
@@ -185,8 +191,8 @@ func readFromDisk(stateDir string) (*Results, bool) {
 	if err := json.Unmarshal(data, &env); err != nil {
 		return nil, false
 	}
-	// Validate the embedded mtime matches the current graph.fb mtime.
-	if env.GraphMtime != graphInfo.ModTime().UnixNano() {
+	// Validate the embedded mtime matches the current graph's mtime.
+	if env.GraphMtime != graphMtime.UnixNano() {
 		return nil, false
 	}
 	return &env.Results, true
@@ -195,14 +201,14 @@ func readFromDisk(stateDir string) (*Results, bool) {
 // writeToDisk atomically writes the results to <stateDir>/algo_results.fb.
 // Uses a temp-file + rename for crash-safety.
 func writeToDisk(stateDir string, r *Results) error {
-	graphPath := graph.CurrentGraphPath(stateDir) // #5891: resolve active gen
-	graphInfo, err := os.Stat(graphPath)
-	if err != nil {
-		return fmt.Errorf("stat graph.fb: %w", err)
+	// #5915 J2 slice-3: segment-set aware (see readFromDisk).
+	graphMtime, ok := graph.CurrentGraphMtime(stateDir)
+	if !ok {
+		return fmt.Errorf("stat graph: no active graph found in %s", stateDir)
 	}
 
 	env := envelope{
-		GraphMtime: graphInfo.ModTime().UnixNano(),
+		GraphMtime: graphMtime.UnixNano(),
 		Results:    *r,
 	}
 	data, err := json.Marshal(&env)