feat(#5901): gen-dir read substrate — manifest, segment descriptor, key-range routing (dark)

internal/daemon/mcp/graph_cache.go

@@ -59,10 +59,16 @@ var ErrUnknownRef = errors.New("graph cache: refusing to load refs/_unknown sent
 // the explicit Invalidate hook from the scheduler is the primary path,
 // mtime check is the belt-and-braces fallback.
 type Entry struct {
-	Path   string
-	Reader *fbreader.Reader
-
-	mtime int64 // unix nano of the underlying graph.fb at open time
+	Path string
+	// Reader is the mmap-backed read surface. #5901: this is now the
+	// fbreader.GraphView interface rather than a concrete *fbreader.Reader, so a
+	// resident handle may back EITHER a single-file graph.<gen>.fb (a *Reader,
+	// the unchanged common path) OR a multi-segment graph.<gen>/ dir (a
+	// *MultiReader over N segment mmaps). Every consumer uses only the shared
+	// read surface, so the change is transparent to handlers.
+	Reader fbreader.GraphView
+
+	mtime int64 // unix nano of the underlying graph.fb (or gen dir) at open time
 	refs  int32 // outstanding Borrow callers; eviction waits for zero
 }
 
@@ -143,7 +149,7 @@ func NewCache(capacity int) *Cache {
 // can reclaim the handle.
 //
 // Stale entries (mtime drift) are transparently reopened.
-func (c *Cache) Get(path string) (*fbreader.Reader, func(), error) {
+func (c *Cache) Get(path string) (fbreader.GraphView, func(), error) {
 	if c.closed.Load() {
 		return nil, func() {}, ErrCacheClosed
 	}
@@ -250,7 +256,7 @@ open:
 //
 // Returns (nil, noop, ErrCacheClosed) when the cache has been closed.
 // Returns (nil, noop, ErrUnknownRef) when ref resolves to refs/_unknown/.
-func (c *Cache) GetForRepoRef(repoPath, ref string) (*fbreader.Reader, func(), error) {
+func (c *Cache) GetForRepoRef(repoPath, ref string) (fbreader.GraphView, func(), error) {
 	stateDir := daemon.StateDirForRepoRef(repoPath, ref)
 	// Refuse to eagerly load the _unknown sentinel into heap (#2141 root-cause D).
 	// The sentinel dir is created for repos indexed before PH1b; its graph.fb is
@@ -262,7 +268,16 @@ func (c *Cache) GetForRepoRef(repoPath, ref string) (*fbreader.Reader, func(), e
 	// graph.fb fallback for un-migrated repos). The cache key remains the
 	// absolute resolved path, so hit/miss semantics are unchanged; a gen swap
 	// yields a new key → cache miss → fresh mmap of the new generation.
+	//
+	// #5901: a segment-set generation resolves to a gen DIR (graph.<gen>/), not
+	// a .fb file. Use the descriptor to key on the gen dir in that case so the
+	// cache opens a MultiReader over its segments; single-file/legacy keep
+	// keying on the resolved .fb path exactly as before. A malformed segment-set
+	// manifest surfaces as an open error (OpenErrors++), not a crash.
 	fbPath := graph.CurrentGraphPath(stateDir)
+	if desc, dErr := graph.CurrentGraphDescriptor(stateDir); dErr == nil && desc.Kind == graph.GraphSegmentSet {
+		fbPath = desc.GenDir
+	}
 	r, release, err := c.Get(fbPath)
 	if err == nil {
 		c.fireAccessHook(repoPath, ref)
@@ -400,18 +415,59 @@ func (c *Cache) releaser(ent *Entry) func() {
 //     reader (Get discards the handle on any openReader error before inserting
 //     it into the LRU). This is the reader-side half of the same migration the
 //     status-plane's ReindexRequiredReason flag reports.
-func openReader(path string) (*fbreader.Reader, int64, error) {
+func openReader(path string) (fbreader.GraphView, int64, error) {
 	info, err := os.Stat(path)
 	if err != nil {
 		return nil, 0, fmt.Errorf("graph cache: stat %s: %w", path, err)
 	}
+	// #5901: a resolved path may name a MULTI-SEGMENT gen dir (graph.<gen>/)
+	// rather than a single graph.<gen>.fb file. GetForRepoRef keys segment-sets
+	// on the gen dir, so detect the directory case and open a MultiReader over
+	// its manifest'd segments (with key routing). The single-file branch below
+	// is unchanged. mtime tracks the dir's mtime — the gen dir, like a gen file,
+	// is immutable once written (a reindex allocates a NEW gen), so this is a
+	// sound staleness signal.
+	if info.IsDir() {
+		v, err := openSegmentSetAndGate(path)
+		if err != nil {
+			return nil, 0, err
+		}
+		return v, info.ModTime().UnixNano(), nil
+	}
 	r, err := openAndGate(path)
 	if err != nil {
 		return nil, 0, err
 	}
 	return r, info.ModTime().UnixNano(), nil
 }
 
+// openSegmentSetAndGate opens a multi-segment gen dir as a unified MultiReader
+// and enforces the SAME crash-recover + fbversion gate as openAndGate. It is
+// the segment-set half of the #5907 cache seam: a corrupt manifest or a
+// stale-format segment degrades to a plain error (a cache miss), never a daemon
+// crash, and never a cached stale-format handle.
+func openSegmentSetAndGate(genDir string) (v fbreader.GraphView, err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			if v != nil {
+				_ = v.Close()
+				v = nil
+			}
+			err = fmt.Errorf("graph cache: recovered panic opening segment-set %s: %v", genDir, rec)
+		}
+	}()
+	v, err = graph.OpenSegmentReader(genDir)
+	if err != nil {
+		return nil, err
+	}
+	if ver := v.Version(); ver < fbversion.Version {
+		_ = v.Close()
+		return nil, fmt.Errorf("graph cache: %w",
+			&graph.FormatVersionError{Found: ver, Required: fbversion.Version})
+	}
+	return v, nil
+}
+
 // openAndGate opens path and enforces the format-version gate, returning a
 // usable reader only for an on-disk format this binary can serve. It is the
 // crash-prevention seam for the zero-copy MCP cache (#5907):

internal/daemon/mcp/graph_cache_segmentset_test.go

@@ -0,0 +1,97 @@
+package mcp
+
+import (
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/cajasmota/grafel/internal/graph"
+	"github.com/cajasmota/grafel/internal/graph/fbwriter"
+)
+
+// writeSegmentSetFixture hand-builds a 2-segment gen dir (graph.<gen>/ with
+// seg-0000.fb + seg-0001.fb + manifest.json) under stateDir and points the
+// `current` pointer at the gen dir. No producer emits this yet (#5901 dark
+// substrate) so the test constructs it directly. Returns the gen dir.
+func writeSegmentSetFixture(t *testing.T, stateDir string, gen uint64) string {
+	t.Helper()
+	docs := []*graph.Document{
+		{Repo: "seg", Entities: []graph.Entity{
+			{ID: "aa1", QualifiedName: "p.A", Kind: "function", Name: "A"},
+			{ID: "aa2", QualifiedName: "p.B", Kind: "struct", Name: "B"},
+		}, Relationships: []graph.Relationship{{FromID: "aa1", ToID: "mm1", Kind: "CALLS"}}},
+		{Repo: "seg", Entities: []graph.Entity{
+			{ID: "mm1", QualifiedName: "p.M", Kind: "function", Name: "M"},
+			{ID: "mm2", QualifiedName: "p.N", Kind: "struct", Name: "N"},
+		}, Relationships: []graph.Relationship{{FromID: "mm1", ToID: "aa1", Kind: "REFERENCES"}}},
+	}
+	genDir := filepath.Join(stateDir, graph.GenDirName(gen))
+	m := &graph.Manifest{FormatVersion: graph.ManifestFormatVersion}
+	for i, doc := range docs {
+		name := graph.SegmentFileName(i)
+		if err := fbwriter.WriteAtomic(filepath.Join(genDir, name), doc); err != nil {
+			t.Fatalf("write %s: %v", name, err)
+		}
+		ids := make([]string, 0, len(doc.Entities))
+		for _, e := range doc.Entities {
+			ids = append(ids, e.ID)
+		}
+		sort.Strings(ids)
+		m.Segments = append(m.Segments, graph.SegmentMeta{
+			File: name, Kind: graph.SegmentEntities,
+			EntityCount: len(doc.Entities), RelCount: len(doc.Relationships),
+			MinKey: ids[0], MaxKey: ids[len(ids)-1],
+		})
+	}
+	if err := graph.WriteManifest(genDir, m); err != nil {
+		t.Fatalf("write manifest: %v", err)
+	}
+	if err := graph.WriteCurrentPointerRaw(stateDir, graph.GenDirName(gen)); err != nil {
+		t.Fatalf("write current: %v", err)
+	}
+	return genDir
+}
+
+// TestCache_SegmentSetGenDir: the daemon graph cache opens a multi-segment gen
+// dir (keyed by the gen dir path, as GetForRepoRef would resolve it) and the
+// returned GraphView reads across all segments, including a cross-segment
+// relationship endpoint.
+func TestCache_SegmentSetGenDir(t *testing.T) {
+	stateDir := t.TempDir()
+	genDir := writeSegmentSetFixture(t, stateDir, 3)
+
+	c := NewCache(4)
+	// Windows teardown: close the cache (unmaps every segment) before the
+	// t.TempDir cleanup unlinks the gen dir.
+	t.Cleanup(func() { _ = c.Close() })
+
+	r, release, err := c.Get(genDir)
+	if err != nil {
+		t.Fatalf("Get(gen dir): %v", err)
+	}
+	defer release()
+
+	if got := r.EntityCount(); got != 4 {
+		t.Errorf("EntityCount across segments = %d, want 4", got)
+	}
+	if got := r.RelationshipCount(); got != 2 {
+		t.Errorf("RelationshipCount across segments = %d, want 2", got)
+	}
+	for _, id := range []string{"aa1", "aa2", "mm1", "mm2"} {
+		if e := r.LookupEntityByID(id); e == nil || string(e.Id()) != id {
+			t.Errorf("LookupEntityByID(%q) failed across segments: %v", id, e)
+		}
+	}
+	// A second Get is a cache hit on the same gen-dir key.
+	r2, rel2, err := c.Get(genDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rel2()
+	if r2 != r {
+		t.Fatal("expected cache hit (same GraphView) for the gen-dir key")
+	}
+	if s := c.Stats(); s.Hits != 1 {
+		t.Fatalf("stats after hit: %+v", s)
+	}
+}

internal/daemon/state_path.go

@@ -469,6 +469,25 @@ func findGraphFileInDir(dir string) (path string, modtime int64) {
 	// linchpin: because writers stop bumping a fixed graph.fb, FindGraphFile /
 	// FindGraphFileAnyRef — and therefore statuswriter's GraphFBMtime — MUST
 	// stat the resolved gen file so a completed rebuild's mtime keeps advancing.
+	//
+	// #5901 segment-set: when the active graph is the multi-segment gen-dir
+	// layout, the resolved artifact is a directory (graph.<gen>/), not a .fb
+	// file. Return the gen dir as the graph "path" (its parent is the state dir,
+	// so lr.GraphFile → filepath.Dir → the correct state dir for the Doc load)
+	// and use the manifest.json mtime as the freshness signal (a real file whose
+	// mtime advances on each rebuild). Downstream readers route on this via the
+	// descriptor; the mmap zero-copy cutover correctly declines a dir (see
+	// internal/mcp/state.go's .fb-ext guard) and serves the segment-set Document.
+	if desc, dErr := graph.CurrentGraphDescriptor(dir); dErr == nil && desc.Kind == graph.GraphSegmentSet {
+		if fi, err := os.Stat(filepath.Join(desc.GenDir, graph.ManifestFileName)); err == nil {
+			segMtime := fi.ModTime().UnixNano()
+			jsonPath := filepath.Join(dir, "graph.json")
+			if jInfo, jErr := os.Stat(jsonPath); jErr == nil && jInfo.ModTime().UnixNano() > segMtime {
+				return jsonPath, jInfo.ModTime().UnixNano()
+			}
+			return desc.GenDir, segMtime
+		}
+	}
 	fbPath := graph.CurrentGraphPath(dir)
 	jsonPath := filepath.Join(dir, "graph.json")
 

internal/graph/fbreader/multireader.go

@@ -41,16 +41,79 @@ import (
 // results, same iteration order) — see TestMultiReaderSingleSegmentParity.
 type MultiReader struct {
 	segs []*Reader
+	// ranges, when non-nil, is parallel to segs and carries each segment's
+	// entity-ID key range from the gen-dir manifest. LookupEntityByID uses it
+	// to SKIP segments that cannot contain the looked-up key, avoiding the O(S)
+	// fan-out (decision 4 of #5901). nil ⇒ no routing info ⇒ every segment is
+	// searched (the OpenSegments legacy behaviour, unchanged).
+	ranges []KeyRange
 }
 
+// KeyRange is the entity-ID key coverage of one segment, as recorded in the
+// gen-dir manifest (SegmentMeta MinKey/MaxKey + a HasEntities flag derived
+// from EntityCount>0). LookupEntityByID consults it to skip segments that
+// provably cannot hold a key.
+//
+// Semantics:
+//   - HasEntities==false ⇒ the segment carries NO entities (a pure-relationship
+//     segment) ⇒ every entity lookup skips it.
+//   - HasEntities==true with Min=="" and Max=="" ⇒ range unknown ⇒ never skip
+//     (safe fallback: search it).
+//   - Otherwise the (lexicographic, inclusive) [Min,Max] window gates the key.
+type KeyRange struct {
+	HasEntities bool
+	Min         string
+	Max         string
+}
+
+// contains reports whether id could live in a segment with this key range.
+func (kr KeyRange) contains(id string) bool {
+	if !kr.HasEntities {
+		return false
+	}
+	if kr.Min == "" && kr.Max == "" {
+		return true // unknown bounds — cannot safely skip
+	}
+	if kr.Min != "" && id < kr.Min {
+		return false
+	}
+	if kr.Max != "" && id > kr.Max {
+		return false
+	}
+	return true
+}
+
+// SegmentSearchHook, when non-nil, is invoked with the segment index each time
+// LookupEntityByID actually performs a binary search on that segment (i.e. NOT
+// skipped by key routing). It is a test/observability seam — nil in production,
+// where it adds a single nil-check on the lookup path — set by the routing
+// tests to assert out-of-range segments are pruned. Not safe for concurrent
+// mutation; set it only from single-threaded test setup.
+var SegmentSearchHook func(seg int)
+
 // OpenSegments memory-maps every path in paths (in the given order) and
 // returns a MultiReader over all of them. paths must be non-empty. If any
 // segment fails to open, every already-opened segment is closed before
 // the error is returned (no leaked mappings on a partial failure).
+//
+// This constructor attaches NO key ranges, so LookupEntityByID fans out across
+// every segment (O(S)). Use OpenSegmentsWithRanges to enable manifest-driven
+// segment skipping.
 func OpenSegments(paths []string) (*MultiReader, error) {
+	return OpenSegmentsWithRanges(paths, nil)
+}
+
+// OpenSegmentsWithRanges is OpenSegments plus per-segment key ranges (from the
+// gen-dir manifest) used to prune LookupEntityByID. When ranges is non-nil it
+// MUST have the same length as paths (one range per segment, same order);
+// a length mismatch is an error. A nil ranges disables routing (== OpenSegments).
+func OpenSegmentsWithRanges(paths []string, ranges []KeyRange) (*MultiReader, error) {
 	if len(paths) == 0 {
 		return nil, fmt.Errorf("fbreader: OpenSegments requires at least one path")
 	}
+	if ranges != nil && len(ranges) != len(paths) {
+		return nil, fmt.Errorf("fbreader: OpenSegmentsWithRanges: %d ranges for %d paths", len(ranges), len(paths))
+	}
 	segs := make([]*Reader, 0, len(paths))
 	for _, p := range paths {
 		r, err := Open(p)
@@ -62,7 +125,7 @@ func OpenSegments(paths []string) (*MultiReader, error) {
 		}
 		segs = append(segs, r)
 	}
-	return &MultiReader{segs: segs}, nil
+	return &MultiReader{segs: segs, ranges: ranges}, nil
 }
 
 // Close releases every segment's underlying mmap. It closes all segments
@@ -127,7 +190,16 @@ func (m *MultiReader) LookupEntityByID(id string) *fb.Entity {
 	if m == nil {
 		return nil
 	}
-	for _, s := range m.segs {
+	for i, s := range m.segs {
+		// Decision 4: when manifest key ranges are attached, skip any segment
+		// whose [Min,Max] window cannot contain id (or that holds no entities).
+		// This turns the O(S) fan-out into "search only plausible segments".
+		if m.ranges != nil && !m.ranges[i].contains(id) {
+			continue
+		}
+		if SegmentSearchHook != nil {
+			SegmentSearchHook(i)
+		}
 		if e := s.LookupEntityByID(id); e != nil {
 			return e
 		}