diff --git a/internal/daemon/mcp/graph_cache.go b/internal/daemon/mcp/graph_cache.go index 2632a3ce3..762284ca9 100644 --- a/internal/daemon/mcp/graph_cache.go +++ b/internal/daemon/mcp/graph_cache.go @@ -59,10 +59,16 @@ var ErrUnknownRef = errors.New("graph cache: refusing to load refs/_unknown sent // the explicit Invalidate hook from the scheduler is the primary path, // mtime check is the belt-and-braces fallback. type Entry struct { - Path string - Reader *fbreader.Reader - - mtime int64 // unix nano of the underlying graph.fb at open time + Path string + // Reader is the mmap-backed read surface. #5901: this is now the + // fbreader.GraphView interface rather than a concrete *fbreader.Reader, so a + // resident handle may back EITHER a single-file graph..fb (a *Reader, + // the unchanged common path) OR a multi-segment graph./ dir (a + // *MultiReader over N segment mmaps). Every consumer uses only the shared + // read surface, so the change is transparent to handlers. + Reader fbreader.GraphView + + mtime int64 // unix nano of the underlying graph.fb (or gen dir) at open time refs int32 // outstanding Borrow callers; eviction waits for zero } @@ -143,7 +149,7 @@ func NewCache(capacity int) *Cache { // can reclaim the handle. // // Stale entries (mtime drift) are transparently reopened. -func (c *Cache) Get(path string) (*fbreader.Reader, func(), error) { +func (c *Cache) Get(path string) (fbreader.GraphView, func(), error) { if c.closed.Load() { return nil, func() {}, ErrCacheClosed } @@ -250,7 +256,7 @@ open: // // Returns (nil, noop, ErrCacheClosed) when the cache has been closed. // Returns (nil, noop, ErrUnknownRef) when ref resolves to refs/_unknown/. -func (c *Cache) GetForRepoRef(repoPath, ref string) (*fbreader.Reader, func(), error) { +func (c *Cache) GetForRepoRef(repoPath, ref string) (fbreader.GraphView, func(), error) { stateDir := daemon.StateDirForRepoRef(repoPath, ref) // Refuse to eagerly load the _unknown sentinel into heap (#2141 root-cause D). // The sentinel dir is created for repos indexed before PH1b; its graph.fb is @@ -262,7 +268,16 @@ func (c *Cache) GetForRepoRef(repoPath, ref string) (*fbreader.Reader, func(), e // graph.fb fallback for un-migrated repos). The cache key remains the // absolute resolved path, so hit/miss semantics are unchanged; a gen swap // yields a new key → cache miss → fresh mmap of the new generation. + // + // #5901: a segment-set generation resolves to a gen DIR (graph./), not + // a .fb file. Use the descriptor to key on the gen dir in that case so the + // cache opens a MultiReader over its segments; single-file/legacy keep + // keying on the resolved .fb path exactly as before. A malformed segment-set + // manifest surfaces as an open error (OpenErrors++), not a crash. fbPath := graph.CurrentGraphPath(stateDir) + if desc, dErr := graph.CurrentGraphDescriptor(stateDir); dErr == nil && desc.Kind == graph.GraphSegmentSet { + fbPath = desc.GenDir + } r, release, err := c.Get(fbPath) if err == nil { c.fireAccessHook(repoPath, ref) @@ -400,11 +415,25 @@ func (c *Cache) releaser(ent *Entry) func() { // reader (Get discards the handle on any openReader error before inserting // it into the LRU). This is the reader-side half of the same migration the // status-plane's ReindexRequiredReason flag reports. -func openReader(path string) (*fbreader.Reader, int64, error) { +func openReader(path string) (fbreader.GraphView, int64, error) { info, err := os.Stat(path) if err != nil { return nil, 0, fmt.Errorf("graph cache: stat %s: %w", path, err) } + // #5901: a resolved path may name a MULTI-SEGMENT gen dir (graph./) + // rather than a single graph..fb file. GetForRepoRef keys segment-sets + // on the gen dir, so detect the directory case and open a MultiReader over + // its manifest'd segments (with key routing). The single-file branch below + // is unchanged. mtime tracks the dir's mtime — the gen dir, like a gen file, + // is immutable once written (a reindex allocates a NEW gen), so this is a + // sound staleness signal. + if info.IsDir() { + v, err := openSegmentSetAndGate(path) + if err != nil { + return nil, 0, err + } + return v, info.ModTime().UnixNano(), nil + } r, err := openAndGate(path) if err != nil { return nil, 0, err @@ -412,6 +441,33 @@ func openReader(path string) (*fbreader.Reader, int64, error) { return r, info.ModTime().UnixNano(), nil } +// openSegmentSetAndGate opens a multi-segment gen dir as a unified MultiReader +// and enforces the SAME crash-recover + fbversion gate as openAndGate. It is +// the segment-set half of the #5907 cache seam: a corrupt manifest or a +// stale-format segment degrades to a plain error (a cache miss), never a daemon +// crash, and never a cached stale-format handle. +func openSegmentSetAndGate(genDir string) (v fbreader.GraphView, err error) { + defer func() { + if rec := recover(); rec != nil { + if v != nil { + _ = v.Close() + v = nil + } + err = fmt.Errorf("graph cache: recovered panic opening segment-set %s: %v", genDir, rec) + } + }() + v, err = graph.OpenSegmentReader(genDir) + if err != nil { + return nil, err + } + if ver := v.Version(); ver < fbversion.Version { + _ = v.Close() + return nil, fmt.Errorf("graph cache: %w", + &graph.FormatVersionError{Found: ver, Required: fbversion.Version}) + } + return v, nil +} + // openAndGate opens path and enforces the format-version gate, returning a // usable reader only for an on-disk format this binary can serve. It is the // crash-prevention seam for the zero-copy MCP cache (#5907): diff --git a/internal/daemon/mcp/graph_cache_segmentset_test.go b/internal/daemon/mcp/graph_cache_segmentset_test.go new file mode 100644 index 000000000..ffcb50620 --- /dev/null +++ b/internal/daemon/mcp/graph_cache_segmentset_test.go @@ -0,0 +1,97 @@ +package mcp + +import ( + "path/filepath" + "sort" + "testing" + + "github.com/cajasmota/grafel/internal/graph" + "github.com/cajasmota/grafel/internal/graph/fbwriter" +) + +// writeSegmentSetFixture hand-builds a 2-segment gen dir (graph./ with +// seg-0000.fb + seg-0001.fb + manifest.json) under stateDir and points the +// `current` pointer at the gen dir. No producer emits this yet (#5901 dark +// substrate) so the test constructs it directly. Returns the gen dir. +func writeSegmentSetFixture(t *testing.T, stateDir string, gen uint64) string { + t.Helper() + docs := []*graph.Document{ + {Repo: "seg", Entities: []graph.Entity{ + {ID: "aa1", QualifiedName: "p.A", Kind: "function", Name: "A"}, + {ID: "aa2", QualifiedName: "p.B", Kind: "struct", Name: "B"}, + }, Relationships: []graph.Relationship{{FromID: "aa1", ToID: "mm1", Kind: "CALLS"}}}, + {Repo: "seg", Entities: []graph.Entity{ + {ID: "mm1", QualifiedName: "p.M", Kind: "function", Name: "M"}, + {ID: "mm2", QualifiedName: "p.N", Kind: "struct", Name: "N"}, + }, Relationships: []graph.Relationship{{FromID: "mm1", ToID: "aa1", Kind: "REFERENCES"}}}, + } + genDir := filepath.Join(stateDir, graph.GenDirName(gen)) + m := &graph.Manifest{FormatVersion: graph.ManifestFormatVersion} + for i, doc := range docs { + name := graph.SegmentFileName(i) + if err := fbwriter.WriteAtomic(filepath.Join(genDir, name), doc); err != nil { + t.Fatalf("write %s: %v", name, err) + } + ids := make([]string, 0, len(doc.Entities)) + for _, e := range doc.Entities { + ids = append(ids, e.ID) + } + sort.Strings(ids) + m.Segments = append(m.Segments, graph.SegmentMeta{ + File: name, Kind: graph.SegmentEntities, + EntityCount: len(doc.Entities), RelCount: len(doc.Relationships), + MinKey: ids[0], MaxKey: ids[len(ids)-1], + }) + } + if err := graph.WriteManifest(genDir, m); err != nil { + t.Fatalf("write manifest: %v", err) + } + if err := graph.WriteCurrentPointerRaw(stateDir, graph.GenDirName(gen)); err != nil { + t.Fatalf("write current: %v", err) + } + return genDir +} + +// TestCache_SegmentSetGenDir: the daemon graph cache opens a multi-segment gen +// dir (keyed by the gen dir path, as GetForRepoRef would resolve it) and the +// returned GraphView reads across all segments, including a cross-segment +// relationship endpoint. +func TestCache_SegmentSetGenDir(t *testing.T) { + stateDir := t.TempDir() + genDir := writeSegmentSetFixture(t, stateDir, 3) + + c := NewCache(4) + // Windows teardown: close the cache (unmaps every segment) before the + // t.TempDir cleanup unlinks the gen dir. + t.Cleanup(func() { _ = c.Close() }) + + r, release, err := c.Get(genDir) + if err != nil { + t.Fatalf("Get(gen dir): %v", err) + } + defer release() + + if got := r.EntityCount(); got != 4 { + t.Errorf("EntityCount across segments = %d, want 4", got) + } + if got := r.RelationshipCount(); got != 2 { + t.Errorf("RelationshipCount across segments = %d, want 2", got) + } + for _, id := range []string{"aa1", "aa2", "mm1", "mm2"} { + if e := r.LookupEntityByID(id); e == nil || string(e.Id()) != id { + t.Errorf("LookupEntityByID(%q) failed across segments: %v", id, e) + } + } + // A second Get is a cache hit on the same gen-dir key. + r2, rel2, err := c.Get(genDir) + if err != nil { + t.Fatal(err) + } + rel2() + if r2 != r { + t.Fatal("expected cache hit (same GraphView) for the gen-dir key") + } + if s := c.Stats(); s.Hits != 1 { + t.Fatalf("stats after hit: %+v", s) + } +} diff --git a/internal/daemon/state_path.go b/internal/daemon/state_path.go index 2debe95b3..73a4b49d0 100644 --- a/internal/daemon/state_path.go +++ b/internal/daemon/state_path.go @@ -469,6 +469,25 @@ func findGraphFileInDir(dir string) (path string, modtime int64) { // linchpin: because writers stop bumping a fixed graph.fb, FindGraphFile / // FindGraphFileAnyRef — and therefore statuswriter's GraphFBMtime — MUST // stat the resolved gen file so a completed rebuild's mtime keeps advancing. + // + // #5901 segment-set: when the active graph is the multi-segment gen-dir + // layout, the resolved artifact is a directory (graph./), not a .fb + // file. Return the gen dir as the graph "path" (its parent is the state dir, + // so lr.GraphFile → filepath.Dir → the correct state dir for the Doc load) + // and use the manifest.json mtime as the freshness signal (a real file whose + // mtime advances on each rebuild). Downstream readers route on this via the + // descriptor; the mmap zero-copy cutover correctly declines a dir (see + // internal/mcp/state.go's .fb-ext guard) and serves the segment-set Document. + if desc, dErr := graph.CurrentGraphDescriptor(dir); dErr == nil && desc.Kind == graph.GraphSegmentSet { + if fi, err := os.Stat(filepath.Join(desc.GenDir, graph.ManifestFileName)); err == nil { + segMtime := fi.ModTime().UnixNano() + jsonPath := filepath.Join(dir, "graph.json") + if jInfo, jErr := os.Stat(jsonPath); jErr == nil && jInfo.ModTime().UnixNano() > segMtime { + return jsonPath, jInfo.ModTime().UnixNano() + } + return desc.GenDir, segMtime + } + } fbPath := graph.CurrentGraphPath(dir) jsonPath := filepath.Join(dir, "graph.json") diff --git a/internal/graph/fbreader/multireader.go b/internal/graph/fbreader/multireader.go index b2087350a..232cd78f8 100644 --- a/internal/graph/fbreader/multireader.go +++ b/internal/graph/fbreader/multireader.go @@ -41,16 +41,79 @@ import ( // results, same iteration order) — see TestMultiReaderSingleSegmentParity. type MultiReader struct { segs []*Reader + // ranges, when non-nil, is parallel to segs and carries each segment's + // entity-ID key range from the gen-dir manifest. LookupEntityByID uses it + // to SKIP segments that cannot contain the looked-up key, avoiding the O(S) + // fan-out (decision 4 of #5901). nil ⇒ no routing info ⇒ every segment is + // searched (the OpenSegments legacy behaviour, unchanged). + ranges []KeyRange } +// KeyRange is the entity-ID key coverage of one segment, as recorded in the +// gen-dir manifest (SegmentMeta MinKey/MaxKey + a HasEntities flag derived +// from EntityCount>0). LookupEntityByID consults it to skip segments that +// provably cannot hold a key. +// +// Semantics: +// - HasEntities==false ⇒ the segment carries NO entities (a pure-relationship +// segment) ⇒ every entity lookup skips it. +// - HasEntities==true with Min=="" and Max=="" ⇒ range unknown ⇒ never skip +// (safe fallback: search it). +// - Otherwise the (lexicographic, inclusive) [Min,Max] window gates the key. +type KeyRange struct { + HasEntities bool + Min string + Max string +} + +// contains reports whether id could live in a segment with this key range. +func (kr KeyRange) contains(id string) bool { + if !kr.HasEntities { + return false + } + if kr.Min == "" && kr.Max == "" { + return true // unknown bounds — cannot safely skip + } + if kr.Min != "" && id < kr.Min { + return false + } + if kr.Max != "" && id > kr.Max { + return false + } + return true +} + +// SegmentSearchHook, when non-nil, is invoked with the segment index each time +// LookupEntityByID actually performs a binary search on that segment (i.e. NOT +// skipped by key routing). It is a test/observability seam — nil in production, +// where it adds a single nil-check on the lookup path — set by the routing +// tests to assert out-of-range segments are pruned. Not safe for concurrent +// mutation; set it only from single-threaded test setup. +var SegmentSearchHook func(seg int) + // OpenSegments memory-maps every path in paths (in the given order) and // returns a MultiReader over all of them. paths must be non-empty. If any // segment fails to open, every already-opened segment is closed before // the error is returned (no leaked mappings on a partial failure). +// +// This constructor attaches NO key ranges, so LookupEntityByID fans out across +// every segment (O(S)). Use OpenSegmentsWithRanges to enable manifest-driven +// segment skipping. func OpenSegments(paths []string) (*MultiReader, error) { + return OpenSegmentsWithRanges(paths, nil) +} + +// OpenSegmentsWithRanges is OpenSegments plus per-segment key ranges (from the +// gen-dir manifest) used to prune LookupEntityByID. When ranges is non-nil it +// MUST have the same length as paths (one range per segment, same order); +// a length mismatch is an error. A nil ranges disables routing (== OpenSegments). +func OpenSegmentsWithRanges(paths []string, ranges []KeyRange) (*MultiReader, error) { if len(paths) == 0 { return nil, fmt.Errorf("fbreader: OpenSegments requires at least one path") } + if ranges != nil && len(ranges) != len(paths) { + return nil, fmt.Errorf("fbreader: OpenSegmentsWithRanges: %d ranges for %d paths", len(ranges), len(paths)) + } segs := make([]*Reader, 0, len(paths)) for _, p := range paths { r, err := Open(p) @@ -62,7 +125,7 @@ func OpenSegments(paths []string) (*MultiReader, error) { } segs = append(segs, r) } - return &MultiReader{segs: segs}, nil + return &MultiReader{segs: segs, ranges: ranges}, nil } // Close releases every segment's underlying mmap. It closes all segments @@ -127,7 +190,16 @@ func (m *MultiReader) LookupEntityByID(id string) *fb.Entity { if m == nil { return nil } - for _, s := range m.segs { + for i, s := range m.segs { + // Decision 4: when manifest key ranges are attached, skip any segment + // whose [Min,Max] window cannot contain id (or that holds no entities). + // This turns the O(S) fan-out into "search only plausible segments". + if m.ranges != nil && !m.ranges[i].contains(id) { + continue + } + if SegmentSearchHook != nil { + SegmentSearchHook(i) + } if e := s.LookupEntityByID(id); e != nil { return e } diff --git a/internal/graph/fbreader/routing_test.go b/internal/graph/fbreader/routing_test.go new file mode 100644 index 000000000..043fad961 --- /dev/null +++ b/internal/graph/fbreader/routing_test.go @@ -0,0 +1,150 @@ +package fbreader_test + +import ( + "path/filepath" + "testing" + + "github.com/cajasmota/grafel/internal/graph" + "github.com/cajasmota/grafel/internal/graph/fbreader" + "github.com/cajasmota/grafel/internal/graph/fbwriter" +) + +// twoRangeSegments writes two disjoint, key-sorted entity segments: +// +// seg0: entities a, b (key range [a,b]) +// seg1: entities m, n (key range [m,n]) +// +// and returns their paths plus the parallel fbreader.KeyRange slice a manifest would +// carry. +func twoRangeSegments(t *testing.T) ([]string, []fbreader.KeyRange) { + t.Helper() + dir := t.TempDir() + seg0 := &graph.Document{Repo: "r", Entities: []graph.Entity{ + {ID: "a", QualifiedName: "p.A", Kind: "function", Name: "A"}, + {ID: "b", QualifiedName: "p.B", Kind: "struct", Name: "B"}, + }} + seg1 := &graph.Document{Repo: "r", Entities: []graph.Entity{ + {ID: "m", QualifiedName: "p.M", Kind: "function", Name: "M"}, + {ID: "n", QualifiedName: "p.N", Kind: "struct", Name: "N"}, + }} + p0 := filepath.Join(dir, "seg-0000.fb") + p1 := filepath.Join(dir, "seg-0001.fb") + if err := fbwriter.WriteAtomic(p0, seg0); err != nil { + t.Fatal(err) + } + if err := fbwriter.WriteAtomic(p1, seg1); err != nil { + t.Fatal(err) + } + ranges := []fbreader.KeyRange{ + {HasEntities: true, Min: "a", Max: "b"}, + {HasEntities: true, Min: "m", Max: "n"}, + } + return []string{p0, p1}, ranges +} + +// TestMultiReader_KeyRoutingSkipsOutOfRangeSegment: a lookup for a key that +// falls only in seg0's range must NOT search seg1 (decision 4). We assert the +// pruning via fbreader.SegmentSearchHook, which records every segment actually searched. +func TestMultiReader_KeyRoutingSkipsOutOfRangeSegment(t *testing.T) { + paths, ranges := twoRangeSegments(t) + mr, err := fbreader.OpenSegmentsWithRanges(paths, ranges) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = mr.Close() }) + + var searched []int + fbreader.SegmentSearchHook = func(seg int) { searched = append(searched, seg) } + t.Cleanup(func() { fbreader.SegmentSearchHook = nil }) + + // "a" lives in seg0 only → seg1 must be pruned. + searched = nil + if e := mr.LookupEntityByID("a"); e == nil || string(e.Id()) != "a" { + t.Fatalf("lookup a: got %v", e) + } + for _, s := range searched { + if s == 1 { + t.Fatalf("seg1 was searched for key 'a' despite being out of range; searched=%v", searched) + } + } + if len(searched) != 1 || searched[0] != 0 { + t.Fatalf("expected only seg0 searched for 'a', got %v", searched) + } + + // "n" lives in seg1 only → seg0 must be pruned. + searched = nil + if e := mr.LookupEntityByID("n"); e == nil || string(e.Id()) != "n" { + t.Fatalf("lookup n: got %v", e) + } + if len(searched) != 1 || searched[0] != 1 { + t.Fatalf("expected only seg1 searched for 'n', got %v", searched) + } + + // A key absent from every range searches NOTHING and returns nil. + searched = nil + if e := mr.LookupEntityByID("zzz"); e != nil { + t.Fatalf("lookup zzz: expected nil, got %v", e) + } + if len(searched) != 0 { + t.Fatalf("out-of-all-range key should search no segment, got %v", searched) + } +} + +// TestMultiReader_NoRangesSearchesAll: OpenSegments (no ranges) preserves the +// legacy fan-out — every segment is searched until a hit. +func TestMultiReader_NoRangesSearchesAll(t *testing.T) { + paths, _ := twoRangeSegments(t) + mr, err := fbreader.OpenSegments(paths) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = mr.Close() }) + + var searched []int + fbreader.SegmentSearchHook = func(seg int) { searched = append(searched, seg) } + t.Cleanup(func() { fbreader.SegmentSearchHook = nil }) + + // "n" is in seg1; with no routing, seg0 is searched first (miss) then seg1. + if e := mr.LookupEntityByID("n"); e == nil { + t.Fatal("lookup n: not found") + } + if len(searched) != 2 || searched[0] != 0 || searched[1] != 1 { + t.Fatalf("no-range lookup should fan out across all segments, got %v", searched) + } +} + +// TestMultiReader_PureRelationshipSegmentSkipped: a segment whose KeyRange has +// HasEntities=false (a pure relationship stream) is never searched for an +// entity key, while an entity segment with unknown bounds is always searched. +func TestMultiReader_PureRelationshipSegmentSkipped(t *testing.T) { + paths, _ := twoRangeSegments(t) + // seg0 declared as a pure-relationship segment (HasEntities=false) even + // though it physically holds entities — routing must trust the manifest and + // skip it; seg1 declared with unknown bounds (HasEntities, no Min/Max) so it + // is always searched. + ranges := []fbreader.KeyRange{ + {HasEntities: false}, + {HasEntities: true}, + } + mr, err := fbreader.OpenSegmentsWithRanges(paths, ranges) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = mr.Close() }) + + var searched []int + fbreader.SegmentSearchHook = func(seg int) { searched = append(searched, seg) } + t.Cleanup(func() { fbreader.SegmentSearchHook = nil }) + + // "a" physically lives in seg0, but seg0 is flagged pure-relationship, so it + // is skipped and the lookup misses (n is only in seg1, unknown-bounds). + _ = mr.LookupEntityByID("a") + for _, s := range searched { + if s == 0 { + t.Fatalf("pure-relationship seg0 must never be searched, got %v", searched) + } + } + if len(searched) != 1 || searched[0] != 1 { + t.Fatalf("unknown-bounds seg1 must always be searched, got %v", searched) + } +} diff --git a/internal/graph/fbreader/view.go b/internal/graph/fbreader/view.go new file mode 100644 index 000000000..4eb569912 --- /dev/null +++ b/internal/graph/fbreader/view.go @@ -0,0 +1,65 @@ +package fbreader + +import ( + fb "github.com/cajasmota/grafel/internal/graph/fbgraph" +) + +// GraphView is the read surface shared by the single-file *Reader and the +// multi-segment *MultiReader. It exists so segment-aware call sites (the #5890 +// gen-dir read substrate, #5901) can hold "a graph" without caring whether it +// is backed by one mmap or N segment mmaps — the single-file fast path returns +// a *Reader, a segment-set returns a *MultiReader, and both satisfy this. +// +// Every method is already implemented identically on both concrete types +// (MultiReader was built in #5906 to mirror Reader's surface exactly), so this +// interface introduces NO behaviour change for the single-file path — it just +// names the common contract. +type GraphView interface { + // Close releases the underlying mmap(s). After Close no field/string + // previously read from this view may be accessed. + Close() error + + // Version is the on-disk FB format version (first segment's header for a + // segment-set). + Version() int + + // EntityCount / RelationshipCount are the totals across all segments. + EntityCount() int + RelationshipCount() int + + // LookupEntityByID returns the entity with id, or nil. + LookupEntityByID(id string) *fb.Entity + // FindEntityByID is the (value, ok) idiom counterpart. + FindEntityByID(id string) (*fb.Entity, bool) + + // EntityAt / RelationshipAt index the concatenated per-segment vectors. + EntityAt(i int) *fb.Entity + RelationshipAt(i int) *fb.Relationship + + // IterateEntities / IterateRelationships visit every row across segments. + IterateEntities(visit func(e *fb.Entity) bool) + IterateRelationships(visit func(rel *fb.Relationship) bool) + + // IterateRelationshipsFromID / ToID collect edges by endpoint. + IterateRelationshipsFromID(id string) []*fb.Relationship + IterateRelationshipsToID(id string) []*fb.Relationship + + // FilterEntitiesByKind collects entities of a given kind. + FilterEntitiesByKind(kind string) []*fb.Entity + + // CommunityCount / CommunityAt expose the Pass-4 aggregate communities. + CommunityCount() int + CommunityAt(i int) *fb.Community + + // LoadGraphMeta / LoadAlgoStats expose the Graph-root header fields. + LoadGraphMeta() GraphMeta + LoadAlgoStats() AlgoStats +} + +// Compile-time proof both concrete readers satisfy the shared contract. If a +// future method is added to one type's public surface that consumers rely on, +// add it here and these assertions keep the two implementations in lock-step. +var ( + _ GraphView = (*Reader)(nil) + _ GraphView = (*MultiReader)(nil) +) diff --git a/internal/graph/genpath.go b/internal/graph/genpath.go index 9be494e1a..ebc240d6d 100644 --- a/internal/graph/genpath.go +++ b/internal/graph/genpath.go @@ -47,11 +47,28 @@ const flatGraphName = "graph.fb" // genFileRe matches a generation graph file: graph..fb. var genFileRe = regexp.MustCompile(`^graph\.(\d+)\.fb$`) +// genDirRe matches a MULTI-SEGMENT generation directory: graph. (no +// .fb suffix). Under the #5890/#5901 segmented layout a graph too large for one +// segment is written as graph./seg-NNNN.fb + manifest.json, and the +// `current` pointer may name that DIR (or its manifest) — so `current` is no +// longer guaranteed to name a *.fb file. This regex is the segment-set +// counterpart to genFileRe, with the same path-traversal hardening (a pointer +// value that is not exactly graph. can never resolve to a gen dir). +var genDirRe = regexp.MustCompile(`^graph\.(\d+)$`) + // GenFileName renders the on-disk filename for a given generation. func GenFileName(gen uint64) string { return fmt.Sprintf("graph.%d.fb", gen) } +// GenDirName renders the on-disk MULTI-SEGMENT generation directory name for a +// given generation: graph. (no .fb suffix). The future streaming writer +// (#5902) creates this dir and fills it with seg-NNNN.fb + manifest.json; the +// dark read substrate only needs the name to recognise + resolve it. +func GenDirName(gen uint64) string { + return fmt.Sprintf("graph.%d", gen) +} + // IsGraphFileName reports whether base (a bare filename, not a path) is a // FlatBuffers graph file recognised by the gen layout: the legacy flat // "graph.fb" or a generation file "graph..fb". Enumeration/GC callers use @@ -121,6 +138,143 @@ func CurrentGraphPath(dir string) string { return filepath.Join(dir, flatGraphName) } +// GraphKind classifies what the `current` pointer (or the legacy flat file) +// resolves to for a state dir. +type GraphKind int + +const ( + // GraphAbsent — no graph on disk (never-indexed repo): neither a resolvable + // gen file/dir nor the legacy flat graph.fb exists. + GraphAbsent GraphKind = iota + // GraphSingleFile — the single-segment fast path: graph..fb (or the + // legacy flat graph.fb). Byte-identical to the pre-#5901 world; the + // overwhelming common case. + GraphSingleFile + // GraphSegmentSet — the multi-segment layout: a graph./ dir holding + // seg-NNNN.fb files + manifest.json. + GraphSegmentSet +) + +// GraphDescriptor is the resolved shape of a state dir's active graph. It is +// the segment-aware superset of CurrentGraphPath: where CurrentGraphPath only +// ever hands back a single .fb path (and is preserved unchanged for the ~11 +// #5891 single-file/legacy callers), CurrentGraphDescriptor additionally +// recognises the segment-set layout so a reader can route to it. +type GraphDescriptor struct { + Kind GraphKind + // Dir is the state dir this descriptor was resolved for. + Dir string + // Path is the single .fb file for GraphSingleFile (and, for GraphAbsent, + // the flat graph.fb path that does not exist — mirroring CurrentGraphPath's + // "return the flat path even when absent" contract). Empty for a segment-set. + Path string + // GenDir is the absolute graph./ directory for GraphSegmentSet; empty + // otherwise. + GenDir string + // Manifest is the parsed, validated manifest for GraphSegmentSet; nil + // otherwise. + Manifest *Manifest + // Segments are the absolute seg-NNNN.fb paths (in manifest order) for + // GraphSegmentSet; nil otherwise. Handed straight to OpenSegmentsWithRanges. + Segments []string +} + +// CurrentGraphDescriptor resolves dir's active graph into a GraphDescriptor, +// classifying it as single-file, segment-set, or absent. Resolution order +// mirrors CurrentGraphPath, extended for the segment-set layout: +// +// 1. `current` names graph..fb (an existing file) → GraphSingleFile. +// 2. `current` names graph. (a dir) or graph./manifest.json → read +// + validate that dir's manifest → GraphSegmentSet. A malformed/hostile +// manifest returns a non-nil error (the reader MUST NOT proceed). +// 3. Otherwise fall back to the legacy flat graph.fb: GraphSingleFile when it +// exists on disk, else GraphAbsent (Path still set to the flat path, for +// parity with CurrentGraphPath's absent semantics). +// +// A missing pointer, a pointer to a missing gen file, or a hostile pointer all +// fall through to step 3 (never an error) — only a pointer that DOES resolve to +// a gen dir whose manifest is corrupt surfaces an error, so a genuinely broken +// segment-set is loud rather than silently mis-read as "absent". +func CurrentGraphDescriptor(dir string) (GraphDescriptor, error) { + if dir == "" { + return GraphDescriptor{Kind: GraphAbsent}, nil + } + flat := filepath.Join(dir, flatGraphName) + raw, ok := readPointerRaw(dir) + if ok { + // (1) Single-file gen pointer. + if _, isGen := parseGen(raw); isGen { + p := filepath.Join(dir, raw) + if fi, err := os.Stat(p); err == nil && !fi.IsDir() { + return GraphDescriptor{Kind: GraphSingleFile, Dir: dir, Path: p}, nil + } + // Pointer names a missing gen file — fall through to flat fallback. + } else if genDirName, isSeg := parseSegPointer(raw); isSeg { + // (2) Segment-set pointer (gen dir or its manifest). + genDir := filepath.Join(dir, genDirName) + if fi, err := os.Stat(genDir); err == nil && fi.IsDir() { + m, mErr := ReadManifest(genDir) + if mErr != nil { + return GraphDescriptor{}, fmt.Errorf("graph: resolve segment-set %s: %w", genDir, mErr) + } + segs := make([]string, 0, len(m.Segments)) + for _, s := range m.Segments { + segs = append(segs, filepath.Join(genDir, s.File)) + } + return GraphDescriptor{ + Kind: GraphSegmentSet, + Dir: dir, + GenDir: genDir, + Manifest: m, + Segments: segs, + }, nil + } + // Pointer names a missing gen dir — fall through to flat fallback. + } + } + // (3) Legacy flat fallback. + if fi, err := os.Stat(flat); err == nil && !fi.IsDir() { + return GraphDescriptor{Kind: GraphSingleFile, Dir: dir, Path: flat}, nil + } + return GraphDescriptor{Kind: GraphAbsent, Dir: dir, Path: flat}, nil +} + +// readPointerRaw returns the trimmed content of /current with NO shape +// validation beyond non-emptiness and a coarse path-safety guard (no separator, +// except the single "/manifest.json" suffix a segment-set pointer may carry; +// no ".."). CurrentGraphDescriptor then classifies the value via parseGen / +// parseSegPointer. This is the segment-aware sibling of readPointer (which only +// accepts the graph..fb single-file shape); the two share the same trust +// boundary — a hostile pointer resolves to nothing and falls back to flat. +func readPointerRaw(dir string) (raw string, ok bool) { + data, err := os.ReadFile(filepath.Join(dir, currentPointerName)) + if err != nil { + return "", false + } + raw = strings.TrimSpace(string(data)) + if raw == "" || strings.Contains(raw, "..") { + return "", false + } + return raw, true +} + +// parseSegPointer recognises a segment-set `current` pointer value and returns +// the bare gen-dir name (graph.). It accepts either the dir itself +// ("graph.") or the dir-qualified manifest ("graph./manifest.json") +// per decision 2. Anything else (a single-file gen name, junk, a deeper path) +// yields ok=false. The returned name is always a bare "graph." with no +// separator, so joining it under dir can never escape dir. +func parseSegPointer(raw string) (genDirName string, ok bool) { + name := raw + if suffix := "/" + ManifestFileName; strings.HasSuffix(raw, suffix) { + name = strings.TrimSuffix(raw, suffix) + } + if !genDirRe.MatchString(name) { + return "", false + } + return name, true +} + // NextGen returns the next generation integer to write in dir: one greater // than the maximum generation observed among the `current` pointer and every // graph..fb already present. Returns 1 for a dir that has never held a @@ -185,6 +339,35 @@ func WriteCurrentPointer(dir, genName string) error { return fmt.Errorf("graph.WriteCurrentPointer: rename: %w", err) } +// WriteCurrentPointerRaw atomically points /current at name, where name +// may be EITHER a single-file gen ("graph..fb") OR a segment-set gen dir +// ("graph.") or its manifest ("graph./manifest.json"). It is the +// segment-aware superset of WriteCurrentPointer (which only accepts the +// single-file shape); the two share the same atomic tmp+rename + bounded retry. +// This is a format primitive for the FUTURE streaming writer (#5902) and the +// test fixtures — no existing producer calls it. +func WriteCurrentPointerRaw(dir, name string) error { + _, single := parseGen(name) + _, seg := parseSegPointer(name) + if !single && !seg { + return fmt.Errorf("graph.WriteCurrentPointerRaw: invalid pointer target %q", name) + } + tmp := filepath.Join(dir, currentPointerName+".tmp") + if err := os.WriteFile(tmp, []byte(name+"\n"), 0o644); err != nil { + return fmt.Errorf("graph.WriteCurrentPointerRaw: write tmp: %w", err) + } + dst := filepath.Join(dir, currentPointerName) + var err error + for i := 0; i < pointerFlipRetries; i++ { + if err = os.Rename(tmp, dst); err == nil { + return nil + } + time.Sleep(pointerFlipRetryDelay) + } + os.Remove(tmp) + return fmt.Errorf("graph.WriteCurrentPointerRaw: rename: %w", err) +} + // GCStaleGens best-effort unlinks generation files older than the // immediately-previous generation, keeping the current gen and the one before // it. The previous gen is retained because serve/MCP may still have it mapped @@ -195,7 +378,15 @@ func WriteCurrentPointer(dir, genName string) error { // This is STRICTLY best-effort and MUST NOT affect the caller's success: a // still-mapped gen on Windows fails to delete with a sharing violation, which // is silently ignored and swept on a later write. A GC error never propagates. -// Returns the list of gen files actually removed (for tests/observability). +// Returns the list of gen entries actually removed (for tests/observability). +// +// #5901: a generation may be EITHER a single-file graph..fb OR a +// multi-segment gen DIR graph./ (holding seg-NNNN.fb + manifest.json). GC +// sweeps both: a stale gen file is unlinked; a stale gen dir is removed +// recursively (os.RemoveAll — every segment plus the manifest). The keep-window +// (current + immediately-previous) is computed over the UNION of gen files and +// gen dirs, so a mixed-history dir (some gens single-file, some segmented) +// still retains exactly the two newest generations regardless of shape. func GCStaleGens(dir string, currentGen uint64) []string { if currentGen <= 1 { return nil // nothing older than the immediately-previous gen exists @@ -207,15 +398,27 @@ func GCStaleGens(dir string, currentGen uint64) []string { } var removed []string for _, e := range ents { + name := e.Name() if e.IsDir() { + // Multi-segment gen dir: graph./ → rm -rf when stale. + v, ok := parseGenDir(name) + if !ok || v >= keepFrom { + continue + } + if err := os.RemoveAll(filepath.Join(dir, name)); err == nil { + removed = append(removed, name) + } + // On failure (e.g. Windows sharing violation on a still-mapped + // segment) swallow and retry on the next write. Never fail. continue } - v, ok := parseGen(e.Name()) + // Single-file gen: graph..fb → unlink when stale. + v, ok := parseGen(name) if !ok || v >= keepFrom { continue } - if err := os.Remove(filepath.Join(dir, e.Name())); err == nil { - removed = append(removed, e.Name()) + if err := os.Remove(filepath.Join(dir, name)); err == nil { + removed = append(removed, name) } // On failure (e.g. Windows ERROR_SHARING_VIOLATION on a still-mapped // gen) swallow the error and try again on the next write. Never fail. @@ -223,6 +426,21 @@ func GCStaleGens(dir string, currentGen uint64) []string { return removed } +// parseGenDir returns the generation integer encoded in a bare gen-DIR name +// (graph., no .fb suffix) and whether it matched. It is the segment-set +// counterpart to parseGen. +func parseGenDir(name string) (uint64, bool) { + m := genDirRe.FindStringSubmatch(name) + if m == nil { + return 0, false + } + v, err := strconv.ParseUint(m[1], 10, 64) + if err != nil { + return 0, false + } + return v, true +} + // WriteGenGraph writes buf as a NEW generation file in dir, flips the `current` // pointer to it, and best-effort GCs stale generations. It is the single // producer primitive behind every writer (full-index and incremental): it diff --git a/internal/graph/load.go b/internal/graph/load.go index c111afb09..1f818773b 100644 --- a/internal/graph/load.go +++ b/internal/graph/load.go @@ -117,6 +117,17 @@ func FormatVersionReason(found, required int) string { // 3. If only graph.json exists, fall back to JSON. // 4. If neither exists, return a non-nil error. func LoadGraphFromDir(dir string) (*Document, error) { + // #5901 segment-set: when the active graph is the multi-segment gen-dir + // layout, materialize the Document from all segments (via the same shared + // core the single-file path uses). A corrupt segment-set manifest surfaces + // the descriptor error. Everything below is the UNCHANGED single-file / + // legacy / json resolution — the common path is byte-identical. + if desc, derr := CurrentGraphDescriptor(dir); derr != nil { + return nil, derr + } else if desc.Kind == GraphSegmentSet { + return loadSegmentSetDocument(desc, false) + } + fbPath := CurrentGraphPath(dir) jsonPath := filepath.Join(dir, "graph.json") @@ -165,6 +176,13 @@ func LoadGraphFromDir(dir string) (*Document, error) { // (absent) memory win. Callers that need a full Document (every CLI/full-Doc // consumer) must keep using LoadGraphFromDir. func LoadGraphHeaderOnlyFromDir(dir string) (*Document, error) { + // #5901 segment-set: header-only load across all segments (counts/meta from + // the manifest+first-segment header; Entities/Relationships left empty). + if desc, derr := CurrentGraphDescriptor(dir); derr != nil { + return nil, derr + } else if desc.Kind == GraphSegmentSet { + return loadSegmentSetDocument(desc, true) + } fbPath := CurrentGraphPath(dir) if _, err := os.Stat(fbPath); err == nil { return loadFBDocumentHeaderOnly(fbPath) @@ -173,6 +191,22 @@ func LoadGraphHeaderOnlyFromDir(dir string) (*Document, error) { return LoadGraphFromDir(dir) } +// loadSegmentSetDocument materializes a *Document from a resolved segment-set +// descriptor. It opens a MultiReader over every segment (with manifest key +// routing) and runs the SAME materialize core as the single-file path, so a +// segment-set Document is byte-identical to the single-file Document that would +// result from the same entities/relationships written as one file — modulo the +// (per-segment) header fields the MultiReader sources from segment 0. +func loadSegmentSetDocument(desc GraphDescriptor, headerOnly bool) (*Document, error) { + paths, ranges := segmentOpenArgs(desc.Manifest, desc.GenDir) + v, err := fbreader.OpenSegmentsWithRanges(paths, ranges) + if err != nil { + return nil, fmt.Errorf("graph.loadSegmentSetDocument: open %s: %w", desc.GenDir, err) + } + defer v.Close() + return materializeGraphView(v, headerOnly) +} + // PersistedStats is a cheap, on-disk view of a repo's index size and // freshness, read from the graph.fb header WITHOUT materializing the // entity/relationship vectors. Used by the dashboard group overview and @@ -329,7 +363,16 @@ func loadFBDoc(path string, headerOnly bool) (*Document, error) { return nil, fmt.Errorf("graph.loadFBDocument: open %s: %w", path, err) } defer r.Close() + return materializeGraphView(r, headerOnly) +} +// materializeGraphView is the shared materialize core behind BOTH the +// single-file path (loadFBDoc, over an *fbreader.Reader) and the segment-set +// path (loadSegmentSetDocument, over an *fbreader.MultiReader). It operates on +// the fbreader.GraphView contract so the two backings run byte-identical +// materialize logic — the ONLY thing that differs is how many mmaps sit under +// the view. It does NOT Close v (the caller owns the view's lifetime). +func materializeGraphView(r fbreader.GraphView, headerOnly bool) (*Document, error) { // #2370 — refuse to read old-format graph.fb files. grafel is // pre-1.0; there is no on-disk compat path. The user is expected to // rerun `grafel index ` to regenerate graph.fb against the diff --git a/internal/graph/manifest.go b/internal/graph/manifest.go new file mode 100644 index 000000000..b82143a23 --- /dev/null +++ b/internal/graph/manifest.go @@ -0,0 +1,245 @@ +// Package graph — manifest.go defines the on-disk manifest for a MULTI-SEGMENT +// generation directory (the #5890 segmented streaming-write epic; this is the +// DARK read-substrate slice #5901, which defines the format + teaches readers +// to route to it but writes NOTHING segmented yet). +// +// # Layout recap +// +// A graph that fits in ONE segment stays a plain graph..fb file — no dir, +// no manifest, byte-identical to today (the single-file fast path). Only a +// MULTI-segment graph uses the directory layout: +// +// graph./ +// seg-0000.fb +// seg-0001.fb +// ... +// manifest.json +// +// The `current` pointer may then name the gen DIR (graph.) or its +// manifest (graph./manifest.json) — so `current` is NO LONGER guaranteed +// to name a *.fb file (see genpath.go's CurrentGraphDescriptor). +// +// # What the manifest records (decisions 3+4+5) +// +// - A FormatVersion for the manifest schema itself (independent of +// fbversion.Version, which versions the FlatBuffers *payload* inside each +// seg-*.fb — this slice does NOT bump fbversion). +// - One SegmentMeta per segment file, carrying its Kind (entity vs +// relationship — so the two streams can be listed/counted independently), +// its entity/relationship counts, and — for entity segments — the MIN and +// MAX entity-ID key it contains. The reader uses that [min,max] range to +// SKIP segments that cannot hold a looked-up key, avoiding the O(S) +// per-segment fan-out (decision 4). +package graph + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// ManifestFileName is the fixed basename of the per-generation segment +// manifest inside a multi-segment gen dir (graph./manifest.json). +const ManifestFileName = "manifest.json" + +// ManifestFormatVersion is the CURRENT on-disk schema version of manifest.json. +// It versions the manifest STRUCTURE only; it is deliberately separate from +// fbversion.Version (which versions the FlatBuffers payload of each seg-*.fb). +// A reader rejects a manifest whose FormatVersion it does not understand. +const ManifestFormatVersion = 1 + +// SegmentKind classifies a segment file as carrying the entity stream or the +// relationship stream. The streams are conceptually independent (decision 5): +// the writer (a later slice) may emit entity-segments and relationship-segments +// separately, and the manifest lists/counts them independently. A single +// segment file MAY in practice carry both (a small graph split only by size), +// in which case Kind names its PRIMARY stream and both counts are populated. +type SegmentKind string + +const ( + // SegmentEntities marks a segment whose primary payload is entities. + SegmentEntities SegmentKind = "entity" + // SegmentRelationships marks a segment whose primary payload is relationships. + SegmentRelationships SegmentKind = "relationship" +) + +// segFileRe validates a segment filename: seg-.fb (zero-padded on +// write, but any digit run accepted on read). Mirrors genpath's genFileRe +// hardening: a manifest whose File names fail this pattern (path traversal, +// absolute paths, "..", nested dirs) is REJECTED, so a hostile/corrupt +// manifest can never make a reader open a file outside the gen dir. +var segFileRe = regexp.MustCompile(`^seg-\d+\.fb$`) + +// SegmentFileName renders the zero-padded on-disk name for the i-th segment +// (seg-0000.fb, seg-0001.fb, ...). Decision 2's zero-padding keeps a plain +// lexicographic dir listing in segment order. Used by the future streaming +// writer (#5902) and the dark-slice test fixtures. +func SegmentFileName(i int) string { + return fmt.Sprintf("seg-%04d.fb", i) +} + +// SegmentMeta describes one segment file within a gen dir's manifest. +type SegmentMeta struct { + // File is the BARE filename (seg-NNNN.fb) of the segment, relative to the + // gen dir. It is never a path — read-side validation (segFileRe) rejects + // any value containing a separator or "..". + File string `json:"file"` + // Kind is the primary stream carried by this segment (entity|relationship). + Kind SegmentKind `json:"kind"` + // EntityCount / RelCount are the number of entities / relationships this + // segment contributes. Either may be zero (a pure-relationship segment has + // EntityCount==0, and vice-versa). + EntityCount int `json:"entity_count"` + RelCount int `json:"rel_count"` + // MinKey / MaxKey are the smallest and largest entity-ID keys present in + // this segment (lexicographic, matching the FlatBuffers `(key)` sort on + // Entity.id). Populated only for segments that carry entities + // (EntityCount>0); empty for pure-relationship segments. The reader uses + // [MinKey,MaxKey] to skip segments during LookupEntityByID (decision 4). + MinKey string `json:"min_key,omitempty"` + MaxKey string `json:"max_key,omitempty"` +} + +// Manifest is the parsed graph./manifest.json. +type Manifest struct { + // FormatVersion is the manifest schema version (ManifestFormatVersion). + FormatVersion int `json:"format_version"` + // Segments lists every segment file in the gen dir, in read/open order. + Segments []SegmentMeta `json:"segments"` +} + +// EntitySegments returns the subset of segments that carry entities +// (EntityCount>0), preserving manifest order. Lets a caller enumerate the +// entity stream independently of the relationship stream (decision 5). +func (m *Manifest) EntitySegments() []SegmentMeta { + if m == nil { + return nil + } + out := make([]SegmentMeta, 0, len(m.Segments)) + for _, s := range m.Segments { + if s.EntityCount > 0 { + out = append(out, s) + } + } + return out +} + +// RelationshipSegments returns the subset of segments that carry relationships +// (RelCount>0), preserving manifest order. +func (m *Manifest) RelationshipSegments() []SegmentMeta { + if m == nil { + return nil + } + out := make([]SegmentMeta, 0, len(m.Segments)) + for _, s := range m.Segments { + if s.RelCount > 0 { + out = append(out, s) + } + } + return out +} + +// TotalEntityCount / TotalRelationshipCount sum the per-segment counts. Cheap +// header-level totals that avoid opening any segment file. +func (m *Manifest) TotalEntityCount() int { + if m == nil { + return 0 + } + n := 0 + for _, s := range m.Segments { + n += s.EntityCount + } + return n +} + +func (m *Manifest) TotalRelationshipCount() int { + if m == nil { + return 0 + } + n := 0 + for _, s := range m.Segments { + n += s.RelCount + } + return n +} + +// validate enforces the manifest's structural invariants, rejecting a +// malformed or hostile manifest before any segment file is opened: +// +// - FormatVersion must be a known version (1..ManifestFormatVersion). +// - At least one segment must be listed. +// - Every File must be a bare seg-NNNN.fb name (no separators, no "..") so a +// path-traversal entry can never escape the gen dir. +func (m *Manifest) validate() error { + if m == nil { + return fmt.Errorf("graph: nil manifest") + } + if m.FormatVersion < 1 || m.FormatVersion > ManifestFormatVersion { + return fmt.Errorf("graph: unsupported manifest format version %d (this build understands 1..%d)", + m.FormatVersion, ManifestFormatVersion) + } + if len(m.Segments) == 0 { + return fmt.Errorf("graph: manifest lists no segments") + } + for i, s := range m.Segments { + if s.File == "" { + return fmt.Errorf("graph: manifest segment %d has empty file name", i) + } + // Reject anything that is not a bare seg-NNNN.fb name. filepath.Base + // strips any directory prefix, so a File that differs from its own Base + // carries a separator (traversal attempt) and is rejected outright. + if s.File != filepath.Base(s.File) || strings.Contains(s.File, "..") || !segFileRe.MatchString(s.File) { + return fmt.Errorf("graph: manifest segment %d has invalid file name %q", i, s.File) + } + } + return nil +} + +// ReadManifest reads and validates graph./manifest.json from genDir. +// A missing, unreadable, malformed, or hostile manifest yields a non-nil +// error and a nil manifest — callers MUST NOT open any segment on error. +func ReadManifest(genDir string) (*Manifest, error) { + data, err := os.ReadFile(filepath.Join(genDir, ManifestFileName)) + if err != nil { + return nil, fmt.Errorf("graph: read manifest in %s: %w", genDir, err) + } + var m Manifest + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("graph: parse manifest in %s: %w", genDir, err) + } + if err := m.validate(); err != nil { + return nil, err + } + return &m, nil +} + +// WriteManifest atomically writes m as graph./manifest.json inside genDir +// (tmp + rename, mirroring WriteGenGraph / WriteCurrentPointer). It validates +// m first so a malformed manifest is never persisted. NOTE: this is the format +// primitive the FUTURE streaming writer (#5902) will call; the dark read slice +// only exercises it from tests to hand-build fixtures — no producer calls it. +func WriteManifest(genDir string, m *Manifest) error { + if err := m.validate(); err != nil { + return err + } + if err := os.MkdirAll(genDir, 0o755); err != nil { + return fmt.Errorf("graph: mkdir gen dir %s: %w", genDir, err) + } + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("graph: marshal manifest: %w", err) + } + dst := filepath.Join(genDir, ManifestFileName) + tmp := dst + ".tmp" + if err := os.WriteFile(tmp, data, 0o644); err != nil { + return fmt.Errorf("graph: write manifest tmp: %w", err) + } + if err := os.Rename(tmp, dst); err != nil { + os.Remove(tmp) + return fmt.Errorf("graph: rename manifest: %w", err) + } + return nil +} diff --git a/internal/graph/manifest_test.go b/internal/graph/manifest_test.go new file mode 100644 index 000000000..22cc1d580 --- /dev/null +++ b/internal/graph/manifest_test.go @@ -0,0 +1,94 @@ +package graph + +import ( + "os" + "path/filepath" + "testing" +) + +// TestManifest_WriteReadRoundTrip: a valid manifest survives an atomic +// write + validated read unchanged, and the totals/stream helpers agree. +func TestManifest_WriteReadRoundTrip(t *testing.T) { + genDir := filepath.Join(t.TempDir(), "graph.7") + m := &Manifest{ + FormatVersion: ManifestFormatVersion, + Segments: []SegmentMeta{ + {File: "seg-0000.fb", Kind: SegmentEntities, EntityCount: 2, RelCount: 1, MinKey: "a", MaxKey: "b"}, + {File: "seg-0001.fb", Kind: SegmentEntities, EntityCount: 2, RelCount: 0, MinKey: "m", MaxKey: "n"}, + {File: "seg-0002.fb", Kind: SegmentRelationships, RelCount: 3}, + }, + } + if err := WriteManifest(genDir, m); err != nil { + t.Fatalf("WriteManifest: %v", err) + } + // Atomic write leaves no .tmp behind. + if _, err := os.Stat(filepath.Join(genDir, ManifestFileName+".tmp")); err == nil { + t.Fatalf("leftover manifest .tmp") + } + got, err := ReadManifest(genDir) + if err != nil { + t.Fatalf("ReadManifest: %v", err) + } + if got.FormatVersion != ManifestFormatVersion || len(got.Segments) != 3 { + t.Fatalf("round-trip mismatch: %+v", got) + } + if got.TotalEntityCount() != 4 { + t.Errorf("TotalEntityCount = %d, want 4", got.TotalEntityCount()) + } + if got.TotalRelationshipCount() != 4 { + t.Errorf("TotalRelationshipCount = %d, want 4", got.TotalRelationshipCount()) + } + if n := len(got.EntitySegments()); n != 2 { + t.Errorf("EntitySegments = %d, want 2 (seg with EntityCount>0)", n) + } + if n := len(got.RelationshipSegments()); n != 2 { + t.Errorf("RelationshipSegments = %d, want 2 (seg with RelCount>0)", n) + } +} + +// TestManifest_RejectsMalformed covers the hostile/malformed inputs the read +// path must reject before any segment file is opened (test requirement vi). +func TestManifest_RejectsMalformed(t *testing.T) { + cases := map[string]string{ + "not-json": `{ this is not json `, + "zero-format-version": `{"format_version":0,"segments":[{"file":"seg-0000.fb","kind":"entity"}]}`, + "future-version": `{"format_version":999,"segments":[{"file":"seg-0000.fb","kind":"entity"}]}`, + "no-segments": `{"format_version":1,"segments":[]}`, + "path-traversal": `{"format_version":1,"segments":[{"file":"../../etc/passwd","kind":"entity"}]}`, + "nested-path": `{"format_version":1,"segments":[{"file":"sub/seg-0000.fb","kind":"entity"}]}`, + "absolute-path": `{"format_version":1,"segments":[{"file":"/tmp/seg-0000.fb","kind":"entity"}]}`, + "wrong-suffix": `{"format_version":1,"segments":[{"file":"seg-0000.json","kind":"entity"}]}`, + "empty-file": `{"format_version":1,"segments":[{"file":"","kind":"entity"}]}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + genDir := t.TempDir() + if err := os.WriteFile(filepath.Join(genDir, ManifestFileName), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if m, err := ReadManifest(genDir); err == nil { + t.Fatalf("ReadManifest accepted malformed %q: %+v", name, m) + } + }) + } +} + +// TestManifest_ReadMissing: an absent manifest is an error (never a nil-but-ok). +func TestManifest_ReadMissing(t *testing.T) { + if _, err := ReadManifest(t.TempDir()); err == nil { + t.Fatal("ReadManifest of a dir with no manifest.json should error") + } +} + +// TestManifest_WriteRejectsInvalid: WriteManifest validates before persisting, +// so a malformed manifest never reaches disk. +func TestManifest_WriteRejectsInvalid(t *testing.T) { + genDir := filepath.Join(t.TempDir(), "graph.1") + bad := &Manifest{FormatVersion: ManifestFormatVersion, Segments: []SegmentMeta{{File: "../evil.fb"}}} + if err := WriteManifest(genDir, bad); err == nil { + t.Fatal("WriteManifest persisted an invalid segment file name") + } + if _, err := os.Stat(filepath.Join(genDir, ManifestFileName)); err == nil { + t.Fatal("invalid manifest was written to disk") + } +} diff --git a/internal/graph/reader_open.go b/internal/graph/reader_open.go new file mode 100644 index 000000000..0c7d6c946 --- /dev/null +++ b/internal/graph/reader_open.go @@ -0,0 +1,82 @@ +// Package graph — reader_open.go is the segment-aware reader-open seam for the +// #5890 gen-dir read substrate (#5901). It turns a state dir (or a resolved +// gen dir) into an fbreader.GraphView WITHOUT the caller needing to know +// whether the graph is a single mmap'd file or N segment mmaps: +// +// - single-file / legacy flat / absent → fbreader.Open (today's *Reader, +// byte-identical — the common path is completely unchanged). +// - segment-set → fbreader.OpenSegmentsWithRanges (a *MultiReader), threading +// the manifest's per-segment [MinKey,MaxKey] so LookupEntityByID skips +// out-of-range segments (decision 4). +// +// The reader-open does NOT itself enforce the fbversion gate — callers that +// need it (load.go, graph_cache.go) compose it with their existing +// FormatVersionError check, so behaviour for an old-flat-below-fbversion graph +// is unchanged (no fbversion bump in this slice). +package graph + +import ( + "path/filepath" + + "github.com/cajasmota/grafel/internal/graph/fbreader" +) + +// segmentOpenArgs builds the absolute segment paths + parallel key ranges for +// a validated manifest under genDir, ready to hand to +// fbreader.OpenSegmentsWithRanges. Segment order follows manifest order. +func segmentOpenArgs(m *Manifest, genDir string) (paths []string, ranges []fbreader.KeyRange) { + paths = make([]string, 0, len(m.Segments)) + ranges = make([]fbreader.KeyRange, 0, len(m.Segments)) + for _, s := range m.Segments { + // s.File is validated (Manifest.validate) to be a bare seg-NNNN.fb with + // no separator or "..", so this join can never escape genDir. + paths = append(paths, filepath.Join(genDir, s.File)) + ranges = append(ranges, fbreader.KeyRange{ + HasEntities: s.EntityCount > 0, + Min: s.MinKey, + Max: s.MaxKey, + }) + } + return paths, ranges +} + +// OpenSegmentReader opens the multi-segment graph in genDir (which MUST contain +// a valid manifest.json) as a unified fbreader.GraphView with cross-segment key +// routing. A malformed/hostile manifest, or any segment failing to mmap, yields +// a non-nil error and a nil view. Callers own the returned view's Close. +// +// This is the path-based entry point (for callers that already hold the gen +// dir, e.g. the daemon graph cache keyed by resolved path). ReaderForDir is the +// state-dir-based entry point that resolves single-file vs segment-set first. +func OpenSegmentReader(genDir string) (fbreader.GraphView, error) { + m, err := ReadManifest(genDir) + if err != nil { + return nil, err + } + paths, ranges := segmentOpenArgs(m, genDir) + return fbreader.OpenSegmentsWithRanges(paths, ranges) +} + +// ReaderForDir resolves dir's active graph and opens it as an fbreader.GraphView: +// +// - GraphSingleFile / GraphAbsent → fbreader.Open(desc.Path). This is the +// UNCHANGED common path — the returned view is the same *Reader over the +// same file today's fbreader.Open(CurrentGraphPath(dir)) returns, so +// single-file callers are byte-identical (an absent flat path returns +// fbreader.Open's usual open error, exactly as before). +// - GraphSegmentSet → OpenSegmentReader(desc.GenDir), a *MultiReader with the +// manifest key ranges attached for LookupEntityByID pruning. +// +// A corrupt segment-set manifest surfaces the descriptor's error. This does NOT +// apply the fbversion gate; callers compose it (see loadFBDoc / graph_cache). +func ReaderForDir(dir string) (fbreader.GraphView, error) { + desc, err := CurrentGraphDescriptor(dir) + if err != nil { + return nil, err + } + if desc.Kind == GraphSegmentSet { + paths, ranges := segmentOpenArgs(desc.Manifest, desc.GenDir) + return fbreader.OpenSegmentsWithRanges(paths, ranges) + } + return fbreader.Open(desc.Path) +} diff --git a/internal/graph/segmentset_test.go b/internal/graph/segmentset_test.go new file mode 100644 index 000000000..c9a25d6a3 --- /dev/null +++ b/internal/graph/segmentset_test.go @@ -0,0 +1,397 @@ +package graph_test + +import ( + "os" + "path/filepath" + "sort" + "testing" + + "github.com/cajasmota/grafel/internal/graph" + fbgraph "github.com/cajasmota/grafel/internal/graph/fbgraph" + "github.com/cajasmota/grafel/internal/graph/fbwriter" +) + +// writeSegmentSet hand-builds a multi-segment generation on disk WITHOUT a +// producer (none exists yet — this is the dark read substrate): it writes each +// doc as its own seg-NNNN.fb inside /graph./, emits a matching +// manifest.json, and points /current at the gen dir. Each doc's entities +// MUST be a disjoint, key-sorted subset (the FlatBuffers `(key)` requirement), +// which the caller guarantees. Returns the gen dir path. +// +// The manifest's per-segment MinKey/MaxKey are derived from each doc's sorted +// entity ids so the reader's key-routing has real bounds to prune with. +func writeSegmentSet(t *testing.T, dir string, gen uint64, docs []*graph.Document) string { + t.Helper() + genDirName := graph.GenDirName(gen) + genDir := filepath.Join(dir, genDirName) + if err := os.MkdirAll(genDir, 0o755); err != nil { + t.Fatal(err) + } + m := &graph.Manifest{FormatVersion: graph.ManifestFormatVersion} + for i, doc := range docs { + name := graph.SegmentFileName(i) + if err := fbwriter.WriteAtomic(filepath.Join(genDir, name), doc); err != nil { + t.Fatalf("write %s: %v", name, err) + } + seg := graph.SegmentMeta{ + File: name, + Kind: graph.SegmentEntities, + EntityCount: len(doc.Entities), + RelCount: len(doc.Relationships), + } + if len(doc.Entities) > 0 { + ids := make([]string, 0, len(doc.Entities)) + for _, e := range doc.Entities { + ids = append(ids, e.ID) + } + sort.Strings(ids) + seg.MinKey, seg.MaxKey = ids[0], ids[len(ids)-1] + if len(doc.Relationships) > 0 { + seg.Kind = graph.SegmentEntities + } + } else { + seg.Kind = graph.SegmentRelationships + } + m.Segments = append(m.Segments, seg) + } + if err := graph.WriteManifest(genDir, m); err != nil { + t.Fatalf("write manifest: %v", err) + } + // Point `current` at the gen DIR (decision 2: current may name the dir). + if err := graph.WriteCurrentPointerRaw(dir, genDirName); err != nil { + t.Fatalf("write current pointer: %v", err) + } + return genDir +} + +// threeSegDocs returns three disjoint, key-sorted entity subsets with a couple +// of cross-segment relationships, mirroring fbreader's threeSegmentFixture. +func threeSegDocs() []*graph.Document { + return []*graph.Document{ + {Repo: "seg", Entities: []graph.Entity{ + {ID: "a", QualifiedName: "p.A", Kind: "function", Name: "A"}, + {ID: "b", QualifiedName: "p.B", Kind: "struct", Name: "B"}, + }, Relationships: []graph.Relationship{{FromID: "a", ToID: "z", Kind: "CALLS"}}}, + {Repo: "seg", Entities: []graph.Entity{ + {ID: "m", QualifiedName: "p.M", Kind: "function", Name: "M"}, + {ID: "n", QualifiedName: "p.N", Kind: "struct", Name: "N"}, + }, Relationships: []graph.Relationship{{FromID: "m", ToID: "n", Kind: "CALLS"}}}, + {Repo: "seg", Entities: []graph.Entity{ + {ID: "y", QualifiedName: "p.Y", Kind: "function", Name: "Y"}, + {ID: "z", QualifiedName: "p.Z", Kind: "struct", Name: "Z"}, + }, Relationships: []graph.Relationship{{FromID: "y", ToID: "a", Kind: "REFERENCES"}}}, + } +} + +// TestGCStaleGens_RemovesStaleGenDir: GC rm -rf's a stale multi-segment gen dir +// while keeping the current + immediately-previous generations (test req v). +func TestGCStaleGens_RemovesStaleGenDir(t *testing.T) { + dir := t.TempDir() + // Three segmented generations: 1 (stale), 2 (previous), 3 (current). + for gen := uint64(1); gen <= 3; gen++ { + writeSegmentSet(t, dir, gen, threeSegDocs()) + } + removed := graph.GCStaleGens(dir, 3) + + // gen 1 dir must be gone (rm -rf, including its segments + manifest). + if _, err := os.Stat(filepath.Join(dir, graph.GenDirName(1))); !os.IsNotExist(err) { + t.Fatalf("stale gen dir graph.1 not removed (err=%v)", err) + } + foundGen1 := false + for _, r := range removed { + if r == graph.GenDirName(1) { + foundGen1 = true + } + } + if !foundGen1 { + t.Errorf("GCStaleGens removed=%v, want it to include %q", removed, graph.GenDirName(1)) + } + // current (3) and previous (2) dirs must survive intact. + for _, keep := range []uint64{2, 3} { + p := filepath.Join(dir, graph.GenDirName(keep), graph.SegmentFileName(0)) + if _, err := os.Stat(p); err != nil { + t.Errorf("gen %d segment removed but should be kept: %v", keep, err) + } + } +} + +// TestGCStaleGens_MixedFileAndDirKeepWindow: a history mixing single-file gens +// and segmented gen dirs still keeps exactly the two newest generations. +func TestGCStaleGens_MixedFileAndDirKeepWindow(t *testing.T) { + dir := t.TempDir() + // gen 1: single-file. gen 2: single-file. gen 3: segmented dir. + for gen := uint64(1); gen <= 2; gen++ { + if err := os.WriteFile(filepath.Join(dir, graph.GenFileName(gen)), []byte("filegen00"), 0o644); err != nil { + t.Fatal(err) + } + } + writeSegmentSet(t, dir, 3, threeSegDocs()) + + graph.GCStaleGens(dir, 3) + + // gen 1 (file) stale → removed; gen 2 (file) + gen 3 (dir) kept. + if _, err := os.Stat(filepath.Join(dir, graph.GenFileName(1))); !os.IsNotExist(err) { + t.Errorf("stale single-file gen 1 not removed") + } + if _, err := os.Stat(filepath.Join(dir, graph.GenFileName(2))); err != nil { + t.Errorf("previous single-file gen 2 removed but should be kept: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, graph.GenDirName(3))); err != nil { + t.Errorf("current segmented gen 3 removed but should be kept: %v", err) + } +} + +// TestGCStaleGens_FailSoftOnHeldSegment: GC never fails even when a stale gen +// dir's segment is still open (mapped). We open a reader over gen 1's segments, +// then GC — on Unix the RemoveAll succeeds (unlinked-but-open is fine), on +// Windows it fails soft. Either way GC returns without panicking and the +// current/previous gens survive; the held reader stays valid until Close. +func TestGCStaleGens_FailSoftOnHeldSegment(t *testing.T) { + dir := t.TempDir() + for gen := uint64(1); gen <= 3; gen++ { + writeSegmentSet(t, dir, gen, threeSegDocs()) + } + // Hold gen 1 open across the GC. + v, err := graph.OpenSegmentReader(filepath.Join(dir, graph.GenDirName(1))) + if err != nil { + t.Fatal(err) + } + // The held reader must remain readable regardless of the GC outcome. + defer func() { + if v.EntityCount() != 6 { + t.Errorf("held reader unreadable after GC: EntityCount=%d", v.EntityCount()) + } + _ = v.Close() + }() + + // Must not panic / propagate. + graph.GCStaleGens(dir, 3) + + // current + previous survive irrespective of the held handle. + for _, keep := range []uint64{2, 3} { + if _, err := os.Stat(filepath.Join(dir, graph.GenDirName(keep))); err != nil { + t.Errorf("gen %d removed but should be kept: %v", keep, err) + } + } +} + +// TestReaderForDir_SegmentSet: ReaderForDir opens a segment-set and the unified +// view sees entities/relationships across ALL segments (EntityCount, lookups, +// iteration), including cross-segment relationship endpoints. +func TestReaderForDir_SegmentSet(t *testing.T) { + dir := t.TempDir() + writeSegmentSet(t, dir, 3, threeSegDocs()) + + v, err := graph.ReaderForDir(dir) + if err != nil { + t.Fatalf("ReaderForDir: %v", err) + } + // Windows teardown lesson: close the view before t.TempDir cleanup unmaps. + t.Cleanup(func() { _ = v.Close() }) + + if got := v.EntityCount(); got != 6 { + t.Errorf("EntityCount across segments = %d, want 6", got) + } + if got := v.RelationshipCount(); got != 3 { + t.Errorf("RelationshipCount across segments = %d, want 3", got) + } + for _, id := range []string{"a", "b", "m", "n", "y", "z"} { + if e := v.LookupEntityByID(id); e == nil || string(e.Id()) != id { + t.Errorf("LookupEntityByID(%q) failed across segments: %v", id, e) + } + } + // Cross-segment relationship: y (seg2) -> a (seg0). + seen := false + v.IterateRelationships(func(rel *fbgraph.Relationship) bool { + if string(rel.FromId()) == "y" && string(rel.ToId()) == "a" { + seen = true + return false + } + return true + }) + if !seen { + t.Error("cross-segment relationship y->a not iterated") + } +} + +// TestLoadGraphFromDir_SegmentSet: the full-Document loader materializes a +// segment-set (the serve/CLI read path) with all entities + relationships. +func TestLoadGraphFromDir_SegmentSet(t *testing.T) { + dir := t.TempDir() + writeSegmentSet(t, dir, 4, threeSegDocs()) + + doc, err := graph.LoadGraphFromDir(dir) + if err != nil { + t.Fatalf("LoadGraphFromDir(segment-set): %v", err) + } + if len(doc.Entities) != 6 { + t.Errorf("Document.Entities = %d, want 6", len(doc.Entities)) + } + if len(doc.Relationships) != 3 { + t.Errorf("Document.Relationships = %d, want 3", len(doc.Relationships)) + } + if doc.Stats.Entities != 6 || doc.Stats.Relationships != 3 { + t.Errorf("Stats = %+v, want 6 ents / 3 rels", doc.Stats) + } +} + +// TestReaderForDir_SingleFileParity: ReaderForDir over a single-file gen graph +// returns exactly what today's fbreader.Open(CurrentGraphPath(dir)) does — same +// counts, same lookups. The common path is unchanged. +func TestReaderForDir_SingleFileParity(t *testing.T) { + dir := t.TempDir() + doc := &graph.Document{Repo: "single", Entities: []graph.Entity{ + {ID: "aaaa0001", QualifiedName: "p.A", Kind: "function", Name: "A"}, + {ID: "aaaa0002", QualifiedName: "p.B", Kind: "struct", Name: "B"}, + }, Relationships: []graph.Relationship{{FromID: "aaaa0001", ToID: "aaaa0002", Kind: "CALLS"}}} + // Single-file gen: write graph.1.fb + point current at it. + if err := fbwriter.WriteAtomic(filepath.Join(dir, graph.GenFileName(1)), doc); err != nil { + t.Fatal(err) + } + if err := graph.WriteCurrentPointer(dir, graph.GenFileName(1)); err != nil { + t.Fatal(err) + } + + v, err := graph.ReaderForDir(dir) + if err != nil { + t.Fatalf("ReaderForDir(single-file): %v", err) + } + t.Cleanup(func() { _ = v.Close() }) + if v.EntityCount() != 2 || v.RelationshipCount() != 1 { + t.Fatalf("single-file counts = %d/%d, want 2/1", v.EntityCount(), v.RelationshipCount()) + } + if e := v.LookupEntityByID("aaaa0002"); e == nil || string(e.Id()) != "aaaa0002" { + t.Fatalf("single-file lookup failed: %v", e) + } +} + +// TestCurrentGraphDescriptor_SegmentSet: a gen-dir + manifest + current pointer +// resolves to GraphSegmentSet with the segment paths in manifest order. +func TestCurrentGraphDescriptor_SegmentSet(t *testing.T) { + dir := t.TempDir() + genDir := writeSegmentSet(t, dir, 3, threeSegDocs()) + + desc, err := graph.CurrentGraphDescriptor(dir) + if err != nil { + t.Fatalf("CurrentGraphDescriptor: %v", err) + } + if desc.Kind != graph.GraphSegmentSet { + t.Fatalf("Kind = %v, want GraphSegmentSet", desc.Kind) + } + if desc.GenDir != genDir { + t.Errorf("GenDir = %q, want %q", desc.GenDir, genDir) + } + if len(desc.Segments) != 3 { + t.Fatalf("Segments = %d, want 3", len(desc.Segments)) + } + if desc.Manifest == nil || desc.Manifest.TotalEntityCount() != 6 { + t.Fatalf("manifest missing or wrong entity total: %+v", desc.Manifest) + } +} + +// TestCurrentGraphDescriptor_ManifestPointer: current may name the manifest +// itself (graph./manifest.json), not just the dir. +func TestCurrentGraphDescriptor_ManifestPointer(t *testing.T) { + dir := t.TempDir() + writeSegmentSet(t, dir, 5, threeSegDocs()) + // Re-point current at the manifest path form. + if err := graph.WriteCurrentPointerRaw(dir, graph.GenDirName(5)+"/"+graph.ManifestFileName); err != nil { + t.Fatal(err) + } + desc, err := graph.CurrentGraphDescriptor(dir) + if err != nil { + t.Fatal(err) + } + if desc.Kind != graph.GraphSegmentSet { + t.Fatalf("manifest-pointer form: Kind = %v, want GraphSegmentSet", desc.Kind) + } +} + +// TestCurrentGraphDescriptor_SingleFileAndLegacy: single-file gen + legacy flat +// both resolve to GraphSingleFile with the SAME path CurrentGraphPath returns +// (parity — the common path is unchanged). +func TestCurrentGraphDescriptor_SingleFileAndLegacy(t *testing.T) { + // Single-file gen. + dir1 := t.TempDir() + genPath, err := graph.WriteGenGraph(dir1, []byte("00000000")) // 8+ bytes for fbreader open guard elsewhere + if err != nil { + t.Fatal(err) + } + d1, err := graph.CurrentGraphDescriptor(dir1) + if err != nil { + t.Fatal(err) + } + if d1.Kind != graph.GraphSingleFile || d1.Path != genPath || d1.Path != graph.CurrentGraphPath(dir1) { + t.Fatalf("single-file gen: %+v (CurrentGraphPath=%q)", d1, graph.CurrentGraphPath(dir1)) + } + + // Legacy flat. + dir2 := t.TempDir() + flat := filepath.Join(dir2, "graph.fb") + if err := os.WriteFile(flat, []byte("legacybytes"), 0o644); err != nil { + t.Fatal(err) + } + d2, err := graph.CurrentGraphDescriptor(dir2) + if err != nil { + t.Fatal(err) + } + if d2.Kind != graph.GraphSingleFile || d2.Path != flat || d2.Path != graph.CurrentGraphPath(dir2) { + t.Fatalf("legacy flat: %+v (CurrentGraphPath=%q)", d2, graph.CurrentGraphPath(dir2)) + } +} + +// TestCurrentGraphDescriptor_Absent: a never-indexed dir resolves to +// GraphAbsent with Path == the flat path (parity with CurrentGraphPath). +func TestCurrentGraphDescriptor_Absent(t *testing.T) { + dir := t.TempDir() + d, err := graph.CurrentGraphDescriptor(dir) + if err != nil { + t.Fatal(err) + } + if d.Kind != graph.GraphAbsent { + t.Fatalf("Kind = %v, want GraphAbsent", d.Kind) + } + if d.Path != graph.CurrentGraphPath(dir) { + t.Errorf("absent Path = %q, want flat %q", d.Path, graph.CurrentGraphPath(dir)) + } +} + +// TestCurrentGraphDescriptor_MalformedManifestErrors: a segment-set pointer to +// a gen dir whose manifest is corrupt surfaces an error (loud, not silent). +func TestCurrentGraphDescriptor_MalformedManifestErrors(t *testing.T) { + dir := t.TempDir() + genDir := filepath.Join(dir, graph.GenDirName(2)) + if err := os.MkdirAll(genDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(genDir, graph.ManifestFileName), []byte("{ broken"), 0o644); err != nil { + t.Fatal(err) + } + if err := graph.WriteCurrentPointerRaw(dir, graph.GenDirName(2)); err != nil { + t.Fatal(err) + } + if _, err := graph.CurrentGraphDescriptor(dir); err == nil { + t.Fatal("expected error for corrupt segment-set manifest, got nil") + } +} + +// TestCurrentGraphDescriptor_MissingGenDirFallsBack: a pointer to a gen dir that +// does not exist falls back to the flat file (never an error), matching +// CurrentGraphPath's missing-gen fallback. +func TestCurrentGraphDescriptor_MissingGenDirFallsBack(t *testing.T) { + dir := t.TempDir() + flat := filepath.Join(dir, "graph.fb") + if err := os.WriteFile(flat, []byte("flatbytes"), 0o644); err != nil { + t.Fatal(err) + } + if err := graph.WriteCurrentPointerRaw(dir, graph.GenDirName(9)); err != nil { + t.Fatal(err) + } + d, err := graph.CurrentGraphDescriptor(dir) + if err != nil { + t.Fatal(err) + } + if d.Kind != graph.GraphSingleFile || d.Path != flat { + t.Fatalf("missing gen dir should fall back to flat: %+v", d) + } +} diff --git a/internal/mcp/state.go b/internal/mcp/state.go index c179aa6e3..ad1f3bb36 100644 --- a/internal/mcp/state.go +++ b/internal/mcp/state.go @@ -1403,6 +1403,19 @@ func (s *State) reloadAllLocked() (int, bool, error) { // an out-of-range slice — the mmap-reader is only meaningful // for the FlatBuffers format (gen graph..fb or flat // graph.fb), so a json-only repo correctly leaves newRdr nil. + // + // #5901 segment-set: when the active graph is the + // multi-segment gen-dir layout, lr.GraphFile does NOT carry a + // .fb extension (it is a graph./ dir or a graph.json + // fallback), so newRdr stays nil and this repo is served from + // the Document that readDocumentFromDir → LoadGraphFromDir + // already materialized segment-aware. The zero-copy mmap + // cutover (a resident *MultiReader + LabelIndex/BM25/adjacency + // sourcing rows straight from segment mmaps) is a later serve + // slice of #5890 — this dark read substrate only guarantees the + // segment-set is READABLE (via the Doc), never mis-mmapped as a + // single file. filepath.Ext of a gen dir is "" so the guard + // below already excludes it; the single-file path is unchanged. fbPath := lr.GraphFile var newRdr *fbreader.Reader if filepath.Ext(fbPath) == ".fb" {