perf(catalog): O(1) unit name lookups + source-build progress

`refresh catalog source` resolved each document by re-reading and
re-bson.Unmarshalling every unit on every describe call (GetRawUnitByName /
GetRawMicroflowByName scanned the full unit list per call). With one describe per
document, that's O(N²) — ~6 hours on a large app (#651).

Fix: the reader builds a one-time index keyed by "$Type\x00QualifiedName" → unit,
decoding only the Name field (a small struct, far cheaper than map[string]any).
Lookups become O(1); the catalog's parallel describe shares one backend, so the
index is built once and reused. Invalidated by InvalidateCache after writes.

Also: buildSource Phase 2 reports incremental progress every 2s (was silent for
the whole multi-minute/hour build — the #651 "no progress" complaint).

Verified: GraphViewer source build (993 microflows) ~3.5min with live progress;
the O(N²) (993² unmarshals ≈ 80min for microflows alone) is gone.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: ako

CHANGELOG.md

@@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ### Fixed
 
+- **`refresh catalog source` no longer O(N²) on large projects** — it resolved each document by re-reading and re-`bson.Unmarshal`ing *every* unit on *every* describe call, so a big app (#651: ~3.3k microflows, ~33k activities) took ~6 hours. The reader now builds a one-time `$Type + qualified-name → unit` index (decoding only the `Name` field, not the whole document), making `GetRawUnitByName` / `GetRawMicroflowByName` O(1); the shared backend means the index is built once across the parallel describe workers. The source phase also reports incremental progress every 2s instead of going silent for the whole build. GraphViewer's source build (993 microflows) dropped to ~3.5 min with live progress; cloud-portal-scale projects go from hours to minutes
 - **Marketplace search now scans the whole catalog** — the Content API has no server-side search and caps `limit` at 100 per page, so `marketplace search` previously only filtered the first 100 items and silently missed matches further in (e.g. External Database Connector `219862`, Mendix Business Events `202649`). It now paginates via `offset`, fetching pages **concurrently** (first page alone so a common early match stays a single request; then bounded-parallel batches), and stops at `--limit` matches or end-of-catalog. Measured ~3m45s → ~44s on a slow link for a deep match; combined with the new cache, repeat searches are instant
 
 ## [0.12.0] - 2026-06-04

mdl/catalog/builder_source.go

@@ -3,8 +3,11 @@
 package catalog
 
 import (
+	"fmt"
 	"runtime"
 	"sync"
+	"sync/atomic"
+	"time"
 )
 
 // sourceItem represents a single element to generate MDL source for.
@@ -110,12 +113,31 @@ func (b *Builder) buildSource() error {
 		return nil
 	}
 
-	// Phase 2: Generate MDL source in parallel
+	// Phase 2: Generate MDL source in parallel. This is the slow phase on large
+	// projects (one describe per document), so report progress periodically —
+	// from a single ticker goroutine, the only caller of report() during the
+	// run, so the progress sink isn't written concurrently.
 	numWorkers := max(min(runtime.NumCPU(), 8), 1)
 
 	results := make([]sourceResult, len(items))
 	work := make(chan int, len(items))
 
+	var done atomic.Int64
+	total := len(items)
+	stop := make(chan struct{})
+	go func() {
+		ticker := time.NewTicker(2 * time.Second)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-ticker.C:
+				b.report(fmt.Sprintf("source %d/%d documents", done.Load(), total), int(done.Load()))
+			}
+		}
+	}()
+
 	var wg sync.WaitGroup
 	for range numWorkers {
 		wg.Go(func() {
@@ -125,6 +147,7 @@ func (b *Builder) buildSource() error {
 				if err == nil && text != "" {
 					results[idx] = sourceResult{item, text}
 				}
+				done.Add(1)
 			}
 		})
 	}
@@ -134,6 +157,7 @@ func (b *Builder) buildSource() error {
 	}
 	close(work)
 	wg.Wait()
+	close(stop)
 
 	// Phase 3: Insert results into FTS5 table (serial — SQLite constraint)
 	stmt, err := b.tx.Prepare(`

sdk/mpr/reader.go

@@ -39,6 +39,12 @@ type Reader struct {
 	// Cache for unit metadata to avoid repeated file reads
 	unitCache      []cachedUnit
 	unitCacheValid bool
+
+	// Lazily-built index of (unit $Type + qualified name) → unit, so name
+	// lookups are O(1) instead of re-scanning and re-parsing every unit per
+	// call (the source-catalog build does thousands of such lookups).
+	nameIndex      map[string]nameIndexEntry
+	nameIndexBuilt bool
 }
 
 // cachedUnit stores metadata about a unit for fast filtering.

sdk/mpr/reader_units.go

@@ -181,6 +181,8 @@ func (r *Reader) buildUnitCache() error {
 // Should be called after any write operation.
 func (r *Reader) InvalidateCache() {
 	r.unitCacheValid = false
+	r.nameIndex = nil
+	r.nameIndexBuilt = false
 }
 
 // readMprContents reads content from the mprcontents folder for v2 format.
@@ -220,40 +222,90 @@ func getTypeFromContents(contents []byte) string {
 	return ""
 }
 
-// GetRawMicroflowByName returns the raw BSON contents for a microflow by qualified name.
-// Used for debugging to compare serialized data.
-func (r *Reader) GetRawMicroflowByName(qualifiedName string) ([]byte, error) {
-	units, err := r.listUnitsByType("Microflows$Microflow")
+// nameIndexEntry records where a named unit lives.
+type nameIndexEntry struct {
+	id         string
+	moduleName string
+}
+
+// nameHeader decodes only the unit's Name field. Decoding into this small
+// struct is much cheaper than unmarshalling the whole document into
+// map[string]any (no map allocation, far less reflection).
+type nameHeader struct {
+	Name string `bson:"Name"`
+}
+
+// buildUnitNameIndex parses every unit's name once and indexes units by
+// "$Type\x00QualifiedName", so per-name lookups are O(1) instead of re-reading
+// and re-parsing every unit on each call. Idempotent; invalidated by
+// InvalidateCache after writes.
+func (r *Reader) buildUnitNameIndex() error {
+	if r.nameIndexBuilt {
+		return nil
+	}
+	units, err := r.listUnitsByType("")
 	if err != nil {
-		return nil, err
+		return err
 	}
-
-	// Build a map of container ID to module name
 	modules, err := r.ListModules()
 	if err != nil {
-		return nil, err
+		return err
 	}
-	moduleMap := make(map[string]string)
+	moduleMap := make(map[string]string, len(modules))
 	for _, m := range modules {
 		moduleMap[string(m.ID)] = m.Name
 	}
+	containerParent, err := r.buildContainerParent()
+	if err != nil {
+		return err
+	}
 
+	idx := make(map[string]nameIndexEntry, len(units))
 	for _, u := range units {
-		// Parse just enough to get the qualified name
-		var raw map[string]any
-		if err := bson.Unmarshal(u.Contents, &raw); err != nil {
+		var h nameHeader
+		if err := bson.Unmarshal(u.Contents, &h); err != nil || h.Name == "" {
 			continue
 		}
-
-		name, _ := raw["Name"].(string)
-		// Get module name from container
-		moduleName := moduleMap[u.ContainerID]
-		if moduleName != "" && moduleName+"."+name == qualifiedName {
-			return u.Contents, nil
+		moduleName := resolveModuleName(u.ContainerID, moduleMap, containerParent)
+		qn := h.Name
+		if moduleName != "" {
+			qn = moduleName + "." + h.Name
 		}
+		idx[u.Type+"\x00"+qn] = nameIndexEntry{id: u.ID, moduleName: moduleName}
 	}
+	r.nameIndex = idx
+	r.nameIndexBuilt = true
+	return nil
+}
 
-	return nil, fmt.Errorf("microflow not found: %s", qualifiedName)
+// lookupUnitByName resolves a (type, qualified name) to its unit via the name
+// index. Returns (nil, "", nil) when not found.
+func (r *Reader) lookupUnitByName(typePrefix, qualifiedName string) (*rawUnit, string, error) {
+	if err := r.buildUnitNameIndex(); err != nil {
+		return nil, "", err
+	}
+	e, ok := r.nameIndex[typePrefix+"\x00"+qualifiedName]
+	if !ok {
+		return nil, "", nil
+	}
+	u, err := r.getUnitByID(e.id)
+	if err != nil {
+		return nil, "", err
+	}
+	return u, e.moduleName, nil
+}
+
+// GetRawMicroflowByName returns the raw BSON contents for a microflow by qualified name.
+// Used for debugging to compare serialized data.
+func (r *Reader) GetRawMicroflowByName(qualifiedName string) ([]byte, error) {
+	u, _, err := r.lookupUnitByName("Microflows$Microflow", qualifiedName)
+	if err != nil {
+		return nil, err
+	}
+	if u == nil {
+		return nil, fmt.Errorf("microflow not found: %s", qualifiedName)
+	}
+	return u.Contents, nil
 }
 
 // RawUnitInfo contains information about a raw unit for BSON debugging.
@@ -309,54 +361,20 @@ func (r *Reader) GetRawUnitByName(objectType, qualifiedName string) (*RawUnitInf
 		return r.getRawAssociationByName(qualifiedName)
 	}
 
-	units, err := r.listUnitsByType(typePrefix)
-	if err != nil {
-		return nil, err
-	}
-
-	// Build module name map and container hierarchy for MPR v2 folder support.
-	modules, err := r.ListModules()
+	u, moduleName, err := r.lookupUnitByName(typePrefix, qualifiedName)
 	if err != nil {
 		return nil, err
 	}
-	moduleMap := make(map[string]string)
-	for _, m := range modules {
-		moduleMap[string(m.ID)] = m.Name
+	if u == nil {
+		return nil, fmt.Errorf("%s not found: %s", objectType, qualifiedName)
 	}
-	containerParent, err := r.buildContainerParent()
-	if err != nil {
-		return nil, err
-	}
-
-	for _, u := range units {
-		var raw map[string]any
-		if err := bson.Unmarshal(u.Contents, &raw); err != nil {
-			continue
-		}
-
-		name, _ := raw["Name"].(string)
-		moduleName := resolveModuleName(u.ContainerID, moduleMap, containerParent)
-
-		// Build full name, handling missing module
-		var fullName string
-		if moduleName != "" {
-			fullName = moduleName + "." + name
-		} else {
-			fullName = name
-		}
-
-		if fullName == qualifiedName {
-			return &RawUnitInfo{
-				ID:            u.ID,
-				QualifiedName: fullName,
-				Type:          u.Type,
-				ModuleName:    moduleName,
-				Contents:      u.Contents,
-			}, nil
-		}
-	}
-
-	return nil, fmt.Errorf("%s not found: %s", objectType, qualifiedName)
+	return &RawUnitInfo{
+		ID:            u.ID,
+		QualifiedName: qualifiedName,
+		Type:          u.Type,
+		ModuleName:    moduleName,
+		Contents:      u.Contents,
+	}, nil
 }
 
 // getRawEntityByName finds an entity within domain models.