Merge pull request #930 from DeusData/fix/ts-scale-parallel

fix: TypeScript-at-scale indexing cascade + parallel-only crash recovery

Author: DeusData

internal/cbm/cbm.c

@@ -533,25 +533,41 @@ static int count_params_from_signature(const char *sig) {
  * or spins forever (an external-scanner infinite loop the quiet-timeout kills).
  * This gives an honest guard — green iff the supervisor actually contains a real
  * fault — instead of a fixture that may stop faulting once a root cause is fixed. */
-/* Crash-supervisor per-file marker (Stage 3c skip-and-continue). In the
- * supervisor's single-threaded recovery re-run, cbm_extract_file records the
- * file it is ABOUT to process here (CBM_INDEX_MARKER_FILE) before touching it,
- * so that if this file hard-crashes the worker the parent can read the marker
- * back and learn the EXACT crasher to quarantine. Only meaningful single-
- * threaded (parallel would race the marker); the env var is set solely by the
- * supervisor during recovery, so it is a no-op on every normal run. */
-static void cbm_index_write_marker(const char *rel_path) {
+/* Crash-supervisor per-file marker JOURNAL (Stage 3c skip-and-continue,
+ * parallel-safe). Recovery re-runs are PARALLEL (there are no sequential
+ * production runs), so a single overwrite-style marker would race across
+ * workers and — worse — go stale during non-extract phases, blaming
+ * whatever file was extracted LAST (that mis-quarantined four innocent
+ * ms-typescript fixtures, one 15-minute retry at a time). Instead every
+ * worker APPENDS one short line per event: "S <rel_path>" when it STARTS
+ * work on a file, "D <rel_path>" when it finishes it. A single short
+ * append of one line is atomic in practice on every target platform, and
+ * the parent discards a torn final line by design. The parent's suspect
+ * set after a crash/hang = files with an S but no D — exactly the
+ * in-flight set; a file is only quarantined after appearing in the
+ * suspect set of TWO CONSECUTIVE failed runs, so a stale or merely
+ * unlucky in-flight file is never quarantined alone. The env var is set
+ * solely by the supervisor during recovery — a no-op on normal runs. */
+static void cbm_index_mark(const char *rel_path, char event) {
     const char *mf = getenv("CBM_INDEX_MARKER_FILE");
     if (!mf || !mf[0] || !rel_path || !rel_path[0]) {
         return;
     }
-    FILE *f = cbm_fopen(mf, "wb");
+    FILE *f = cbm_fopen(mf, "ab");
     if (f) {
-        (void)fputs(rel_path, f);
+        (void)fprintf(f, "%c %s\n", event, rel_path);
         (void)fclose(f);
     }
 }
 
+void cbm_index_mark_start(const char *rel_path) {
+    cbm_index_mark(rel_path, 'S');
+}
+
+void cbm_index_mark_done(const char *rel_path) {
+    cbm_index_mark(rel_path, 'D');
+}
+
 /* ── Crash-quarantine set (Stage 3c skip-and-continue) ──────────────────────
  * After a crash the supervisor re-runs the worker single-threaded, passing
  * CBM_INDEX_QUARANTINE_FILE — a newline-delimited list of repo-relative paths
@@ -667,9 +683,29 @@ static void cbm_test_fault_inject(const char *rel_path) {
     }
 }
 
+static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len,
+                                            CBMLanguage language, const char *project,
+                                            const char *rel_path, int64_t timeout_micros,
+                                            const char **extra_defines, const char **include_paths);
+
+/* Public entry: run the extraction and journal completion. The DONE mark on
+ * every ordinary return (including error/timeout results) tells the crash
+ * supervisor this file did NOT kill the worker — only a file whose S has no
+ * D is a crash/hang suspect. */
 CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage language,
                                 const char *project, const char *rel_path, int64_t timeout_micros,
                                 const char **extra_defines, const char **include_paths) {
+    CBMFileResult *r = cbm_extract_file_impl(source, source_len, language, project, rel_path,
+                                             timeout_micros, extra_defines, include_paths);
+    cbm_index_mark_done(rel_path);
+    return r;
+}
+
+static CBMFileResult *cbm_extract_file_impl(const char *source, int source_len,
+                                            CBMLanguage language, const char *project,
+                                            const char *rel_path, int64_t timeout_micros,
+                                            const char **extra_defines,
+                                            const char **include_paths) {
     // Allocate result on heap (arena inside for all string data)
     enum { SINGLE = 1 };
     CBMFileResult *result = (CBMFileResult *)calloc(SINGLE, sizeof(CBMFileResult));
@@ -691,7 +727,7 @@ CBMFileResult *cbm_extract_file(const char *source, int source_len, CBMLanguage
         return result;
     }
 
-    cbm_index_write_marker(rel_path);
+    cbm_index_mark_start(rel_path);
     cbm_test_fault_inject(rel_path);
 
     // Get language spec

internal/cbm/cbm.h

@@ -532,6 +532,15 @@ bool cbm_index_is_quarantined(const char *rel_path);
 // extract loops to report the skip's phase in skipped[] (falls back to "crash").
 const char *cbm_index_quarantine_phase(const char *rel_path);
 
+// Crash-supervisor marker journal (parallel-safe): appends "S <rel_path>" /
+// "D <rel_path>" to CBM_INDEX_MARKER_FILE. Files with an S but no D form the
+// parent's crash/hang suspect set. No-ops when the env var is unset.
+// cbm_extract_file journals its own start/done; long-running per-file phases
+// (cross-LSP resolve) call these around their per-file work so a hang there
+// is attributed to the RIGHT file instead of a stale extraction marker.
+void cbm_index_mark_start(const char *rel_path);
+void cbm_index_mark_done(const char *rel_path);
+
 // Extract all data from one file. Caller must call cbm_free_result().
 // source must remain valid for the duration of the call.
 // timeout_micros: per-file parse timeout in microseconds (0 = no timeout).

internal/cbm/extract_defs.c

@@ -5691,6 +5691,58 @@ static void wd_push(wd_stack_t *s, TSNode node, const char *enclosing_qn) {
     s->data[s->top++] = (walk_defs_frame_t){node, enclosing_qn};
 }
 
+/* Push all children of `node` in REVERSE order (so they pop in source order)
+ * using a LINEAR traversal. Index-based ts_node_child(i) is O(i) per call in
+ * tree-sitter, so a reverse index loop is O(n^2) per node — a file whose
+ * root has ~580k flat siblings (ms-typescript's reallyLargeFile.ts fourslash
+ * fixture) needed ~1.7e11 child-iterator steps and hung extraction for
+ * hours; the supervisor then killed the silent worker as a hang and
+ * quarantined innocent files off the stale marker. Collect the children
+ * FORWARD with a TSTreeCursor (O(1) amortized per step) into a scratch
+ * buffer, then push in reverse. Small nodes keep the direct index loop —
+ * no cursor/buffer overhead on the overwhelmingly common case. */
+enum { WD_CURSOR_MIN_CHILDREN = 64 };
+
+/* Collect all `cc` children of `node` linearly via a TSTreeCursor into a
+ * malloc'd array (caller frees). Returns NULL for small nodes and on OOM —
+ * the caller then uses indexed ts_node_child access, which is fine (and
+ * cheaper) at small child counts and merely quadratic-but-correct on OOM. */
+static TSNode *wd_collect_children(TSNode node, uint32_t cc) {
+    if (cc < WD_CURSOR_MIN_CHILDREN) {
+        return NULL;
+    }
+    TSNode *buf = (TSNode *)malloc((size_t)cc * sizeof(TSNode));
+    if (!buf) {
+        return NULL;
+    }
+    TSTreeCursor cur = ts_tree_cursor_new(node);
+    uint32_t got = 0;
+    if (ts_tree_cursor_goto_first_child(&cur)) {
+        do {
+            buf[got++] = ts_tree_cursor_current_node(&cur);
+        } while (got < cc && ts_tree_cursor_goto_next_sibling(&cur));
+    }
+    ts_tree_cursor_delete(&cur);
+    if (got != cc) {
+        /* Defensive: cursor and child_count disagree — fall back to indexed. */
+        free(buf);
+        return NULL;
+    }
+    return buf;
+}
+
+static void wd_push_children_reverse(wd_stack_t *s, TSNode node, const char *enclosing_qn) {
+    uint32_t cc = ts_node_child_count(node);
+    if (cc == 0) {
+        return;
+    }
+    TSNode *kids = wd_collect_children(node, cc);
+    for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) {
+        wd_push(s, kids ? kids[i] : ts_node_child(node, (uint32_t)i), enclosing_qn);
+    }
+    free(kids);
+}
+
 // Push nested class nodes from a class body container onto the defs stack.
 // Iteratively walks into wrapper nodes (field_declaration, template_declaration).
 static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_stack_t *s,
@@ -5702,8 +5754,10 @@ static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_sta
     while (nc_stack.count > 0) {
         TSNode cur = ts_nstack_pop(&nc_stack);
         uint32_t nc = ts_node_child_count(cur);
+        /* Linear child access for wide class bodies (see wd_collect_children). */
+        TSNode *kids = wd_collect_children(cur, nc);
         for (int i = (int)nc - SKIP_CHAR; i >= 0; i--) {
-            TSNode child = ts_node_child(cur, (uint32_t)i);
+            TSNode child = kids ? kids[i] : ts_node_child(cur, (uint32_t)i);
             if (cbm_kind_in_set(child, spec->class_node_types)) {
                 wd_push(s, child, enclosing_qn);
             } else {
@@ -5714,6 +5768,7 @@ static void push_nested_class_nodes(TSNode body, const CBMLangSpec *spec, wd_sta
                 }
             }
         }
+        free(kids);
     }
 }
 
@@ -6220,10 +6275,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
             // resolve a null name and, for grammars where the kind has a `name`
             // field, double-mint). Push children so nested tags/defs are still
             // traversed, then skip the generic func path.
-            uint32_t cc = ts_node_child_count(node);
-            for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) {
-                wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn);
-            }
+            wd_push_children_reverse(&s, node, frame.enclosing_class_qn);
             continue;
         }
 
@@ -6234,10 +6286,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
             // generic extract_func_def below would double-mint a def whose name
             // still carries the quotes. Push children so nested defines are still
             // traversed, then skip the generic func path.
-            uint32_t cc = ts_node_child_count(node);
-            for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) {
-                wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn);
-            }
+            wd_push_children_reverse(&s, node, frame.enclosing_class_qn);
             continue;
         }
 
@@ -6262,10 +6311,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
             (strcmp(kind, "export_item") == 0 || strcmp(kind, "import_item") == 0) &&
             ts_node_is_null(
                 find_first_descendant_by_kind(node, "func_type", CBM_DESCENDANT_MAX_DEPTH))) {
-            uint32_t cc = ts_node_child_count(node);
-            for (int i = (int)cc - SKIP_CHAR; i >= 0; i--) {
-                wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn);
-            }
+            wd_push_children_reverse(&s, node, frame.enclosing_class_qn);
             continue;
         }
 
@@ -6303,10 +6349,7 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
          * the class/func paths on the namespace node itself. */
         if (is_namespace_scope_kind(ctx->language, kind)) {
             const char *new_enclosing = compute_class_qn(ctx, node, frame.enclosing_class_qn);
-            uint32_t nsc = ts_node_child_count(node);
-            for (int i = (int)nsc - SKIP_CHAR; i >= 0; i--) {
-                wd_push(&s, ts_node_child(node, (uint32_t)i), new_enclosing);
-            }
+            wd_push_children_reverse(&s, node, new_enclosing);
             continue;
         }
 
@@ -6317,10 +6360,11 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
             continue;
         }
 
-        uint32_t count = ts_node_child_count(node);
-        for (int i = (int)count - SKIP_CHAR; i >= 0; i--) {
-            wd_push(&s, ts_node_child(node, (uint32_t)i), frame.enclosing_class_qn);
-        }
+        /* Default descent — THE hot loop: a file whose root has hundreds of
+         * thousands of flat siblings (580k comment lines in ms-typescript's
+         * reallyLargeFile.ts) lands here every visit, so linear child
+         * collection is mandatory (see wd_push_children_reverse). */
+        wd_push_children_reverse(&s, node, frame.enclosing_class_qn);
     }
     free(s.data);
 }

internal/cbm/lsp/ts_lsp.c

@@ -27,10 +27,49 @@
  */
 
 #include "ts_lsp.h"
+#include <stdatomic.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
+/* TEST HOOK: full per-file cross-registry builds (stdlib + every cross-file
+ * def registered + finalized, per file). This is the sequential-mode quadratic
+ * that ground an 81k-file TS corpus for hours — the shared-registry dispatch
+ * must keep it at ZERO; any nonzero count means a resolve path regressed to
+ * the per-file build. */
+static _Atomic long g_ts_full_reg_builds;
+long cbm_ts_full_registry_builds(void) {
+    return atomic_load(&g_ts_full_reg_builds);
+}
+void cbm_ts_full_registry_builds_reset(void) {
+    atomic_store(&g_ts_full_reg_builds, 0);
+}
+
+/* Dynamic work budget for parse_ts_type_text (thread-local, reset at every
+ * per-file/per-build entry point). The type-text parser is self-recursive
+ * over string SLICES and re-derives lengths per call, so an adversarial or
+ * generated type text can cost far more than its bytes suggest. Instead of
+ * a fixed depth cap, the total work SCALES WITH THE INPUT: budget =
+ * 1M + 64 x source_len units, each call charging 1 + slice_len/16 — i.e.
+ * total scanned bytes are bounded at ~1024x the source size, generous
+ * enough that no real-world file ever hits it. CBM_TS_TYPE_BUDGET (absolute
+ * units) overrides. On exhaustion: WARN once per window, then return
+ * UNKNOWN — the same graceful degradation the parser already uses for
+ * constructs it does not model. -1 = unlimited (no entry point armed it). */
+static _Thread_local long g_ts_type_budget = -1;
+static _Thread_local bool g_ts_type_budget_warned;
+
+static void ts_type_budget_reset(size_t source_len) {
+    const char *e = getenv("CBM_TS_TYPE_BUDGET");
+    if (e && e[0]) {
+        long v = atol(e);
+        g_ts_type_budget = (v > 0) ? v : -1;
+    } else {
+        g_ts_type_budget = 1000000 + (long)source_len * 64;
+    }
+    g_ts_type_budget_warned = false;
+}
+
 #define TS_LSP_MAX_EVAL_DEPTH 64
 #define TS_LSP_FIELD_LEN(s) ((uint32_t)(sizeof(s) - 1))
 
@@ -156,6 +195,22 @@ static const CBMType *parse_ts_type_text(CBMArena *arena, const char *text, cons
     if (len == 0)
         return cbm_type_unknown();
 
+    /* Dynamic budget (see g_ts_type_budget): charge proportional to this
+     * slice; on exhaustion degrade to UNKNOWN instead of grinding. */
+    if (g_ts_type_budget >= 0) {
+        g_ts_type_budget -= 1 + (long)(len / 16);
+        if (g_ts_type_budget < 0) {
+            if (!g_ts_type_budget_warned) {
+                g_ts_type_budget_warned = true;
+                fprintf(stderr,
+                        "  [tslsp] type-text parse budget exhausted (module %s); "
+                        "returning unknown\n",
+                        module_qn ? module_qn : "?");
+            }
+            return cbm_type_unknown();
+        }
+    }
+
     // Function type `(params) => returnType` (TS function type literal).
     if (text[0] == '(') {
         int depth = 0;
@@ -4784,6 +4839,7 @@ void cbm_run_ts_lsp(CBMArena *arena, CBMFileResult *result, const char *source,
                     TSNode root, bool js_mode, bool jsx_mode, bool dts_mode) {
     if (!arena || !result || !source || ts_node_is_null(root))
         return;
+    ts_type_budget_reset((size_t)source_len);
 
     // Diagnostic / benchmarking knob: setting `CBM_LSP_DISABLED=1` skips the resolver.
     // This is used by the baseline-vs-LSP comparison tests to measure how many calls
@@ -4977,6 +5033,8 @@ CBMTypeRegistry *cbm_ts_build_cross_registry(CBMArena *arena, CBMLSPDef *defs, i
         return NULL;
     cbm_registry_init(reg, arena);
     cbm_ts_stdlib_register(reg, arena);
+    /* Budget scales with the def volume this build parses type texts for. */
+    ts_type_budget_reset((size_t)(def_count > 0 ? def_count : 1) * 1024);
     for (int i = 0; i < def_count; i++) {
         CBMLSPDef *d = &defs[i];
         if (d->lang != CBM_LANG_JAVASCRIPT && d->lang != CBM_LANG_TYPESCRIPT &&
@@ -5006,6 +5064,8 @@ void cbm_run_ts_lsp_cross_with_registry(CBMArena *arena, const char *source, int
     if (!arena || !out || !reg)
         return;
 
+    ts_type_budget_reset((size_t)source_len);
+
     /* Per-file overlay: register only the file's own-module defs so the
      * AST passes can refine them. Imports/stdlib resolve via fallback. */
     CBMTypeRegistry overlay;
@@ -5075,6 +5135,8 @@ void cbm_run_ts_lsp_cross(CBMArena *arena, const char *source, int source_len,
     if (!arena || !out)
         return;
 
+    atomic_fetch_add(&g_ts_full_reg_builds, 1);
+    ts_type_budget_reset((size_t)source_len);
     CBMTypeRegistry reg;
     cbm_registry_init(&reg, arena);
     cbm_ts_stdlib_register(&reg, arena);

internal/cbm/lsp/ts_lsp.h

@@ -123,6 +123,11 @@ void cbm_run_ts_lsp_cross_with_registry(CBMArena *arena, const char *source, int
 // will replace this in v1.3.
 void cbm_ts_stdlib_register(CBMTypeRegistry *reg, CBMArena *arena);
 
+// TEST HOOKS: count of full per-file cross-registry builds (the quadratic the
+// shared-registry dispatch eliminates; must stay 0 on the shared path).
+long cbm_ts_full_registry_builds(void);
+void cbm_ts_full_registry_builds_reset(void);
+
 // --- Batch cross-file LSP ---
 
 // Per-file input for batch TS LSP processing.

src/mcp/index_supervisor.c

@@ -53,6 +53,16 @@ int cbm_index_supervisor_spawn_count(void) {
     return g_spawn_count;
 }
 
+/* Test hook: counts SINGLE-THREADED spawns. Production recovery is parallel-
+ * only (there are no sequential production runs); this must stay ZERO on
+ * every supervised path — any nonzero count means a recovery/probe regressed
+ * to the sequential crawl that ground an 81k-file TS corpus for hours. */
+static int g_spawn_st_count = 0;
+
+int cbm_index_supervisor_spawn_st_count(void) {
+    return g_spawn_st_count;
+}
+
 /* #845: opt-in host mark — see the header. Set once from the real binary's
  * main(); embedders never set it, so should_wrap() stays false for them. */
 static bool g_host_marked = false;
@@ -137,6 +147,9 @@ static void worker_tmp_path(char *out, size_t out_sz, int pid, const char *suffi
 int cbm_index_spawn_worker(const char *args_json, bool single_thread, const char *marker_file,
                            const char *quarantine_file, cbm_index_worker_result_t *result) {
     g_spawn_count++; /* test hook (#845) — see cbm_index_supervisor_spawn_count */
+    if (single_thread) {
+        g_spawn_st_count++; /* test hook — must stay 0: recovery is parallel-only */
+    }
     result->outcome = CBM_PROC_SPAWN_FAILED;
     result->exit_code = -1;
     result->term_signal = 0;