Merge branch 'main' into fix/java-enum-extraction

Author: sahil-mangla

.github/workflows/_build.yml

@@ -8,9 +8,15 @@ on:
         description: 'Version string (e.g. v0.8.0)'
         type: string
         default: ''
+      attest:
+        description: 'Generate build provenance attestations for release artifacts'
+        type: boolean
+        default: false
 
 permissions:
   contents: read
+  id-token: write
+  attestations: write
 
 jobs:
   build-unix:
@@ -83,6 +89,12 @@ jobs:
           tar -czf "codebase-memory-mcp-${GOOS}-${GOARCH}.tar.gz" \
             -C build/c codebase-memory-mcp LICENSE install.sh THIRD_PARTY_NOTICES.md
 
+      - name: Attest standard binary provenance
+        if: ${{ inputs.attest }}
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+        with:
+          subject-path: codebase-memory-mcp-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz
+
       - name: Build UI binary
         env:
           VERSION: ${{ inputs.version }}
@@ -113,6 +125,12 @@ jobs:
           tar -czf "codebase-memory-mcp-ui-${GOOS}-${GOARCH}.tar.gz" \
             -C build/c codebase-memory-mcp LICENSE install.sh THIRD_PARTY_NOTICES.md
 
+      - name: Attest UI binary provenance
+        if: ${{ inputs.attest }}
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+        with:
+          subject-path: codebase-memory-mcp-ui-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz
+
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: binaries-${{ matrix.goos }}-${{ matrix.goarch }}
@@ -158,6 +176,12 @@ jobs:
           scripts/gen-third-party-notices.sh THIRD_PARTY_NOTICES.md
           zip codebase-memory-mcp-windows-amd64.zip codebase-memory-mcp.exe LICENSE install.ps1 THIRD_PARTY_NOTICES.md
 
+      - name: Attest standard binary provenance
+        if: ${{ inputs.attest }}
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+        with:
+          subject-path: codebase-memory-mcp-windows-amd64.zip
+
       - name: Build UI binary
         shell: msys2 {0}
         env:
@@ -178,6 +202,12 @@ jobs:
           scripts/gen-third-party-notices.sh THIRD_PARTY_NOTICES.md
           zip codebase-memory-mcp-ui-windows-amd64.zip codebase-memory-mcp.exe LICENSE install.ps1 THIRD_PARTY_NOTICES.md
 
+      - name: Attest UI binary provenance
+        if: ${{ inputs.attest }}
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+        with:
+          subject-path: codebase-memory-mcp-ui-windows-amd64.zip
+
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: binaries-windows-amd64
@@ -227,6 +257,12 @@ jobs:
           scripts/gen-third-party-notices.sh THIRD_PARTY_NOTICES.md
           zip codebase-memory-mcp-windows-arm64.zip codebase-memory-mcp.exe LICENSE install.ps1 THIRD_PARTY_NOTICES.md
 
+      - name: Attest standard binary provenance
+        if: ${{ inputs.attest }}
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+        with:
+          subject-path: codebase-memory-mcp-windows-arm64.zip
+
       - name: Build UI binary
         shell: msys2 {0}
         env:
@@ -247,6 +283,12 @@ jobs:
           scripts/gen-third-party-notices.sh THIRD_PARTY_NOTICES.md
           zip codebase-memory-mcp-ui-windows-arm64.zip codebase-memory-mcp.exe LICENSE install.ps1 THIRD_PARTY_NOTICES.md
 
+      - name: Attest UI binary provenance
+        if: ${{ inputs.attest }}
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+        with:
+          subject-path: codebase-memory-mcp-ui-windows-arm64.zip
+
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: binaries-windows-arm64
@@ -299,6 +341,12 @@ jobs:
           tar -czf "codebase-memory-mcp-linux-${ARCH}-portable.tar.gz" \
             -C build/c codebase-memory-mcp LICENSE install.sh THIRD_PARTY_NOTICES.md
 
+      - name: Attest standard binary provenance
+        if: ${{ inputs.attest }}
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+        with:
+          subject-path: codebase-memory-mcp-linux-${{ matrix.arch }}-portable.tar.gz
+
       - name: Build UI binary (static)
         env:
           VERSION: ${{ inputs.version }}
@@ -318,6 +366,12 @@ jobs:
           tar -czf "codebase-memory-mcp-ui-linux-${ARCH}-portable.tar.gz" \
             -C build/c codebase-memory-mcp LICENSE install.sh THIRD_PARTY_NOTICES.md
 
+      - name: Attest UI binary provenance
+        if: ${{ inputs.attest }}
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
+        with:
+          subject-path: codebase-memory-mcp-ui-linux-${{ matrix.arch }}-portable.tar.gz
+
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: binaries-linux-${{ matrix.arch }}-portable

.github/workflows/_smoke.yml

@@ -248,9 +248,20 @@ jobs:
         run: |
           & "C:\Program Files\Windows Defender\MpCmdRun.exe" -SignatureUpdate 2>$null
           $result = & "C:\Program Files\Windows Defender\MpCmdRun.exe" -Scan -ScanType 3 -File "$PWD\codebase-memory-mcp.exe" -DisableRemediation
+          $code = $LASTEXITCODE
           Write-Host $result
-          if ($LASTEXITCODE -ne 0) { Write-Host "BLOCKED: Windows Defender flagged binary!"; exit 1 }
-          Write-Host "=== Windows Defender: clean ==="
+          # MpCmdRun -Scan exit codes: 0 = clean, 2 = threat found. Any OTHER non-zero
+          # means the scan engine could not run at all (e.g. hr=0x800106ba: the Defender
+          # antimalware service is unavailable on the runner) — that is NOT a detection.
+          # Fail soft on an engine failure so a transient runner-side AV outage can't
+          # false-fail a release; only a real detection (exit 2) hard-blocks.
+          if ($code -eq 2) {
+            Write-Host "BLOCKED: Windows Defender flagged binary!"; exit 1
+          } elseif ($code -ne 0) {
+            Write-Host "::warning::Windows Defender scan could not run (exit $code) - skipping AV gate on this runner"
+          } else {
+            Write-Host "=== Windows Defender: clean ==="
+          }
 
   smoke-linux-portable:
     needs: setup-matrix

.github/workflows/_test.yml

@@ -88,6 +88,18 @@ jobs:
         env:
           CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }}
 
+  test-tsan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Install deps (Ubuntu)
+        run: sudo apt-get update && sudo apt-get install -y clang zlib1g-dev
+
+      - name: ThreadSanitizer tests
+        run: make -f Makefile.cbm test-tsan CC=clang CXX=clang++
+
   test-windows:
     needs: setup-matrix
     strategy:

.github/workflows/dry-run.yml

@@ -56,7 +56,13 @@ jobs:
   build:
     if: ${{ inputs.skip_builds != true && !cancelled() && (needs.test.result == 'success' || needs.test.result == 'skipped') }}
     needs: [test]
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
     uses: ./.github/workflows/_build.yml
+    with:
+      attest: false
 
   # ── Smoke test every binary ────────────────────────────────────
   # Run unless builds were skipped or a build leg FAILED. Every build leg

.github/workflows/release.yml

@@ -56,9 +56,14 @@ jobs:
   # ── 3. Build all platforms ──────────────────────────────────────
   build:
     needs: [test]
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
     uses: ./.github/workflows/_build.yml
     with:
       version: ${{ inputs.version }}
+      attest: true
 
   # ── 4. Smoke test every binary ──────────────────────────────────
   smoke:
@@ -99,10 +104,10 @@ jobs:
       - name: Generate checksums
         run: sha256sum *.tar.gz *.zip > checksums.txt
 
-      - name: Attest build provenance
+      - name: Attest checksum provenance
         uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1
         with:
-          subject-path: '*.tar.gz,*.zip,checksums.txt'
+          subject-path: checksums.txt
 
       - name: Generate SBOM
         run: |

Makefile.cbm

@@ -67,10 +67,11 @@ CFLAGS_TEST = $(CFLAGS_COMMON) -g -O1 $(SANITIZE)
 CXXFLAGS_TEST = $(CXXFLAGS_COMMON) -g -O1 $(SANITIZE)
 
 # TSan (can't combine with ASan)
+TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer
 CFLAGS_TSAN = $(CFLAGS_COMMON) -g -O1 \
-              -fsanitize=thread -fno-omit-frame-pointer
+              $(TSAN_SANITIZE)
 CXXFLAGS_TSAN = $(CXXFLAGS_COMMON) -g -O1 \
-                -fsanitize=thread -fno-omit-frame-pointer
+                $(TSAN_SANITIZE)
 
 # Windows needs ws2_32 (Winsock), psapi (GetProcessMemoryInfo), shell32
 # (CommandLineToArgvW — the UTF-8 argv path in main.c, #423/#20), and
@@ -89,7 +90,7 @@ endif
 
 LDFLAGS = -lm -lstdc++ -lpthread -lz $(WIN32_LIBS) $(STATIC_FLAGS)
 LDFLAGS_TEST = -lm -lstdc++ -lpthread -lz $(SANITIZE) $(WIN32_LIBS)
-LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz -fsanitize=thread $(WIN32_LIBS)
+LDFLAGS_TSAN = -lm -lstdc++ -lpthread -lz $(TSAN_SANITIZE) $(WIN32_LIBS)
 
 # ── Source files ─────────────────────────────────────────────────
 
@@ -317,6 +318,7 @@ TEST_EXTRACTION_SRCS = \
     tests/test_extraction.c \
     tests/test_extraction_inheritance.c \
     tests/test_extraction_imports.c \
+    tests/test_parse_coverage.c \
     tests/test_grammar_regression.c \
     tests/test_grammar_labels.c \
     tests/test_grammar_imports.c \
@@ -445,6 +447,7 @@ TEST_REPRO_SRCS = \
     tests/repro/repro_issue363.c \
     tests/repro/repro_issue581.c \
     tests/repro/repro_issue787.c \
+    tests/repro/repro_issue842.c \
     tests/repro/repro_invariant_calls.c \
     tests/repro/repro_invariant_graph.c \
     tests/repro/repro_invariant_breadth.c \
@@ -483,13 +486,17 @@ GRAMMAR_CFLAGS = -std=c11 -D_DEFAULT_SOURCE -O2 -w -I$(CBM_DIR) -I$(TS_INCLUDE)
 GRAMMAR_CFLAGS_TEST = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
                       $(SANITIZE)
 GRAMMAR_CFLAGS_TSAN = -std=c11 -D_DEFAULT_SOURCE -g -O1 -w -I$(CBM_DIR) -I$(TS_INCLUDE) -I$(TS_SRC) \
-                      -fsanitize=thread -fno-omit-frame-pointer
+                      $(TSAN_SANITIZE)
 
 # Object files for grammars + ts_runtime + lsp_all + preprocessor
 GRAMMAR_OBJS_TEST = $(patsubst $(CBM_DIR)/%.c,$(BUILD_DIR)/%.o,$(GRAMMAR_SRCS))
 TS_RUNTIME_OBJ_TEST = $(BUILD_DIR)/ts_runtime.o
 LSP_OBJ_TEST = $(BUILD_DIR)/lsp_all.o
 PP_OBJ_TEST = $(BUILD_DIR)/preprocessor.o
+GRAMMAR_OBJS_TSAN = $(patsubst $(CBM_DIR)/%.c,$(BUILD_DIR)/tsan_%.o,$(GRAMMAR_SRCS))
+TS_RUNTIME_OBJ_TSAN = $(BUILD_DIR)/tsan_ts_runtime.o
+LSP_OBJ_TSAN = $(BUILD_DIR)/tsan_lsp_all.o
+PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o
 
 # ── Targets ──────────────────────────────────────────────────────
 
@@ -520,60 +527,94 @@ $(BUILD_DIR)/lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
 $(BUILD_DIR)/preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
 	$(CXX) $(CXXFLAGS_TEST) -w -I$(CBM_DIR)/vendored -c -o $@ $<
 
+$(BUILD_DIR)/tsan_%.o: $(CBM_DIR)/%.c | $(BUILD_DIR)
+	$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
+
+$(BUILD_DIR)/tsan_ts_runtime.o: $(CBM_DIR)/ts_runtime.c | $(BUILD_DIR)
+	$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
+
+$(BUILD_DIR)/tsan_lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)
+	$(CC) $(GRAMMAR_CFLAGS_TSAN) -c -o $@ $<
+
+$(BUILD_DIR)/tsan_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)
+	$(CXX) $(CXXFLAGS_TSAN) -w -I$(CBM_DIR)/vendored -c -o $@ $<
+
 # ── Full test binary ─────────────────────────────────────────────
 
 # mimalloc object files
 MIMALLOC_OBJ_TEST = $(BUILD_DIR)/mimalloc.o
+MIMALLOC_OBJ_TSAN = $(BUILD_DIR)/tsan_mimalloc.o
 MIMALLOC_OBJ_PROD = $(BUILD_DIR)/prod_mimalloc.o
 
 $(BUILD_DIR)/mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR)
 	$(CC) $(MIMALLOC_CFLAGS_TEST) -c -o $@ $<
 
+$(BUILD_DIR)/tsan_mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR)
+	$(CC) $(MIMALLOC_CFLAGS_TEST) $(TSAN_SANITIZE) -c -o $@ $<
+
 $(BUILD_DIR)/prod_mimalloc.o: $(MIMALLOC_SRC) | $(BUILD_DIR)
 	$(CC) $(MIMALLOC_CFLAGS) -c -o $@ $<
 
 # sqlite3 object files (vendored amalgamation)
 SQLITE3_OBJ_TEST = $(BUILD_DIR)/sqlite3.o
+SQLITE3_OBJ_TSAN = $(BUILD_DIR)/tsan_sqlite3.o
 SQLITE3_OBJ_PROD = $(BUILD_DIR)/prod_sqlite3.o
 
 $(BUILD_DIR)/sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
 	$(CC) $(SQLITE3_CFLAGS_TEST) $(SANITIZE) -c -o $@ $<
 
+$(BUILD_DIR)/tsan_sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
+	$(CC) $(SQLITE3_CFLAGS_TEST) $(TSAN_SANITIZE) -c -o $@ $<
+
 $(BUILD_DIR)/prod_sqlite3.o: $(SQLITE3_SRC) | $(BUILD_DIR)
 	$(CC) $(SQLITE3_CFLAGS) -c -o $@ $<
 
 # TRE regex (only compiled on Windows — POSIX uses system <regex.h>)
 TRE_OBJ_TEST :=
+TRE_OBJ_TSAN :=
 TRE_OBJ_PROD :=
 ifeq ($(IS_MINGW),yes)
 TRE_OBJ_TEST = $(BUILD_DIR)/tre.o
+TRE_OBJ_TSAN = $(BUILD_DIR)/tsan_tre.o
 TRE_OBJ_PROD = $(BUILD_DIR)/prod_tre.o
 
 $(BUILD_DIR)/tre.o: $(TRE_SRC) | $(BUILD_DIR)
 	$(CC) $(TRE_CFLAGS) $(SANITIZE) -fno-sanitize=alignment -c -o $@ $<
 
+$(BUILD_DIR)/tsan_tre.o: $(TRE_SRC) | $(BUILD_DIR)
+	$(CC) $(TRE_CFLAGS) $(TSAN_SANITIZE) -c -o $@ $<
+
 $(BUILD_DIR)/prod_tre.o: $(TRE_SRC) | $(BUILD_DIR)
 	$(CC) $(TRE_CFLAGS) -O2 -c -o $@ $<
 endif
 
 # Vendored LZ4 (test build)
 LZ4_OBJ_TEST = $(BUILD_DIR)/test_lz4.o $(BUILD_DIR)/test_lz4hc.o
+LZ4_OBJ_TSAN = $(BUILD_DIR)/tsan_lz4.o $(BUILD_DIR)/tsan_lz4hc.o
 $(BUILD_DIR)/test_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
 	$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR) -c -o $@ $<
 $(BUILD_DIR)/test_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
 	$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
+$(BUILD_DIR)/tsan_lz4.o: $(CBM_DIR)/vendored/lz4/lz4.c | $(BUILD_DIR)
+	$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR) -c -o $@ $<
+$(BUILD_DIR)/tsan_lz4hc.o: $(CBM_DIR)/vendored/lz4/lz4hc.c | $(BUILD_DIR)
+	$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR)/vendored/lz4 -c -o $@ $<
 
 # Vendored zstd (test build)
 ZSTD_OBJ_TEST = $(BUILD_DIR)/test_zstd.o
+ZSTD_OBJ_TSAN = $(BUILD_DIR)/tsan_zstd.o
 $(BUILD_DIR)/test_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
 	$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(SANITIZE) -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
+$(BUILD_DIR)/tsan_zstd.o: $(CBM_DIR)/vendored/zstd/zstd.c | $(BUILD_DIR)
+	$(CC) -std=c11 -D_DEFAULT_SOURCE -g -O1 $(TSAN_SANITIZE) -w -I$(CBM_DIR)/vendored/zstd -c -o $@ $<
 
 # nomic-embed-code pretrained vector blob
 UNIXCODER_OBJ = $(BUILD_DIR)/unixcoder_blob.o
 $(UNIXCODER_OBJ): $(UNIXCODER_BLOB_SRC) vendored/nomic/code_vectors.bin | $(BUILD_DIR)
 	$(CC) -c -o $@ $<
 
 OBJS_VENDORED_TEST = $(MIMALLOC_OBJ_TEST) $(SQLITE3_OBJ_TEST) $(TRE_OBJ_TEST) $(GRAMMAR_OBJS_TEST) $(TS_RUNTIME_OBJ_TEST) $(LSP_OBJ_TEST) $(PP_OBJ_TEST) $(LZ4_OBJ_TEST) $(ZSTD_OBJ_TEST) $(UNIXCODER_OBJ)
+OBJS_VENDORED_TSAN = $(MIMALLOC_OBJ_TSAN) $(SQLITE3_OBJ_TSAN) $(TRE_OBJ_TSAN) $(GRAMMAR_OBJS_TSAN) $(TS_RUNTIME_OBJ_TSAN) $(LSP_OBJ_TSAN) $(PP_OBJ_TSAN) $(LZ4_OBJ_TSAN) $(ZSTD_OBJ_TSAN) $(UNIXCODER_OBJ)
 
 $(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TEST) | $(BUILD_DIR)
 	$(CC) $(CFLAGS_TEST) -Itests -Itests/repro -o $@ \
@@ -601,8 +642,18 @@ test-repro: $(BUILD_DIR)/test-repro-runner
 
 # ── TSan full test ───────────────────────────────────────────────
 
-test-tsan:
-	@echo "TSan not yet wired for full extraction tests"
+TEST_TSAN_SUITES ?= mem slab_alloc parallel
+TSAN_OPTIONS ?= halt_on_error=1
+
+$(BUILD_DIR)/test-runner-tsan: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) $(OBJS_VENDORED_TSAN) | $(BUILD_DIR)
+	$(CC) $(CFLAGS_TSAN) -Itests -Itests/repro -o $@ \
+		$(ALL_TEST_SRCS) $(PROD_SRCS) \
+		$(EXTRACTION_SRCS) $(AC_LZ4_SRCS) $(ZSTD_SRCS) $(SQLITE_WRITER_SRC) \
+		$(OBJS_VENDORED_TSAN) \
+		$(LDFLAGS_TSAN)
+
+test-tsan: $(BUILD_DIR)/test-runner-tsan
+	cd $(CURDIR) && TSAN_OPTIONS="$(TSAN_OPTIONS)" $(BUILD_DIR)/test-runner-tsan $(TEST_TSAN_SUITES)
 
 # ── Production binary ────────────────────────────────────────────