Bug-Squash/'Works-on-my-machine' - 4 PRDs in one PR :( 👎

.github/workflows/ci.yml

@@ -1,19 +1,24 @@
 name: CI
 
 on:
+  # Run on direct pushes to protected branches only
   push:
     branches:
       - main
-      - dev
+  # Run CI for PRs targeting these branches
   pull_request:
+    branches:
+      - main
+      - dev
   workflow_dispatch:
 
 permissions:
   contents: read
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+  # Ensure push and PR for the same commit share one slot
+  group: ci-${{ github.event.pull_request.head.sha || github.sha }}
+  cancel-in-progress: true
 
 jobs:
   lint:
@@ -83,6 +88,8 @@ jobs:
       TEST_DB_HOST: 127.0.0.1
       TEST_DB_USER: tldw
       TEST_DB_PASSWORD: tldw
+      # Align AuthNZ_Postgres conftest default DB name with tests
+      TEST_DB_NAME: tldw_test
       TLDW_TEST_POSTGRES_REQUIRED: '1'
     steps:
       - name: Checkout
@@ -115,6 +122,18 @@ jobs:
         run: |
           echo "POSTGRES_TEST_PORT=${{ job.services.postgres.ports[5432] }}" >> "$GITHUB_ENV"
           echo "TEST_DB_PORT=${{ job.services.postgres.ports[5432] }}" >> "$GITHUB_ENV"
+          # Provide a unified DSN so any test preferring TEST_DATABASE_URL uses the right DB
+          echo "TEST_DATABASE_URL=postgresql://tldw:[email protected]:${{ job.services.postgres.ports[5432] }}/tldw_content" >> "$GITHUB_ENV"
+          echo "DATABASE_URL=postgresql://tldw:[email protected]:${{ job.services.postgres.ports[5432] }}/tldw_content" >> "$GITHUB_ENV"
+
+      - name: Ensure base DB exists
+        shell: bash
+        env:
+          PGPASSWORD: tldw
+        run: |
+          PORT="${{ job.services.postgres.ports[5432] }}"
+          psql -h 127.0.0.1 -p "$PORT" -U tldw -d postgres -tc "SELECT 1 FROM pg_database WHERE datname='tldw_content'" | grep -q 1 || \
+            psql -h 127.0.0.1 -p "$PORT" -U tldw -d postgres -c "CREATE DATABASE tldw_content"
 
       - name: Install additional deps for PG tests
         run: |

.github/workflows/codeql.yml

@@ -1,12 +1,20 @@
 name: CodeQL
 
 on:
+  # Run on direct pushes to protected/default branches only
   push:
-    branches: [ main, master, dev ]
+    branches: [ main, master ]
+  # Run on PRs targeting main/master/dev, not every branch
   pull_request:
-    branches: [ '**' ]
+    branches: [ main, master, dev ]
   schedule:
     - cron: '0 6 * * 1'
+  # Allow manual runs
+  workflow_dispatch:
+
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   analyze:

.github/workflows/jobs-suite.yml

@@ -104,6 +104,7 @@ jobs:
       POSTGRES_TEST_DB: tldw_content
       POSTGRES_TEST_USER: tldw
       POSTGRES_TEST_PASSWORD: tldw
+      TEST_DB_NAME: tldw_test
       TLDW_TEST_POSTGRES_REQUIRED: '1'
     steps:
       - name: Checkout
@@ -131,6 +132,14 @@ jobs:
           echo "POSTGRES_TEST_PORT=${{ job.services.postgres.ports[5432] }}" >> "$GITHUB_ENV"
           echo "TEST_DB_PORT=${{ job.services.postgres.ports[5432] }}" >> "$GITHUB_ENV"
 
+      - name: Ensure base DB exists
+        env:
+          PGPASSWORD: tldw
+        run: |
+          PORT="${{ job.services.postgres.ports[5432] }}"
+          psql -h 127.0.0.1 -p "$PORT" -U tldw -d postgres -tc "SELECT 1 FROM pg_database WHERE datname='tldw_content'" | grep -q 1 || \
+            psql -h 127.0.0.1 -p "$PORT" -U tldw -d postgres -c "CREATE DATABASE tldw_content"
+
       - name: Install pytest and psycopg
         run: |
           python -m pip install --upgrade pip

.github/workflows/sbom.yml

@@ -21,40 +21,102 @@ jobs:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
 
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Generate Python SBOM (CycloneDX)
+        run: |
+          set -euo pipefail
+          python -m pip install -q cyclonedx-python
+          GEN_CMD="python -m cyclonedx_py"
+          if [ -n "${pythonLocation:-}" ] && [ -x "${pythonLocation}/bin/cyclonedx-py" ]; then \
+            GEN_CMD="${pythonLocation}/bin/cyclonedx-py"; \
+          elif command -v cyclonedx-py >/dev/null 2>&1; then \
+            GEN_CMD="cyclonedx-py"; \
+          fi
+          gen_from_requirements() { \
+            local req="$1"; \
+            echo "Generating Python SBOM from ${req} using ${GEN_CMD}"; \
+            ${GEN_CMD} -r "${req}" --output-file sbom-python.cdx.json || { \
+              echo "Primary generator failed; attempting cyclonedx-bom fallback"; \
+              python -m pip install -q cyclonedx-bom || true; \
+              if command -v cyclonedx-bom >/dev/null 2>&1; then \
+                cyclonedx-bom -r "${req}" -o sbom-python.cdx.json; \
+              else \
+                python -m cyclonedx_bom -r "${req}" -o sbom-python.cdx.json; \
+              fi; \
+            }; \
+          }
+          if [ -f tldw_Server_API/requirements.txt ]; then \
+            gen_from_requirements tldw_Server_API/requirements.txt; \
+          elif [ -f requirements.txt ]; then \
+            gen_from_requirements requirements.txt; \
+          else \
+            echo "No requirements file found; cannot generate SBOM"; \
+            exit 1; \
+          fi
 
-      # No registry login required for public Docker Hub images used below
       - name: Setup Node
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v4
         with:
           node-version: '20'
 
-      - name: Generate Python SBOM from pyproject (cdxgen)
+      - name: Generate Node SBOM (CycloneDX NPM)
         run: |
-          if [ -f pyproject.toml ]; then \
-            npx @appthreat/cdxgen -t python -o sbom-python.cdx.json; \
+          if [ -f package-lock.json ]; then \
+            npx -y @cyclonedx/cyclonedx-npm --output-file sbom-node.cdx.json; \
+          elif [ -f tldw-frontend/package-lock.json ]; then \
+            (cd tldw-frontend && npx -y @cyclonedx/cyclonedx-npm --output-file ../sbom-node.cdx.json); \
+          else \
+            echo "No package-lock.json found; skipping Node SBOM"; \
+          fi
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Resolve CycloneDX CLI digest
+        run: |
+          set -euo pipefail
+          ref="ghcr.io/cyclonedx/cyclonedx-cli:0.30.0"
+          echo "Resolving digest for ${ref}"
+          # Prefer buildx imagetools; fallback to manifest inspect if needed
+          if docker buildx imagetools inspect "$ref" >/dev/null 2>&1; then \
+            digest=$(docker buildx imagetools inspect "$ref" | awk '/^Digest:/ {print $2; exit}'); \
+          else \
+            digest=$(docker manifest inspect "$ref" | jq -r '.manifests[0].digest' || true); \
+          fi
+          if [ -z "${digest:-}" ] || ! echo "$digest" | grep -Eq '^sha256:[0-9a-f]{64}$'; then \
+            echo "Failed to resolve digest for $ref"; \
+            exit 1; \
           fi
+          echo "CDX_CLI_DIGEST=$digest" >> "$GITHUB_ENV"
+          echo "Resolved digest: $digest"
 
-      - name: Generate SBOM (CycloneDX JSON)
-        uses: anchore/sbom-action@8e94d75ddd33f69f691467e42275782e4bfefe84 # v0.20.9
+      - name: Merge SBOMs (CycloneDX CLI)
+        run: |
+          if [ -f sbom-python.cdx.json ] && [ -f sbom-node.cdx.json ]; then \
+            docker run --rm -v "$PWD":/work -w /work ghcr.io/cyclonedx/cyclonedx-cli@${CDX_CLI_DIGEST} \
+              merge --input-files sbom-python.cdx.json sbom-node.cdx.json --output-file sbom.cdx.json; \
+          elif [ -f sbom-python.cdx.json ]; then \
+            cp sbom-python.cdx.json sbom.cdx.json; \
+          elif [ -f sbom-node.cdx.json ]; then \
+            cp sbom-node.cdx.json sbom.cdx.json; \
+          else \
+            echo "No SBOMs generated"; \
+            exit 1; \
+          fi
+
+      - name: Upload SBOM artifact
+        uses: actions/upload-artifact@v4
         with:
-          path: .
-          format: cyclonedx-json
-          output-file: sbom.cdx.json
-          upload-artifact: true
-          artifact-name: sbom-cyclonedx
-
-      - name: Validate SBOM (CycloneDX CLI - pinned)
-        id: validate_cli_pinned
+          name: sbom-cyclonedx
+          path: sbom.cdx.json
+
+      - name: Validate SBOM (CycloneDX CLI - pinned digest)
         if: ${{ hashFiles('sbom.cdx.json') != '' }}
         continue-on-error: true
-        uses: docker://cyclonedx/cyclonedx-cli:0.30.0
-        with:
-          args: >-
-            validate --input-file sbom.cdx.json
-
-      - name: Validate SBOM (CycloneDX CLI - pinned fallback)
-        if: ${{ hashFiles('sbom.cdx.json') != '' && steps.validate_cli_pinned.outcome == 'failure' }}
-        uses: docker://cyclonedx/cyclonedx-cli:0.29.1
-        with:
-          args: >-
+        run: |
+          docker run --rm -v "$PWD":/work -w /work ghcr.io/cyclonedx/cyclonedx-cli@${CDX_CLI_DIGEST} \
             validate --input-file sbom.cdx.json

Docs/Design/IMPLEMENTATION_PLAN.md

@@ -0,0 +1,96 @@
+## Implementation Plan — Browser Extension
+
+This document tracks staged implementation with concrete success criteria and test notes.
+
+---
+
+## Stage 1: Connectivity & Auth
+**Goal**: Establish server connectivity and both auth modes (API Key and JWT).
+
+**Success Criteria**:
+- Options page captures server URL and credentials; health check returns OK.
+- Background proxy injects headers; tokens never exposed to content scripts.
+- 401 triggers single‑flight refresh and one retry; no duplicate requests.
+
+**Tests**:
+- Unit: auth storage, header injection, refresh queue.
+- Integration: health endpoint, login/logout, API key validation.
+- Manual: revoke permission and re‑grant host permission flow.
+
+**Status**: Not Started
+
+---
+
+## Stage 2: Chat & Models
+**Goal**: Streaming chat via `/api/v1/chat/completions` with model selection.
+
+**Success Criteria**:
+- Models/providers fetched and rendered; selection persisted per session.
+- Non‑stream and SSE stream both work; cancel stops network within ~200ms.
+- Exact path strings (no 307 redirects observed in logs).
+
+**Tests**:
+- Unit: SSE parser, backoff, abort controller.
+- Integration: stream across two models; cancel and resume.
+- Manual: slow network simulation; ensure UI stays responsive.
+
+**Status**: Not Started
+
+---
+
+## Stage 3: RAG & Media
+**Goal**: RAG search UI and URL ingest with progress notifications.
+
+**Success Criteria**:
+- RAG `/api/v1/rag/search` returns results; snippets insert into chat context.
+- URL ingest calls `/api/v1/media/process`; user sees progress and final status.
+- Errors are actionable (permission, size limits, server busy).
+
+**Tests**:
+- Unit: request builders, snippet insertion.
+- Integration: RAG queries; media process happy path and failure modes.
+- Manual: ingest current tab URL; verify server reflects new media.
+
+**Status**: Not Started
+
+---
+
+## Stage 4: Notes/Prompts & STT
+**Goal**: Notes/Prompts basic flows and STT upload/transcribe.
+
+**Success Criteria**:
+- Notes: create/search; export works; selection‑to‑note from content script.
+- Prompts: browse/import/export; insert chosen prompt into chat input.
+- STT: upload short clip; transcript displayed; non‑supported formats fail clearly.
+
+**Tests**:
+- Unit: notes/prompts stores; MIME/type validation.
+- Integration: `/api/v1/notes/*`, `/api/v1/prompts/*`, `/api/v1/audio/transcriptions`.
+- Manual: 20s audio clip round‑trip; error message clarity for oversized files.
+
+**Status**: Not Started
+
+---
+
+## Stage 5: TTS & Polish
+**Goal**: TTS synthesis/playback and UX polish.
+
+**Success Criteria**:
+- Voices list loads from `/api/v1/audio/voices/catalog`; selection persisted.
+- `/api/v1/audio/speech` returns audio; playback controls functional.
+- Accessibility audit passes key checks; performance within budgets.
+
+**Tests**:
+- Unit: audio player controls and error states.
+- Integration: voices catalog and synthesis endpoints.
+- Manual: latency spot checks; keyboard navigation.
+
+**Status**: Not Started
+
+---
+
+## Notes
+- Centralize route constants and validate against OpenAPI at startup (warn on mismatch).
+- Keep tokens in background memory; only persist refresh tokens if strictly necessary.
+- Use optional host permissions for user‑configured origins (Chrome/Edge MV3).
+