Add Notion datasource sync foundation

.github/workflows/ci.yml.genie.ts

@@ -630,12 +630,18 @@ const extraJobs: Record<string, any> = {
       profile: 'namespace-profile-linux-x86-64',
       runId: '${{ github.run_id }}',
     }),
-    'timeout-minutes': jobTimeoutMinutes,
+    'timeout-minutes': 90,
     defaults: bashShellDefaults,
     env: {
       ...standardCIEnv,
       NOTION_API_TOKEN: '${{ secrets.NOTION_API_TOKEN }}',
       NOTION_TEST_PARENT_PAGE_ID: '${{ secrets.NOTION_TEST_PARENT_PAGE_ID }}',
+      NOTION_DATASOURCE_SYNC_PARENT_PAGE_ID:
+        '${{ secrets.NOTION_DATASOURCE_SYNC_PARENT_PAGE_ID || secrets.NOTION_TEST_PARENT_PAGE_ID }}',
+      NOTION_DATASOURCE_SYNC_E2E_LEDGER_PAGE_ID:
+        '${{ secrets.NOTION_DATASOURCE_SYNC_E2E_LEDGER_PAGE_ID }}',
+      NOTION_DATASOURCE_SYNC_DEMO_PAGE_ID:
+        "${{ github.event_name == 'workflow_dispatch' && (inputs.run_datasource_sync_demo == true || inputs.run_datasource_sync_demo == 'true') && secrets.NOTION_DATASOURCE_SYNC_DEMO_PAGE_ID || '' }}",
     },
     steps: [
       ...baseSteps,
@@ -689,6 +695,13 @@ export default ciWorkflow({
     workflow_dispatch: {
       inputs: {
         ...ciMeasurementBaselineWorkflowDispatchInputs,
+        run_datasource_sync_demo: {
+          description:
+            'Run the credentialed notion-datasource-sync demo showcase in the Notion integration lane. Requires NOTION_DATASOURCE_SYNC_DEMO_PAGE_ID.',
+          required: false,
+          default: false,
+          type: 'boolean',
+        },
         debug_force_nix_diagnostics_failure: {
           description:
             'Temporary debug switch (#272): force post-validation failure to verify diagnostics artifact + summary',

CHANGELOG.md

@@ -6,6 +6,20 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
+- **@overeng/notion-datasource-sync**: Add the initial standalone Notion datasource sync package with VRS and developer-facing docs, Effect/Schema contracts, self-contained SQLite event/outbox/projection core, user-facing `<database-id>.sqlite` replica/API contract for local reads and guarded write intents, pure planner/guard/conflict logic, fake and live Notion gateways, real filesystem workspace materialization, sync-first remote adoption from existing Notion data sources and single-child database URLs with bounded dry-run previews for large databases, first-class data-source metadata title/description sync, NotionMD-backed `.nmd` body observation/materialization/push boundaries, one-shot and watch CLI surfaces, user conflict/delete/restore commands, OpenTelemetry instrumentation, an automated live Notion demo spanning multiple realistic data-source schemas including a 500-row high-cardinality surface, and broad unit/integration/E2E coverage for pagination, high-watermarks, schema safety, metadata safety, body safety, conflicts, tombstones, daemon restart/cancellation/backpressure, live fixture ledgers, and intentional fail-closed unsupported surfaces.
+- **@overeng/notion-datasource-sync**: Promote explicit public SQLite row creation through the `changes` surface, with guarded create-page planning, local client request keys, returned `remote_page_id` settlement, effective read views for pending creates, direct private row-table insert blocking, fake/CLI E2E coverage, and disposable live Notion proof.
+- **@overeng/notion-datasource-sync**: Promote public SQLite data-source metadata CDC for title/description through `changes`, projecting canonical metadata values into `<database-id>.sqlite`, computing expected post-write metadata hashes, and verifying settlement through fake/CLI and disposable live Notion coverage while keeping separate database metadata, files, views, relation/people writes, and schema migrations fail-closed.
+- **@overeng/notion-datasource-sync**: Promote public SQLite database metadata CDC for title/description through `changes` with database authority, base-hash guards, post-write hash settlement, fake/CLI coverage, and disposable live Notion proof while keeping icon/cover/parent/trash/lock fail-closed.
+- **@overeng/notion-datasource-sync**: Promote public SQLite external URL file attachments through explicit `changes` staging for empty `files` properties, with dry-run non-settlement, fake/CLI coverage, and disposable live Notion proof while keeping local uploads, signed Notion URLs, replacement/deletion, and non-empty preservation fail-closed.
+- **@overeng/notion-datasource-sync**: Add read-only Notion UI view inventory diagnostics through `debug_*`, with fake and disposable live coverage for stable rerun hashes while keeping view write CDC and view-query membership semantics fail-closed.
+- **@overeng/notion-datasource-sync**: Make auto-discovered Notion schema the default `sync --from-notion`/observation contract for canonical `rows`, projecting live property descriptors and values into `schema_properties`, private/debug diagnostics, and property columns without user schema JSON, with focused fake/local and disposable live coverage plus updated VRS/docs/demo guidance.
+- **@overeng/notion-datasource-sync**: Promote guarded public SQLite relation removals, reorders, and additions when the relation base is fully paginated and every added target has already been observed through the same relation property, preserving existing related pages through replacement-shaped Notion writes with fake and disposable live E2E coverage while keeping unobserved relation targets and people writes fail-closed.
+- **@overeng/notion-datasource-sync**: Add explicit public SQLite lifecycle fields for local file uploads and fail-closed `changes` entries for Notion UI view writes so remaining unpromoted file-upload and view writes are modeled without planning unsafe remote mutations.
+- **@overeng/notion-datasource-sync**: Observe live data-source schema properties from the Notion gateway by default, record them as dedicated schema observation events, and populate scalar row cells from retrieved page property values without requiring CLI schema JSON injection.
+- **@overeng/notion-datasource-sync**: Document the clean-break 1:1 canonical `<database-id>.sqlite` replica contract, including the writable `rows` surface, `schema` / `schema_properties` inspection views, public `changes` / `conflicts` / `sync_status`, read-only `debug_*`, private `_nds_*`, direct row mutation examples, fail-closed delete/internal-edit behavior, SQLite backup/copy semantics, and the multi-data-source demo model where each source is its own database-ID-named replica artifact.
+- **@overeng/notion-datasource-sync**: Add clean-break self-contained SQLite E2E contract coverage for database-ID artifact naming, sidecar-free discovery, sqlite*master surface taxonomy, canonical `rows` mutation behavior, private `\_nds*\*` tamper fail-closed checks, and backup/copy portability.
+- **@overeng/notion-datasource-sync**: Make full Notion database replicas the only user-facing product behavior for `<database-id>.sqlite`, reject CLI query-contract establishment and path overrides, keep row limits as dry-run-only previews, and update storage-contract tests plus docs/VRS/demo guidance so filtered/query-contract scans remain internal debug/test concerns only.
+- **@overeng/notion-datasource-sync docs**: Clarify the product SQLite contract: `rows` is the primary writable API, `watch` must process local SQLite CDC, `changes` / `conflicts` / `sync_status` are public observability surfaces, `_nds_*` remains private state, and `<database-id>.sqlite` is always a full-database replica.
 - **@overeng/react-inspector**: Lineage annotation namespace (#687). New `Lineage` module with `SourceOfTruth | Derived | Projection | Cache | Mirror | External | Computed` tagged union, plus composable companion annotations (`Authority`, `Freshness`, `ForeignKey`). All annotations are self-describing Effect Schemas with ergonomic `pipe`-style constructors (`Lineage.derivedFrom`, `Lineage.cache`, `Lineage.authority`, etc.). The schema-aware renderer surfaces a small superscript glyph next to annotated field names and a dedicated `LINEAGE` / `AUTHORITY` / `FRESHNESS` / `REF` block in the schema tooltip. `SchemaInfo` gains an optional `lineage: LineageBundle` field. Source-field path references in `Derived.from` carry `data-lineage-target` attributes for future jump-to-source wiring. Round-trip-tested via vitest.
 - **@overeng/react-inspector**: Map/Set container labels (#686). `Schema.Map({key, value})` renders as `Map<K, V>(N)`, `Schema.Set(T)` as `Set<T>(N)`, plus the `Readonly*` variants. Detected via the `effect/annotation/TypeConstructor` annotation on `Declaration` ASTs.
 - **@overeng/react-inspector**: Runtime tagged-union narrowing (#686). When a field's declared schema is `Schema.Union(A, B, C)` of `_tag`-discriminated variants and the runtime value carries a matching `_tag`, the inspector narrows the display (name, tooltip, container label, nested field resolution) to the matched variant. Narrowing happens on every path segment, not just the leaf, so nested fields under a tagged union resolve through the matched variant. `SchemaProvider` gains a `rootData` prop and the context exposes a new `getContextForPathWithValue(path, value)` method.
@@ -56,6 +70,15 @@ All notable changes to this project will be documented in this file.
 
 ### Fixed
 
+- **@overeng/notion-datasource-sync**: Fail closed for direct public SQLite `people` and `files` current-state cell edits before visible replica mutation or CDC append; these surfaces must use deterministic user-identity pagination or explicit file staging/upload lifecycle before promotion.
+- **@overeng/notion-datasource-sync**: Surface ambiguous public SQLite row-create outcomes as `needs_reconciliation` instead of leaving them as planned, with E2E coverage for the expired-running create crash window and no duplicate `createPage` retry.
+- **@overeng/notion-datasource-sync**: Make live remote adoption reruns idempotent for unchanged capability/property observations, settle public SQLite page-property CDC writes against live Notion, and use lifecycle hashes for row archive/restore CDC so direct `<database-id>.sqlite` row edits do not replay stale archive intents.
+- **@overeng/notion-datasource-sync**: Move public `<database-id>.sqlite` write capture to `changes` for cell and row changes while preserving guarded direct SQLite edits.
+- **@overeng/notion-datasource-sync**: Extend the public SQLite CDC surface with typed body, metadata, schema, and conflict-resolution tables; promote row restore, guarded body pushes, and store-backed safe conflict-resolution choices into the guarded planner/outbox path; settle public CDC status from planner/outbox outcomes including the CLI `sync <workspace>` wiring path; validate body content hashes before planning remote writes; and keep public metadata/schema CDC, database metadata, files, views, and destructive schema migrations fail-closed until verified post-write reconciliation is modeled.
+- **@overeng/notion-datasource-sync**: Harden `changes` correctness so repeated direct SQLite edits coalesce to the latest effective desired state, row lifecycle toggles cannot replay stale archive intents, queued changes remain visible to later scans, and invalid direct cell payloads fail before visible replica mutation.
+- **@overeng/notion-datasource-sync**: Keep the public `<database-id>.sqlite` replica coherent after direct writable cell edits, reject computed/system cell mutations before visible state changes, use a weaker queued state instead of premature planned status for validated local changes, fail closed for invalid local changes, and align docs with the shipped replica schema and intent names.
+- **@overeng/notion-cli**: Gate `db dump` live fixture tests on writable fixture configuration and provision shared Notion integration fixtures before reading `TEST_IDS`, avoiding accidental live API calls with empty IDs when only `NOTION_API_TOKEN` is present.
+- **nix/oxc-config-plugin**: Refresh the pnpm dependency fixed-output hash so the devenv shell can realize the oxlint package used by `check:all`.
 - **@overeng/tui-react**: Let Effect CLI own Ctrl-C entrypoint handling for `run(App, handler)`. Apps whose action schema includes `Interrupted` now dispatch it during normal Effect interruption finalization, map interrupt-only CLI exits to code 130, and suppress noisy interrupt-only error output in `runTuiMain`.
 - **@overeng/megarepo**: Stop store repository discovery from walking internal scratch roots like `tmp` before scanning repos, yield during repository discovery so Effect interruption can propagate promptly, and tighten CLI OTel flush timing so interrupted TTY commands return quickly while still exporting traces.
 - **@overeng/megarepo**: Improve `mr store gc --dry-run --output tty` progress UX with early phase updates, heartbeat refreshes, realtime worktree discovery/active-check counts, explicit interrupted output, exit code 130 for Ctrl-C, and more granular OTel spans for removal status checks. GC removal checks now use a single `git status --untracked-files=normal` dirty preflight before the upstream check, avoiding expensive recursive untracked-file enumeration while still failing closed for dirty worktrees.
@@ -84,6 +107,7 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
+- **@overeng/notion-effect-client / @overeng/notion-md**: Share canonical `.notion-md` object-store paths, sync-state paths, object refs, and storage-size thresholds from the NMD schema layer so local metadata decisions derive from one source of truth.
 - **@overeng/notion-md**: Breaking CLI simplification: collapse the user-facing page workflow around `sync` and `status`; replace the old explicit `pull` / `push` entrypoints with `sync <page-id-or-url> <file.nmd>` for bootstrap and guarded `sync <file.nmd>` for reconciliation.
 - **@overeng/notion-md**: Remove legacy compatibility paths for batch `push` and local-first `page_id: null` page creation; existing Notion pages must be materialized with `sync <page-id-or-url> <target>`.
 - **@overeng/pty-effect/client**: `spawnDaemon` now delegates to `@myobie/pty.spawnDaemon` instead of duplicating the daemon spawn pipeline. The Bun-on-Node case is routed through upstream's new `launcher` option (still honors `NODE_BIN`). Eliminates a divergent in-house spawn path so consumers automatically inherit upstream improvements such as bundle-safe spawn (myobie/pty#38). Public API and `PtyDaemonSpec` schema unchanged.

devenv.nix

@@ -110,6 +110,7 @@ let
     "packages/@overeng/kdl-effect"
     "packages/@overeng/megarepo"
     "packages/@overeng/notion-cli"
+    "packages/@overeng/notion-datasource-sync"
     "packages/@overeng/notion-effect-client"
     "packages/@overeng/notion-effect-schema"
     "packages/@overeng/notion-md"
@@ -159,6 +160,10 @@ let
       path = "packages/@overeng/notion-cli";
       name = "notion-cli";
     }
+    {
+      path = "packages/@overeng/notion-datasource-sync";
+      name = "notion-datasource-sync";
+    }
     {
       path = "packages/@overeng/notion-effect-client";
       name = "notion-effect-client";

genie/ci-workflow/merge-queue.ts

@@ -190,7 +190,7 @@ export const mergeQueueAdmittedJob = ({
   ...jobOptions
 }: MergeQueueAdmittedJobOptions): WorkflowJob => ({
   if: nonScheduleRequiredGateIf,
-  'runs-on': Array.isArray(runsOn) ? [...runsOn] : runsOn,
+  'runs-on': Array.isArray(runsOn) === true ? [...runsOn] : runsOn,
   permissions: mergeRequiredAdmissionPermissions(permissions),
   steps: [
     mergeQueueAdmissionStep({
@@ -216,7 +216,7 @@ export const mergeQueueAdmissionGateJob = ({
 }: MergeQueueAdmissionGateJobOptions = {}): WorkflowJob => ({
   name: 'mq/admission',
   if: nonScheduleRequiredGateIf,
-  'runs-on': Array.isArray(runsOn) ? [...runsOn] : runsOn,
+  'runs-on': Array.isArray(runsOn) === true ? [...runsOn] : runsOn,
   'timeout-minutes': timeoutMinutes,
   permissions: defaultMergeQueuePermissions,
   steps: [mergeQueueAdmissionStep(stepOptions)],
@@ -251,7 +251,7 @@ export const mergeQueueSemanticGateJob = ({
 }: MergeQueueSemanticGateJobOptions): WorkflowJob => ({
   name: requiredGateCheckName(name),
   if: nonScheduleRequiredGateIf,
-  'runs-on': Array.isArray(runsOn) ? [...runsOn] : runsOn,
+  'runs-on': Array.isArray(runsOn) === true ? [...runsOn] : runsOn,
   'timeout-minutes': timeoutMinutes,
   permissions: defaultMergeQueuePermissions,
   needs: [...needs],

genie/ci-workflow/setup.ts

@@ -676,7 +676,7 @@ export const standardSelfHostedDevenvTaskJob = ({
   step,
   ...jobOptions
 }: StandardSelfHostedDevenvTaskJobOptions): WorkflowJob => ({
-  'runs-on': Array.isArray(runsOn) ? [...runsOn] : runsOn,
+  'runs-on': Array.isArray(runsOn) === true ? [...runsOn] : runsOn,
   defaults,
   env,
   steps: [

genie/ci-workflow/shared.ts

@@ -208,7 +208,7 @@ export const cachixHostsFromBinaryCaches = (caches: readonly NixBinaryCache[]) =
   ...new Set(
     caches.flatMap((cache) => {
       const host = new URL(cache.uri).host
-      return host.endsWith('.cachix.org') ? [host] : []
+      return host.endsWith('.cachix.org') === true ? [host] : []
     }),
   ),
 ]