From a011186ccc779215dfc9a0948df6f41201e9e1f8 Mon Sep 17 00:00:00 2001 From: xNet Test Date: Tue, 6 Oct 2026 10:11:29 -0700 Subject: [PATCH 1/4] feat(library): share spatial snapshots and tested controller flight - Reuse graph filtering, search and deterministic layout outside Electron preload. - Keep positions paired with snapshot revisions and resource identities. - Add explicit flight arming, analog acceleration, brakes and interruption rearming. - Verify frame-rate integration, invalid inputs and full-fixture searchability. Signed-off-by: xNet Test --- .../components/library-graph/model.ts | 137 +----------------- .../components/library-graph/navigation.ts | 72 +-------- .../src/shared/library-flight.test.ts | 89 ++++++++++++ apps/electron/src/shared/library-flight.ts | 95 ++++++++++++ .../src/shared/library-graph/model.ts | 136 +++++++++++++++++ .../src/shared/library-graph/navigation.ts | 71 +++++++++ .../src/shared/spatial-library.test.ts | 49 +++++++ apps/electron/src/shared/spatial-library.ts | 103 +++++++++++++ apps/electron/src/spatial/fixtures.ts | 57 ++++++++ ...68_[_]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md | 6 +- 10 files changed, 605 insertions(+), 210 deletions(-) create mode 100644 apps/electron/src/shared/library-flight.test.ts create mode 100644 apps/electron/src/shared/library-flight.ts create mode 100644 apps/electron/src/shared/library-graph/model.ts create mode 100644 apps/electron/src/shared/library-graph/navigation.ts create mode 100644 apps/electron/src/shared/spatial-library.test.ts create mode 100644 apps/electron/src/shared/spatial-library.ts create mode 100644 apps/electron/src/spatial/fixtures.ts diff --git a/apps/electron/src/renderer/components/library-graph/model.ts b/apps/electron/src/renderer/components/library-graph/model.ts index a822f43f1..0a0aca0d9 100644 --- a/apps/electron/src/renderer/components/library-graph/model.ts +++ b/apps/electron/src/renderer/components/library-graph/model.ts @@ -1,136 +1 @@ -import type { LibraryGraph, LibraryGraphRelation } from '../../../shared/library-graph' - -export const relationKinds: LibraryGraphRelation[] = ['collection', 'tag', 'category', 'creator'] -export const graphColors: Record = { - youtube: '#fb7185', - instagram: '#e879f9', - github: '#c4b5fd', - x: '#7dd3fc', - tiktok: '#2dd4bf', - reddit: '#fb923c', - generic: '#a3e635', - openai: '#6ee7b7', - claude: '#fcd34d', - grok: '#94a3b8', - collection: '#fbbf24', - tag: '#a78bfa', - category: '#67e8f9', - creator: '#fda4af' -} -export const graphColor = (kind: string, platform: string) => - graphColors[kind === 'link' ? platform : kind] ?? '#cbd5e1' - -export function graphAdjacency(graph: LibraryGraph): number[][] { - const adjacency = graph.nodes.map(() => [] as number[]) - graph.edges.forEach(({ source, target }) => { - adjacency[source].push(target) - adjacency[target].push(source) - }) - return adjacency -} - -export function selectGraph( - graph: LibraryGraph, - platform: string, - kinds: LibraryGraphRelation[], - focus: string | null, - groups: string[] = [], - match: 'any' | 'all' = 'any' -): LibraryGraph { - const accepted = new Set(kinds) - const selected = new Set(groups) - const memberships = new Map>() - if (selected.size) { - // Group membership filters do not depend on which relationship lines are shown. - graph.edges.forEach(({ source, target }) => { - const group = graph.nodes[target].id - if (!selected.has(group)) return - const values = memberships.get(source) ?? new Set() - values.add(group) - memberships.set(source, values) - }) - } - const included = new Set( - graph.nodes.flatMap((node, index) => - node.kind === 'link' && - (!platform || node.platform === platform) && - (!selected.size || - (match === 'all' ? memberships.get(index)?.size === selected.size : memberships.has(index))) - ? [index] - : [] - ) - ) - const edges = graph.edges.filter((edge) => accepted.has(edge.kind) && included.has(edge.source)) - let scope: Set | null = null - if (focus) { - const center = graph.nodes.findIndex((node) => node.id === focus) - scope = new Set([center]) - // Two hops from a link includes other links with the same known relationship. - for (let hop = 0; hop < (graph.nodes[center]?.kind === 'link' ? 2 : 1); hop++) { - const previous = new Set(scope) - edges.forEach((edge) => { - if (previous.has(edge.source)) scope!.add(edge.target) - if (previous.has(edge.target)) scope!.add(edge.source) - }) - } - } - const chosenEdges = edges.filter( - (edge) => !scope || (scope.has(edge.source) && scope.has(edge.target)) - ) - const hubs = new Set(chosenEdges.map((edge) => edge.target)) - const indices = new Map() - const nodes = graph.nodes.filter((node, i) => { - const visible = node.kind === 'link' ? included.has(i) && (!scope || scope.has(i)) : hubs.has(i) - if (visible) indices.set(i, indices.size) - return visible - }) - return { - ...graph, - nodes, - linkCount: nodes.filter((node) => node.kind === 'link').length, - edges: chosenEdges - .filter((edge) => indices.has(edge.source) && indices.has(edge.target)) - .map((edge) => ({ - ...edge, - source: indices.get(edge.source)!, - target: indices.get(edge.target)! - })) - } -} - -export function initialPositions(graph: Pick): Float32Array { - const result = new Float32Array(graph.nodes.length * 3) - const hubs = graph.nodes.flatMap((node, i) => (node.kind !== 'link' ? [i] : [])) - const radius = Math.max(80, Math.cbrt(graph.nodes.length) * 28) - graph.nodes.forEach((node, i) => { - let hash = 2166136261 - for (const character of node.id) hash = Math.imul(hash ^ character.charCodeAt(0), 16777619) - for (let axis = 0; axis < 3; axis++) { - hash = Math.imul(hash ^ (hash >>> 16), 2246822507) - result[i * 3 + axis] = ((hash >>> 0) / 4294967296 - 0.5) * radius * 2 - } - const length = Math.hypot(result[i * 3], result[i * 3 + 1], result[i * 3 + 2]) || 1 - const scale = (radius * Math.cbrt((hash >>> 0) / 4294967296)) / length - for (let axis = 0; axis < 3; axis++) result[i * 3 + axis] *= scale - }) - hubs.forEach((index, i) => { - const y = 1 - (2 * (i + 0.5)) / hubs.length - const ring = Math.sqrt(1 - y * y) - result.set( - [Math.cos(i * 2.399963) * ring * radius, y * radius, Math.sin(i * 2.399963) * ring * radius], - index * 3 - ) - }) - const weights = new Float32Array(graph.nodes.length) - const centers = new Float32Array(result.length) - graph.edges.forEach(({ source, target }) => { - weights[source]++ - for (let axis = 0; axis < 3; axis++) centers[source * 3 + axis] += result[target * 3 + axis] - }) - graph.nodes.forEach((_node, i) => { - if (!weights[i]) return - for (let axis = 0; axis < 3; axis++) - result[i * 3 + axis] = centers[i * 3 + axis] / weights[i] + result[i * 3 + axis] * 0.12 - }) - return result -} +export * from '../../../shared/library-graph/model' diff --git a/apps/electron/src/renderer/components/library-graph/navigation.ts b/apps/electron/src/renderer/components/library-graph/navigation.ts index d9c62ac4a..557110056 100644 --- a/apps/electron/src/renderer/components/library-graph/navigation.ts +++ b/apps/electron/src/renderer/components/library-graph/navigation.ts @@ -1,71 +1 @@ -import type { LibraryGraph, LibraryGraphNode } from '../../../shared/library-graph' - -export const groupNames = { - collection: 'Playlists & collections', - tag: 'Tags & topics', - category: 'Categories', - creator: 'Creator names' -} - -export const normalizeSearch = (value: string) => - value.normalize('NFKD').replace(/\p{M}/gu, '').toLocaleLowerCase().trim() - -export type GraphGroup = { node: LibraryGraphNode; count: number } - -export function graphGroups(graph: LibraryGraph, platform: string): GraphGroup[] { - const members = new Map>() - graph.edges.forEach(({ source, target }) => { - if (platform && graph.nodes[source].platform !== platform) return - const links = members.get(target) ?? new Set() - links.add(source) - members.set(target, links) - }) - return [...members] - .map(([index, links]) => ({ node: graph.nodes[index], count: links.size })) - .sort((a, b) => b.count - a.count || a.node.label.localeCompare(b.node.label)) -} - -export function graphSearchIndex(graph: LibraryGraph) { - const degrees = new Uint32Array(graph.nodes.length) - graph.edges.forEach(({ source, target }) => { - degrees[source]++ - degrees[target]++ - }) - return graph.nodes.map((node, index) => ({ - node, - label: normalizeSearch(node.label), - url: normalizeSearch(node.url ?? ''), - count: degrees[index] - })) -} - -export function searchGraph(index: ReturnType, query: string, limit = 12) { - const text = normalizeSearch(query) - const terms = text.split(/\s+/u) - const matches = index.flatMap((entry) => { - if (!text) return entry.node.kind === 'link' ? [] : [{ ...entry, rank: 0 }] - if (!terms.every((term) => entry.label.includes(term) || entry.url.includes(term))) return [] - const rank = - entry.label === text - ? 0 - : entry.label.startsWith(text) - ? 1 - : terms.every((term) => - entry.label.split(/[^\p{L}\p{N}]+/u).some((word) => word.startsWith(term)) - ) - ? 2 - : terms.every((term) => entry.label.includes(term)) - ? 3 - : 4 - return [{ ...entry, rank }] - }) - matches.sort( - (a, b) => - a.rank - b.rank || - Number(a.node.kind === 'link') - Number(b.node.kind === 'link') || - (a.node.kind === 'link' ? 0 : b.count - a.count) || - a.label.localeCompare(b.label) || - a.node.id.localeCompare(b.node.id) - ) - return { total: matches.length, items: matches.slice(0, limit) } -} +export * from '../../../shared/library-graph/navigation' diff --git a/apps/electron/src/shared/library-flight.test.ts b/apps/electron/src/shared/library-flight.test.ts new file mode 100644 index 000000000..16e300710 --- /dev/null +++ b/apps/electron/src/shared/library-flight.test.ts @@ -0,0 +1,89 @@ +import type { FlightInput, FlightState } from './library-flight' +import { describe, expect, it } from 'vitest' +import { + brakeFlight, + flightLimits, + initialFlight, + stepFlight, + travelBack, + travelTo +} from './library-flight' +const input: FlightInput = { + tracked: true, + visible: true, + trigger: 0, + direction: [0, 0, -1], + arm: true +} +const armed = () => stepFlight(initialFlight(), input, 0) +const run = (hz: number, trigger = 1) => { + let state = armed() + for (let i = 0; i < hz * 5; i++) + state = stepFlight(state, { ...input, arm: false, trigger }, 1 / hz) + return state +} +describe('controller flight', () => { + it('integrates equivalent elapsed time across frame rates', () => { + const reference = run(120) + for (const hz of [30, 60, 90, 144]) { + const actual = run(hz) + expect(actual.position[2]).toBeCloseTo(reference.position[2], 3) + expect(actual.velocity[2]).toBeCloseTo(reference.velocity[2], 3) + } + }) + it('requires explicit arming and a released trigger, never resumes on tracking return', () => { + let state = stepFlight(initialFlight(), { ...input, trigger: 1 }, 0.016) + expect(state.position).toEqual([0, 0, 0]) + state = stepFlight(state, { ...input, arm: false, trigger: 0 }, 0.016) + state = stepFlight(state, { ...input, arm: false, trigger: 1 }, 0.016) + expect(state.position[2]).toBeLessThan(0) + state = stepFlight(state, { ...input, tracked: false }, 0.016) + for (const trigger of [1, 0, 1]) + state = stepFlight(state, { ...input, arm: false, trigger }, 0.016) + expect(state.mode).toBe('stopped') + expect(state.velocity).toEqual([0, 0, 0]) + }) + it('scales acceleration with pressure, applies release drag and caps speed', () => { + expect(Math.abs(run(90, 0.5).velocity[2])).toBeLessThan(Math.abs(run(90).velocity[2])) + let state = { ...armed(), velocity: [0, 0, -20] } as FlightState + state = stepFlight(state, { ...input, arm: false, trigger: 1 }, 0.016) + expect(Math.hypot(...state.velocity)).toBeLessThanOrEqual(flightLimits.speed) + const next = stepFlight(state, { ...input, arm: false }, 0.016) + expect(Math.hypot(...next.velocity)).toBeLessThan(Math.hypot(...state.velocity)) + }) + it.each([ + { visible: false }, + { tracked: false }, + { trigger: NaN }, + { direction: [Infinity, 0, 0] as const }, + { direction: [0, 0, 0] as const } + ])('immediately stops invalid input %j', (patch) => { + const result = stepFlight(run(90), { ...input, arm: false, ...patch }, 0.016) + expect(result.mode).toBe('stopped') + expect(result.velocity).toEqual([0, 0, 0]) + }) + it.each([NaN, Infinity, -1, 0.2, 60])('stops after invalid or suspended time %s', (dt) => { + const state = run(60) + const next = stepFlight(state, input, dt) + expect(next.position).toEqual(state.position) + expect(next.mode).toBe('stopped') + expect(next.velocity).toEqual([0, 0, 0]) + }) + it('brakes without changing the viewpoint and does not take a head pose', () => { + const state = run(60) + expect(brakeFlight(state)).toEqual({ + ...state, + mode: 'inspect', + released: false, + velocity: [0, 0, 0] + }) + expect(stepFlight(state, { ...input, trigger: 1, brake: true }, 0.016)).toEqual( + brakeFlight(state) + ) + }) + it('keeps explicit travel reversible with a bounded trail', () => { + const state = { revision: 'r1', position: [0, 0, 0] as const, trail: [] } + expect(travelBack(travelTo(state, [1, 2, 3])).position).toEqual(state.position) + expect(() => travelTo(state, [NaN, 0, 0])).toThrow() + }) +}) diff --git a/apps/electron/src/shared/library-flight.ts b/apps/electron/src/shared/library-flight.ts new file mode 100644 index 000000000..007903429 --- /dev/null +++ b/apps/electron/src/shared/library-flight.ts @@ -0,0 +1,95 @@ +export type Vec3 = readonly [number, number, number] +export type FlightState = { + mode: 'inspect' | 'armed' | 'stopped' + position: Vec3 + velocity: Vec3 + released: boolean +} +export type FlightInput = { + tracked: boolean + visible: boolean + trigger: number + /** World-space unit aim. The XR adapter transforms reference space exactly once. */ + direction: Vec3 + arm?: boolean + brake?: boolean +} +export const flightLimits = { acceleration: 0.8, speed: 0.8, drag: 2, deadZone: 0.08, maxGap: 0.1 } +const zero: Vec3 = [0, 0, 0] +export const initialFlight = (position: Vec3 = zero): FlightState => ({ + mode: 'inspect', + position, + velocity: zero, + released: false +}) +export const brakeFlight = (state: FlightState): FlightState => ({ + ...state, + mode: 'inspect', + velocity: zero, + released: false +}) +export function stepFlight(state: FlightState, input: FlightInput, dt: number): FlightState { + const length = Math.hypot(...input.direction) + if ( + !input.visible || + !input.tracked || + !Number.isFinite(dt) || + dt < 0 || + dt > flightLimits.maxGap || + !Number.isFinite(input.trigger) || + input.trigger < 0 || + input.trigger > 1 || + !input.direction.every(Number.isFinite) || + length < 0.0001 || + !state.position.every(Number.isFinite) || + !state.velocity.every(Number.isFinite) + ) + return { + ...initialFlight(state.position.every(Number.isFinite) ? state.position : zero), + mode: 'stopped' + } + if (input.brake) return brakeFlight(state) + const released = state.released || input.trigger <= flightLimits.deadZone + const mode = input.arm ? 'armed' : state.mode + if (mode !== 'armed') return { ...state, velocity: zero, released, mode } + // An explicit arm with a held trigger cannot inherit an earlier release. + const ready = input.arm ? input.trigger <= flightLimits.deadZone : released + const thrust = ready + ? Math.pow( + Math.max(0, (input.trigger - flightLimits.deadZone) / (1 - flightLimits.deadZone)), + 1.5 + ) + : 0 + let velocity = [...state.velocity] + let position = [...state.position] + const steps = Math.max(1, Math.ceil(dt * 240)) + const h = dt / steps + const decay = Math.exp(-flightLimits.drag * h) + const gain = (1 - decay) / flightLimits.drag + for (let step = 0; step < steps; step++) { + const acceleration = input.direction.map( + (v) => (v / length) * thrust * flightLimits.acceleration + ) + const next = velocity.map((v, i) => v * decay + acceleration[i] * gain) + const scale = Math.min(1, flightLimits.speed / (Math.hypot(...next) || 1)) + position = position.map((p, i) => p + ((velocity[i] + next[i] * scale) * h) / 2) + velocity = next.map((v) => v * scale) + } + return { + mode, + position: position as unknown as Vec3, + velocity: velocity as unknown as Vec3, + released: ready + } +} + +export type TravelState = { revision: string; position: Vec3; trail: Vec3[] } +export const travelTo = (state: TravelState, destination: Vec3): TravelState => { + if (!destination.every(Number.isFinite)) throw new Error('Destination is unavailable.') + return { ...state, position: destination, trail: [...state.trail.slice(-19), state.position] } +} +export const travelBack = (state: TravelState): TravelState => ({ + ...state, + position: state.trail.at(-1) ?? state.position, + trail: state.trail.slice(0, -1) +}) diff --git a/apps/electron/src/shared/library-graph/model.ts b/apps/electron/src/shared/library-graph/model.ts new file mode 100644 index 000000000..3f19cff56 --- /dev/null +++ b/apps/electron/src/shared/library-graph/model.ts @@ -0,0 +1,136 @@ +import type { LibraryGraph, LibraryGraphRelation } from '../library-graph' + +export const relationKinds: LibraryGraphRelation[] = ['collection', 'tag', 'category', 'creator'] +export const graphColors: Record = { + youtube: '#fb7185', + instagram: '#e879f9', + github: '#c4b5fd', + x: '#7dd3fc', + tiktok: '#2dd4bf', + reddit: '#fb923c', + generic: '#a3e635', + openai: '#6ee7b7', + claude: '#fcd34d', + grok: '#94a3b8', + collection: '#fbbf24', + tag: '#a78bfa', + category: '#67e8f9', + creator: '#fda4af' +} +export const graphColor = (kind: string, platform: string) => + graphColors[kind === 'link' ? platform : kind] ?? '#cbd5e1' + +export function graphAdjacency(graph: LibraryGraph): number[][] { + const adjacency = graph.nodes.map(() => [] as number[]) + graph.edges.forEach(({ source, target }) => { + adjacency[source].push(target) + adjacency[target].push(source) + }) + return adjacency +} + +export function selectGraph( + graph: LibraryGraph, + platform: string, + kinds: LibraryGraphRelation[], + focus: string | null, + groups: string[] = [], + match: 'any' | 'all' = 'any' +): LibraryGraph { + const accepted = new Set(kinds) + const selected = new Set(groups) + const memberships = new Map>() + if (selected.size) { + // Group membership filters do not depend on which relationship lines are shown. + graph.edges.forEach(({ source, target }) => { + const group = graph.nodes[target].id + if (!selected.has(group)) return + const values = memberships.get(source) ?? new Set() + values.add(group) + memberships.set(source, values) + }) + } + const included = new Set( + graph.nodes.flatMap((node, index) => + node.kind === 'link' && + (!platform || node.platform === platform) && + (!selected.size || + (match === 'all' ? memberships.get(index)?.size === selected.size : memberships.has(index))) + ? [index] + : [] + ) + ) + const edges = graph.edges.filter((edge) => accepted.has(edge.kind) && included.has(edge.source)) + let scope: Set | null = null + if (focus) { + const center = graph.nodes.findIndex((node) => node.id === focus) + scope = new Set([center]) + // Two hops from a link includes other links with the same known relationship. + for (let hop = 0; hop < (graph.nodes[center]?.kind === 'link' ? 2 : 1); hop++) { + const previous = new Set(scope) + edges.forEach((edge) => { + if (previous.has(edge.source)) scope!.add(edge.target) + if (previous.has(edge.target)) scope!.add(edge.source) + }) + } + } + const chosenEdges = edges.filter( + (edge) => !scope || (scope.has(edge.source) && scope.has(edge.target)) + ) + const hubs = new Set(chosenEdges.map((edge) => edge.target)) + const indices = new Map() + const nodes = graph.nodes.filter((node, i) => { + const visible = node.kind === 'link' ? included.has(i) && (!scope || scope.has(i)) : hubs.has(i) + if (visible) indices.set(i, indices.size) + return visible + }) + return { + ...graph, + nodes, + linkCount: nodes.filter((node) => node.kind === 'link').length, + edges: chosenEdges + .filter((edge) => indices.has(edge.source) && indices.has(edge.target)) + .map((edge) => ({ + ...edge, + source: indices.get(edge.source)!, + target: indices.get(edge.target)! + })) + } +} + +export function initialPositions(graph: Pick): Float32Array { + const result = new Float32Array(graph.nodes.length * 3) + const hubs = graph.nodes.flatMap((node, i) => (node.kind !== 'link' ? [i] : [])) + const radius = Math.max(80, Math.cbrt(graph.nodes.length) * 28) + graph.nodes.forEach((node, i) => { + let hash = 2166136261 + for (const character of node.id) hash = Math.imul(hash ^ character.charCodeAt(0), 16777619) + for (let axis = 0; axis < 3; axis++) { + hash = Math.imul(hash ^ (hash >>> 16), 2246822507) + result[i * 3 + axis] = ((hash >>> 0) / 4294967296 - 0.5) * radius * 2 + } + const length = Math.hypot(result[i * 3], result[i * 3 + 1], result[i * 3 + 2]) || 1 + const scale = (radius * Math.cbrt((hash >>> 0) / 4294967296)) / length + for (let axis = 0; axis < 3; axis++) result[i * 3 + axis] *= scale + }) + hubs.forEach((index, i) => { + const y = 1 - (2 * (i + 0.5)) / hubs.length + const ring = Math.sqrt(1 - y * y) + result.set( + [Math.cos(i * 2.399963) * ring * radius, y * radius, Math.sin(i * 2.399963) * ring * radius], + index * 3 + ) + }) + const weights = new Float32Array(graph.nodes.length) + const centers = new Float32Array(result.length) + graph.edges.forEach(({ source, target }) => { + weights[source]++ + for (let axis = 0; axis < 3; axis++) centers[source * 3 + axis] += result[target * 3 + axis] + }) + graph.nodes.forEach((_node, i) => { + if (!weights[i]) return + for (let axis = 0; axis < 3; axis++) + result[i * 3 + axis] = centers[i * 3 + axis] / weights[i] + result[i * 3 + axis] * 0.12 + }) + return result +} diff --git a/apps/electron/src/shared/library-graph/navigation.ts b/apps/electron/src/shared/library-graph/navigation.ts new file mode 100644 index 000000000..bbbd4516f --- /dev/null +++ b/apps/electron/src/shared/library-graph/navigation.ts @@ -0,0 +1,71 @@ +import type { LibraryGraph, LibraryGraphNode } from '../library-graph' + +export const groupNames = { + collection: 'Playlists & collections', + tag: 'Tags & topics', + category: 'Categories', + creator: 'Creator names' +} + +export const normalizeSearch = (value: string) => + value.normalize('NFKD').replace(/\p{M}/gu, '').toLocaleLowerCase().trim() + +export type GraphGroup = { node: LibraryGraphNode; count: number } + +export function graphGroups(graph: LibraryGraph, platform: string): GraphGroup[] { + const members = new Map>() + graph.edges.forEach(({ source, target }) => { + if (platform && graph.nodes[source].platform !== platform) return + const links = members.get(target) ?? new Set() + links.add(source) + members.set(target, links) + }) + return [...members] + .map(([index, links]) => ({ node: graph.nodes[index], count: links.size })) + .sort((a, b) => b.count - a.count || a.node.label.localeCompare(b.node.label)) +} + +export function graphSearchIndex(graph: LibraryGraph) { + const degrees = new Uint32Array(graph.nodes.length) + graph.edges.forEach(({ source, target }) => { + degrees[source]++ + degrees[target]++ + }) + return graph.nodes.map((node, index) => ({ + node, + label: normalizeSearch(node.label), + url: normalizeSearch(node.url ?? ''), + count: degrees[index] + })) +} + +export function searchGraph(index: ReturnType, query: string, limit = 12) { + const text = normalizeSearch(query) + const terms = text.split(/\s+/u) + const matches = index.flatMap((entry) => { + if (!text) return entry.node.kind === 'link' ? [] : [{ ...entry, rank: 0 }] + if (!terms.every((term) => entry.label.includes(term) || entry.url.includes(term))) return [] + const rank = + entry.label === text + ? 0 + : entry.label.startsWith(text) + ? 1 + : terms.every((term) => + entry.label.split(/[^\p{L}\p{N}]+/u).some((word) => word.startsWith(term)) + ) + ? 2 + : terms.every((term) => entry.label.includes(term)) + ? 3 + : 4 + return [{ ...entry, rank }] + }) + matches.sort( + (a, b) => + a.rank - b.rank || + Number(a.node.kind === 'link') - Number(b.node.kind === 'link') || + (a.node.kind === 'link' ? 0 : b.count - a.count) || + a.label.localeCompare(b.label) || + a.node.id.localeCompare(b.node.id) + ) + return { total: matches.length, items: matches.slice(0, limit) } +} diff --git a/apps/electron/src/shared/spatial-library.test.ts b/apps/electron/src/shared/spatial-library.test.ts new file mode 100644 index 000000000..1a3717ead --- /dev/null +++ b/apps/electron/src/shared/spatial-library.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest' +import { fixtureAdapter } from '../spatial/fixtures' +import { selectGraph, relationKinds } from './library-graph/model' +import { graphSearchIndex, searchGraph } from './library-graph/navigation' +import { projectSpatialSnapshot, validateSpatialSnapshot } from './spatial-library' + +describe('spatial snapshots', () => { + it('rejects mismatched revisions, invalid positions, duplicate identities and bad edges', async () => { + const snapshot = await fixtureAdapter(1000).snapshot(new AbortController().signal) + expect(validateSpatialSnapshot(snapshot)).toBe(snapshot) + expect(() => validateSpatialSnapshot({ ...snapshot, layoutRevision: 'other' })).toThrow() + expect(() => validateSpatialSnapshot({ ...snapshot, positions: [NaN] })).toThrow() + expect(() => + validateSpatialSnapshot({ + ...snapshot, + graph: { + ...snapshot.graph, + nodes: [snapshot.graph.nodes[0], ...snapshot.graph.nodes.slice(0, -1)] + } + }) + ).toThrow() + expect(() => + validateSpatialSnapshot({ + ...snapshot, + graph: { ...snapshot.graph, edges: [{ ...snapshot.graph.edges[0], source: -1 }] } + }) + ).toThrow() + }) + it('retains resource coordinates under source, group and neighborhood filtering', async () => { + const snapshot = await fixtureAdapter(1000).snapshot(new AbortController().signal) + const visible = selectGraph(snapshot.graph, 'youtube', relationKinds, null, ['group:0']) + const projected = projectSpatialSnapshot(snapshot, visible) + for (const [i, node] of visible.nodes.entries()) { + const old = snapshot.graph.nodes.findIndex((n) => n.id === node.id) + expect(projected.positions.slice(i * 3, i * 3 + 3)).toEqual( + snapshot.positions.slice(old * 3, old * 3 + 3) + ) + } + expect(visible.linkCount).toBeGreaterThan(0) + expect(visible.linkCount).toBeLessThan(1000) + }) + it('keeps the full fixture searchable independently of rendering budgets', async () => { + const snapshot = await fixtureAdapter(56_000).snapshot(new AbortController().signal) + expect(snapshot.graph.linkCount).toBe(56_000) + expect( + searchGraph(graphSearchIndex(snapshot.graph), 'Learning resource 56000').items[0].node.id + ).toBe('demo:55999') + }) +}) diff --git a/apps/electron/src/shared/spatial-library.ts b/apps/electron/src/shared/spatial-library.ts new file mode 100644 index 000000000..45e0506ee --- /dev/null +++ b/apps/electron/src/shared/spatial-library.ts @@ -0,0 +1,103 @@ +import type { LibraryGraph } from './library-graph' +import { initialPositions } from './library-graph/model' + +/** Ephemeral viewer projection, not a persisted Library schema. */ +export type SpatialSnapshot = { + revision: string + layoutRevision: string + graph: LibraryGraph + positions: number[] +} +export type SpatialDetail = { + id: string + title: string + description: string + platform: string + author: string + coverage: Record + transcript: { source: string; language: string; cues: number } | null + thumbnail: boolean + text?: string +} +export interface SpatialLibraryAdapter { + snapshot(signal: AbortSignal): Promise + detail(id: string, signal: AbortSignal, text?: boolean): Promise + thumbnail(id: string, signal: AbortSignal): Promise + close(): void +} + +export function createSpatialSnapshot(graph: LibraryGraph, revision: string): SpatialSnapshot { + const raw = initialPositions(graph) + let radius = 1 + for (let i = 0; i < raw.length; i += 3) + radius = Math.max(radius, Math.hypot(raw[i], raw[i + 1], raw[i + 2])) + // A four-metre overview with its center ahead of a stationary viewer. + const positions = Array.from( + raw, + (value, i) => (value / radius) * 2 + (i % 3 === 1 ? 1.4 : i % 3 === 2 ? -3 : 0) + ) + return { revision, layoutRevision: revision, graph, positions } +} + +export function validateSpatialSnapshot(value: SpatialSnapshot): SpatialSnapshot { + const { graph, positions, revision, layoutRevision } = value + if ( + typeof revision !== 'string' || + !revision || + revision !== layoutRevision || + !graph || + !Array.isArray(graph.nodes) || + !Array.isArray(graph.edges) || + graph.nodes.length > 120_000 || + graph.edges.length > 1_000_000 || + !Array.isArray(positions) || + positions.length !== graph.nodes.length * 3 || + !positions.every((n) => Number.isFinite(n) && Math.abs(n) <= 10_000) + ) + throw new Error('The graph and layout do not form a valid matching snapshot.') + const ids = new Set() + for (const node of graph.nodes) { + if ( + !node || + typeof node.id !== 'string' || + !node.id || + ids.has(node.id) || + typeof node.label !== 'string' || + typeof node.platform !== 'string' || + !['link', 'collection', 'tag', 'category', 'creator'].includes(node.kind) || + (node.url !== undefined && typeof node.url !== 'string') + ) + throw new Error('The graph has an invalid or duplicate resource.') + ids.add(node.id) + } + for (const edge of graph.edges) { + if ( + !edge || + !Number.isInteger(edge.source) || + !Number.isInteger(edge.target) || + !graph.nodes[edge.source] || + !graph.nodes[edge.target] || + !['collection', 'tag', 'category', 'creator'].includes(edge.kind) || + !['import', 'metadata', 'hashtag'].includes(edge.evidence) + ) + throw new Error('The graph has an invalid relationship.') + } + return value +} + +/** Filtering never recomputes positions or combines offsets from different revisions. */ +export function projectSpatialSnapshot( + snapshot: SpatialSnapshot, + graph: LibraryGraph +): SpatialSnapshot { + const indices = new Map(snapshot.graph.nodes.map((node, index) => [node.id, index])) + return validateSpatialSnapshot({ + ...snapshot, + graph, + positions: graph.nodes.flatMap((node) => { + const index = indices.get(node.id) + if (index === undefined) throw new Error('Filtered resource is absent from this snapshot.') + return snapshot.positions.slice(index * 3, index * 3 + 3) + }) + }) +} diff --git a/apps/electron/src/spatial/fixtures.ts b/apps/electron/src/spatial/fixtures.ts new file mode 100644 index 000000000..a90b6035e --- /dev/null +++ b/apps/electron/src/spatial/fixtures.ts @@ -0,0 +1,57 @@ +import type { SpatialLibraryAdapter } from '../shared/spatial-library' +import { createSpatialSnapshot } from '../shared/spatial-library' + +export function fixtureAdapter(count: number): SpatialLibraryAdapter { + const groups = ['collection', 'tag', 'category', 'creator'] as const + const snapshot = createSpatialSnapshot( + { + nodes: [ + ...Array.from({ length: count }, (_, i) => ({ + id: `demo:${i}`, + label: `Learning resource ${i + 1}`, + kind: 'link' as const, + platform: ['youtube', 'github', 'web'][i % 3], + url: `https://example.org/resource/${i + 1}` + })), + ...Array.from({ length: 40 }, (_, i) => ({ + id: `group:${i}`, + label: `${groups[i % 4]} ${i + 1}`, + kind: groups[i % 4], + platform: '' + })) + ], + edges: Array.from({ length: count * 2 }, (_, i) => ({ + source: Math.floor(i / 2), + target: count + (i % 40), + kind: groups[i % 4], + evidence: 'import' as const + })), + linkCount: count, + resourceCount: count, + builtAt: 0, + warnings: ['Synthetic data. No personal Library content is loaded.'] + }, + `synthetic-${count}-v1` + ) + return { + snapshot: async () => snapshot, + detail: async (id) => { + const node = snapshot.graph.nodes.find((node) => node.id === id) + return node + ? { + id, + title: node.label, + description: + 'A synthetic saved link for testing selection, search, filters and headset performance.', + platform: node.platform, + author: 'Demo', + coverage: { description: { state: 'partial', reason: 'Synthetic preview only' } }, + transcript: null, + thumbnail: false + } + : null + }, + thumbnail: async () => null, + close() {} + } +} diff --git a/docs/explorations/0468_[_]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md b/docs/explorations/0468_[_]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md index 967cc88ec..fc5a274eb 100644 --- a/docs/explorations/0468_[_]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md +++ b/docs/explorations/0468_[_]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md @@ -261,10 +261,10 @@ Use the existing desktop interface as the accessible fallback. In-headset contro ### 2. Make one small graph useful in the headset -- [ ] Extract the graph data boundary and share model/layout logic without making the browser depend on Electron preload. +- [x] Extract the graph data boundary and share model/layout logic without making the browser depend on Electron preload. - [ ] Add XR session lifecycle, metre scaling, a locomotion rig, and stable precomputed positions; restore desktop controls on exit. - [ ] Add stationary overview, selected-resource highlighting, spatial labels, one metadata card, and a home action. -- [ ] Implement point-and-accelerate flight, configurable handedness, braking, and interruption handling as pure state transitions with tested math. +- [x] Implement point-and-accelerate flight, configurable handedness, braking, and interruption handling as pure state transitions with tested math. - [ ] Add explicit search destination travel and a return trail before enabling long-distance free flight. - [ ] Prototype full-pose flight behind a separate opt-in only after the default flight interaction passes hardware review. @@ -285,7 +285,7 @@ The named consumer of this validation is the **spatial Library release review**, - [ ] Demonstrate that trigger pressure changes acceleration, the speed cap holds, the head remains independent, and braking does not require a precise gesture. - [ ] Demonstrate full position/rotation input separately from ordinary gamepad button support; record which modes the tested platform supports. - [ ] Exercise tracking loss, invalid poses, controller disconnect, session suspension, and re-entry with a held trigger; all must stop motion until explicitly rearmed. -- [ ] Verify deterministic flight integration with in-memory inputs at several frame rates, including large time gaps and non-finite input. If a new automated gate is added, include negative controls that make it fail when braking or limits are broken. +- [x] Verify deterministic flight integration with in-memory inputs at several frame rates, including large time gaps and non-finite input. If a new automated gate is added, include negative controls that make it fail when braking or limits are broken. - [ ] Confirm selection and card inspection never cause unintended acceleration or automatic viewpoint jumps. - [ ] Complete a proposed 20-minute browsing trial, record Chris's comfort and task feedback, and verify the stationary mode remains useful. This is a product trial, not a universal comfort guarantee. - [ ] Meet the declared frame-time and memory budgets on the full-size fixture, including selection and filtering; retain every resource's searchability under level-of-detail reduction. From c6821c67349d4e8c5ca28b1478006569207129ac Mon Sep 17 00:00:00 2001 From: xNet Test Date: Tue, 6 Oct 2026 10:25:49 -0700 Subject: [PATCH 2/4] feat(library): pair scoped spatial readers over private HTTPS - Exchange one-use invitations for expiring and revocable read-only capabilities. - Restrict reads to selected link snapshots, details and bounded thumbnails. - Exclude conversation records from graph projections and test verified TLS revocation. - Add the native pairing control and record the transport decision with a tripwire. Signed-off-by: xNet Test --- apps/electron/src/library/graph.test.ts | 24 ++ apps/electron/src/library/graph.ts | 6 +- apps/electron/src/library/store.test.ts | 9 + apps/electron/src/library/store.ts | 2 +- apps/electron/src/main/index.ts | 3 + apps/electron/src/main/spatial-bridge.test.ts | 92 +++++++ apps/electron/src/main/spatial-bridge.ts | 102 ++++++++ apps/electron/src/main/spatial-library-ipc.ts | 236 ++++++++++++++++++ apps/electron/src/main/spatial-server.test.ts | 127 ++++++++++ apps/electron/src/main/spatial-server.ts | 79 ++++++ apps/electron/src/preload/index.ts | 12 + .../renderer/components/LibraryGraphView.tsx | 6 + .../library-graph/SpatialPairing.tsx | 117 +++++++++ .../docs/docs/architecture/decisions.mdx | 34 +++ 14 files changed, 847 insertions(+), 2 deletions(-) create mode 100644 apps/electron/src/main/spatial-bridge.test.ts create mode 100644 apps/electron/src/main/spatial-bridge.ts create mode 100644 apps/electron/src/main/spatial-library-ipc.ts create mode 100644 apps/electron/src/main/spatial-server.test.ts create mode 100644 apps/electron/src/main/spatial-server.ts create mode 100644 apps/electron/src/renderer/components/library-graph/SpatialPairing.tsx diff --git a/apps/electron/src/library/graph.test.ts b/apps/electron/src/library/graph.test.ts index 3bb3d1b63..9c0087252 100644 --- a/apps/electron/src/library/graph.test.ts +++ b/apps/electron/src/library/graph.test.ts @@ -13,6 +13,30 @@ const resource = (id: string, values: Partial = {}): GraphResourc ...values }) +it('excludes conversations and archive text even when they have ordinary web URLs', () => { + const builder = createGraphBuilder( + [ + resource('link'), + resource('private-conversation', { + resourceKind: 'conversation', + hashtags: ['private-topic'], + author: 'Private person' + }), + resource('archive-body', { resourceKind: 'archive-text' }) + ], + 3 + ) + builder.collection('private-list', { title: 'Private collection' }) + builder.membership({ collection: 'private-list', item: 'private-conversation' }) + builder.content('private-conversation', { metadataJson: JSON.stringify({ tags: ['secret'] }) }) + expect(builder.finish()).toMatchObject({ + nodes: [expect.objectContaining({ id: 'link' })], + edges: [], + linkCount: 1, + resourceCount: 3 + }) +}) + it('keeps every link, including orphans, and deduplicates repeated memberships', () => { const builder = createGraphBuilder( Array.from({ length: 1200 }, (_, i) => resource(`${i}`)), diff --git a/apps/electron/src/library/graph.ts b/apps/electron/src/library/graph.ts index c4575ee6f..c588f68a6 100644 --- a/apps/electron/src/library/graph.ts +++ b/apps/electron/src/library/graph.ts @@ -9,6 +9,7 @@ import { export type GraphResource = { id: string + resourceKind?: string | null url: string title: string platform: string @@ -42,7 +43,10 @@ export const hashtagsIn = (value: string): string[] => [ ] /** Hubs preserve source relationships without expanding a playlist into a quadratic clique. */ -export function createGraphBuilder(resources: GraphResource[], resourceCount: number) { +export function createGraphBuilder(allResources: GraphResource[], resourceCount: number) { + // A conversation can carry an HTTP source URL. A URL alone is not permission + // to include its title, hashtags or relationships in the saved-link projection. + const resources = allResources.filter((resource) => !resource.resourceKind) const graph: LibraryGraph = { nodes: resources.map((resource) => ({ id: resource.id, diff --git a/apps/electron/src/library/store.test.ts b/apps/electron/src/library/store.test.ts index 45c434ad1..3a114c5ae 100644 --- a/apps/electron/src/library/store.test.ts +++ b/apps/electron/src/library/store.test.ts @@ -20,6 +20,14 @@ const resource = (id = 'video-one'): LibraryResource => ({ privacy: 'private', addedAt: 1 }) + +it('preserves text-resource kinds even when their source URL is HTTP', () => { + store.put({ ...resource('conversation'), kind: 'conversation', sourceText: 'private body' }) + const row = store.graphResources().find((item) => item.id === 'conversation') + expect(row?.resourceKind).toBe('conversation') + expect(row).not.toHaveProperty('body') + expect(store.get('conversation')?.sourceText).toBe('private body') +}) beforeEach(async () => { root = await mkdtemp(join(tmpdir(), 'xnet-library-')) path = join(root, 'library.db') @@ -384,6 +392,7 @@ it('projects every web link for the graph without source bodies or caption paylo expect(rows.find((row) => row.id === 'local')).toBeUndefined() expect(rows.find((row) => row.id === 'tagged')).toEqual({ id: 'tagged', + resourceKind: null, title: 'Fetched title', url: resource().url, platform: 'youtube', diff --git a/apps/electron/src/library/store.ts b/apps/electron/src/library/store.ts index bc8033cba..aa7d642a0 100644 --- a/apps/electron/src/library/store.ts +++ b/apps/electron/src/library/store.ts @@ -243,7 +243,7 @@ export class LibraryStore { // source bodies or provider evidence with the overview's compact nodes. const rows = this.db .prepare( - `SELECT id,url,title,platform, + `SELECT id,url,title,platform,json_extract(payload,'$.kind') AS resourceKind, COALESCE(json_extract(payload,'$.networkPlatform'),platform) AS provider, COALESCE(NULLIF(json_extract(payload,'$.metadata.author'),''),json_extract(payload,'$.actor'),'') AS author, COALESCE(json_extract(payload,'$.sourceText'),'') || char(10) || diff --git a/apps/electron/src/main/index.ts b/apps/electron/src/main/index.ts index f6ce5248f..2ede18ac5 100644 --- a/apps/electron/src/main/index.ts +++ b/apps/electron/src/main/index.ts @@ -35,6 +35,7 @@ import { checkpointWorkspace, recoveryPath, recoveryIsBusy, setupRecovery } from import { flushRenderers, resumeRenderers, setupRendererFlush } from './renderer-flush' import { setupServiceIPC, cleanupServices } from './service-ipc' import { setupSocialImportIPC, hasActiveSocialImports } from './social-import-ipc' +import { setupSpatialLibraryIPC, stopSpatialLibrary } from './spatial-library-ipc' import { showStartupRecovery } from './startup-recovery' import { setupStorybookIPC, stopStorybook } from './storybook-ipc' import { hasDownloadedUpdate, initAutoUpdater, installDownloadedUpdate } from './updater' @@ -77,6 +78,7 @@ let writersStopped = false async function stopWorkspaceWriters(): Promise { await shutdownRecordingCapture() await stopAgentBridge() + await stopSpatialLibrary() await stopLocalAPI() await cleanupServices() await stopCloudflareTunnel() @@ -486,6 +488,7 @@ app bootTrace('data process ready') await configureLibrary() setupLibraryIPC(() => mainWindow) + setupSpatialLibraryIPC() // Setup IPC handlers for main process operations setupIPC() diff --git a/apps/electron/src/main/spatial-bridge.test.ts b/apps/electron/src/main/spatial-bridge.test.ts new file mode 100644 index 000000000..048a6b714 --- /dev/null +++ b/apps/electron/src/main/spatial-bridge.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it, vi } from 'vitest' +import { fixtureAdapter } from '../spatial/fixtures' +import { createSpatialBridge } from './spatial-bridge' + +const origin = 'https://mac.example:8443' +async function setup() { + let clock = 1000 + const adapter = fixtureAdapter(10), + signal = new AbortController().signal + const reader = { + snapshot: await adapter.snapshot(signal), + detail: vi.fn((id: string) => adapter.detail(id, signal)), + thumbnail: vi.fn(async () => ({ data: new Uint8Array([1]), contentType: 'image/png' })) + } + const original = JSON.stringify(reader.snapshot) + const bridge = createSpatialBridge(reader, origin, () => clock) + const request = (path: string, token = '', method = 'GET', source = origin) => + bridge.handle( + new Request(origin + path, { + method, + headers: { Origin: source, Authorization: `Bearer ${token}` } + }) + ) + const paired = await request('/pair', bridge.invitation, 'POST') + const { token } = (await paired.json()) as { token: string } + return { + bridge, + reader, + request, + token, + original, + time: (n: number) => { + clock += n + } + } +} +describe('scoped spatial pairing', () => { + it('requires a single-use invitation and never accepts the general API token', async () => { + const { bridge, request, token } = await setup() + expect((await request('/pair', bridge.invitation, 'POST')).status).toBe(401) + expect((await request('/api/snapshot')).status).toBe(401) + expect((await request('/api/snapshot', 'general-api-token')).status).toBe(401) + expect((await request('/api/snapshot', token)).status).toBe(200) + expect((await request('/api/snapshot', token, 'GET', 'https://evil.example')).status).toBe(403) + }) + it('has no writes, arbitrary blob lookup, SQL or outside-scope details', async () => { + const { request, token, reader, original } = await setup() + expect((await request('/api/detail/demo%3A0', token, 'POST')).status).toBe(405) + expect((await request('/api/detail/private-conversation', token)).status).toBe(404) + expect((await request('/api/thumbnail/arbitrary-cid', token)).status).toBe(404) + expect((await request('/sql', token)).status).toBe(404) + expect(reader.detail).not.toHaveBeenCalled() + expect(reader.thumbnail).not.toHaveBeenCalled() + expect(JSON.stringify(reader.snapshot)).toBe(original) + }) + it('supports scoped snapshot, search, detail and image reads', async () => { + const { request, token } = await setup() + for (const path of [ + '/api/snapshot', + '/api/search?q=resource', + '/api/detail/demo%3A0', + '/api/thumbnail/demo%3A0' + ]) { + const response = await request(path, token) + expect(response.status).toBe(200) + expect(response.headers.get('cache-control')).toBe('no-store') + expect(response.headers.get('access-control-allow-origin')).toBeNull() + } + }) + it('revokes in-flight reads as well as subsequent requests', async () => { + const { bridge, reader, request, token } = await setup() + let finish: (value: null) => void = () => undefined + reader.detail.mockImplementation( + () => + new Promise((resolve) => { + finish = resolve + }) + ) + const pending = request('/api/detail/demo%3A0', token) + bridge.revoke() + finish(null) + expect((await pending).status).toBe(401) + expect((await request('/api/snapshot', token)).status).toBe(401) + }) + it('expires the reader and surfaces storage errors without plausible empty data', async () => { + const { time, request, token, reader } = await setup() + reader.detail.mockRejectedValue(new Error('database unavailable')) + expect((await request('/api/detail/demo%3A0', token)).status).toBe(503) + time(60 * 60_000) + expect((await request('/api/snapshot', token)).status).toBe(401) + }) +}) diff --git a/apps/electron/src/main/spatial-bridge.ts b/apps/electron/src/main/spatial-bridge.ts new file mode 100644 index 000000000..53152e117 --- /dev/null +++ b/apps/electron/src/main/spatial-bridge.ts @@ -0,0 +1,102 @@ +import type { SpatialDetail, SpatialSnapshot } from '../shared/spatial-library' +import { randomBytes, timingSafeEqual } from 'node:crypto' +import { graphSearchIndex, searchGraph } from '../shared/library-graph/navigation' + +type SpatialReader = { + snapshot: SpatialSnapshot + detail(id: string, text: boolean): Promise + thumbnail(id: string): Promise<{ data: Uint8Array; contentType: string } | null> +} +const secret = () => randomBytes(32).toString('base64url') +const same = (a: string, b: string) => + a.length === b.length && timingSafeEqual(Buffer.from(a), Buffer.from(b)) +const headers = { + 'Cache-Control': 'no-store', + 'X-Content-Type-Options': 'nosniff', + 'Referrer-Policy': 'no-referrer', + 'Content-Type': 'application/json' +} +const json = (value: unknown, status = 200) => + new Response(JSON.stringify(value), { status, headers }) + +/** One owner-approved snapshot, one invitation, one short-lived reader. No write routes. */ +export function createSpatialBridge(reader: SpatialReader, origin: string, now = Date.now) { + const invitation = secret(), + token = secret() + const issued = now(), + expiresAt = issued + 60 * 60_000 + const scope = new Set( + reader.snapshot.graph.nodes.filter((node) => node.kind === 'link').map((node) => node.id) + ) + const snapshot = JSON.stringify(reader.snapshot) + const search = graphSearchIndex(reader.snapshot.graph) + let redeemed = false, + revoked = false, + active = 0 + let requests: number[] = [] + const valid = () => !revoked && now() < expiresAt + return { + invitation, + expiresAt, + revoke() { + revoked = true + }, + async handle(request: Request): Promise { + const url = new URL(request.url) + if ( + url.origin !== origin || + (request.headers.has('origin') && request.headers.get('origin') !== origin) + ) + return json({ error: 'Origin denied' }, 403) + if (!valid()) return json({ error: 'Pairing expired or revoked' }, 401) + const auth = request.headers.get('authorization')?.replace(/^Bearer /, '') ?? '' + if (!/^[A-Za-z0-9_-]{43}$/.test(auth)) return json({ error: 'Unauthorized' }, 401) + if (url.pathname === '/pair') { + if (request.method !== 'POST') return json({ error: 'Method denied' }, 405) + if (redeemed || now() - issued > 5 * 60_000 || !same(auth, invitation)) + return json({ error: 'Invitation unavailable' }, 401) + redeemed = true + return json({ token, expiresAt }) + } + if (!redeemed || !same(auth, token)) return json({ error: 'Unauthorized' }, 401) + if (request.method !== 'GET') return json({ error: 'Read-only viewer' }, 405) + requests = requests.filter((time) => time > now() - 60_000) + if (active >= 4 || requests.length >= 120) + return new Response('Read limit reached', { + status: 429, + headers: { ...headers, 'Retry-After': '60' } + }) + requests.push(now()) + active++ + try { + if (url.pathname === '/api/snapshot') return new Response(snapshot, { headers }) + if (url.pathname === '/api/search') + return json(searchGraph(search, (url.searchParams.get('q') ?? '').slice(0, 300), 30)) + const match = url.pathname.match(/^\/api\/(detail|thumbnail)\/([^/]+)$/) + if (!match) return json({ error: 'Not found' }, 404) + const id = decodeURIComponent(match[2]) + if (!scope.has(id)) return json({ error: 'Outside paired scope' }, 404) + if (match[1] === 'detail') { + const detail = await reader.detail(id, url.searchParams.get('text') === '1') + if (!valid()) return json({ error: 'Pairing expired or revoked' }, 401) + return detail ? json(detail) : json({ error: 'Resource unavailable' }, 404) + } + const blob = await reader.thumbnail(id) + if (!valid()) return json({ error: 'Pairing expired or revoked' }, 401) + if (!blob) return json({ error: 'Thumbnail unavailable' }, 404) + if ( + blob.data.byteLength > 8 * 1024 * 1024 || + !['image/png', 'image/jpeg', 'image/webp'].includes(blob.contentType) + ) + return json({ error: 'Unsupported thumbnail' }, 415) + return new Response(Buffer.from(blob.data), { + headers: { ...headers, 'Content-Type': blob.contentType } + }) + } catch { + return json({ error: 'The Mac could not read this item' }, 503) + } finally { + active-- + } + } + } +} diff --git a/apps/electron/src/main/spatial-library-ipc.ts b/apps/electron/src/main/spatial-library-ipc.ts new file mode 100644 index 000000000..ee62b4fe8 --- /dev/null +++ b/apps/electron/src/main/spatial-library-ipc.ts @@ -0,0 +1,236 @@ +import type { LibraryGraph, LibraryGraphDetail } from '../shared/library-graph' +import type { SpatialDetail } from '../shared/spatial-library' +import type { Server } from 'node:https' +import { X509Certificate, randomUUID } from 'node:crypto' +import { readFile, readdir } from 'node:fs/promises' +import { isIP } from 'node:net' +import { join, resolve } from 'node:path' +import { app, clipboard, dialog, ipcMain } from 'electron' +import { createSpatialSnapshot } from '../shared/spatial-library' +import { sendDataProcessRequest } from './data-process-manager' +import { createSpatialBridge } from './spatial-bridge' +import { createSpatialServer } from './spatial-server' + +let server: Server | null = null +let revoke: (() => void) | null = null +let expiry: ReturnType | null = null +let starting = false +let generation = 0 +let pairing: { url: string; expiresAt: number; count: number } | null = null +export async function stopSpatialLibrary(): Promise { + generation++ + revoke?.() + revoke = null + pairing = null + if (expiry) clearTimeout(expiry) + expiry = null + const old = server + server = null + if (old) { + old.closeAllConnections() + await new Promise((done) => old.close(() => done())) + } +} +async function resource(id: string) { + const response = (await sendDataProcessRequest('library:graph-detail', { id }, 120_000)) as { + value: LibraryGraphDetail + } + const item = response.value.resource + return item && !item.kind && /^https?:\/\//.test(item.url) ? item : null +} +export function setupSpatialLibraryIPC() { + ipcMain.handle('xnet:spatial:status', () => + pairing ? { ...pairing, url: new URL(pairing.url).origin } : null + ) + ipcMain.handle('xnet:spatial:stop', stopSpatialLibrary) + ipcMain.handle('xnet:spatial:copy', () => { + if (!pairing) throw new Error('Create a pairing first.') + clipboard.writeText(pairing.url) + }) + ipcMain.handle('xnet:spatial:start', async (_event, input: { origin: string; ids: string[] }) => { + if (starting || server) throw new Error('Revoke the current pairing before creating another.') + if ( + !input || + typeof input.origin !== 'string' || + !Array.isArray(input.ids) || + input.ids.length < 1 || + input.ids.length > 100_000 || + !input.ids.every((id) => typeof id === 'string' && id.length <= 500) + ) + throw new Error('Choose saved links to share with the viewer.') + const url = new URL(input.origin) + if ( + url.protocol !== 'https:' || + url.username || + url.password || + url.pathname !== '/' || + url.search || + url.hash || + !url.port || + Number(url.port) < 1024 + ) + throw new Error('Use your Mac’s trusted HTTPS address with a port above 1023.') + starting = true + const attempt = generation + try { + const certChoice = await dialog.showOpenDialog({ + title: 'Choose the trusted HTTPS certificate for this Mac', + properties: ['openFile'], + filters: [{ name: 'Certificate', extensions: ['pem', 'crt'] }] + }) + if (certChoice.canceled) throw new Error('Pairing canceled.') + const keyChoice = await dialog.showOpenDialog({ + title: 'Choose its private TLS key (kept on this Mac)', + properties: ['openFile'], + filters: [{ name: 'Private key', extensions: ['pem', 'key'] }] + }) + if (keyChoice.canceled) throw new Error('Pairing canceled.') + const cert = await readFile(certChoice.filePaths[0]), + key = await readFile(keyChoice.filePaths[0]) + const certificate = new X509Certificate(cert) + const hostname = url.hostname.replace(/^\[|\]$/g, '') + if ( + !(isIP(hostname) ? certificate.checkIP(hostname) : certificate.checkHost(hostname)) || + Date.parse(certificate.validTo) < Date.now() || + Date.parse(certificate.validFrom) > Date.now() + ) + throw new Error( + 'The certificate is expired, not yet valid, or does not cover this address.' + ) + // Serve only compiled browser assets. Never proxy the renderer dev server or debugger. + const directory = resolve(app.getAppPath(), 'out/spatial') + const assets = new Map() + const readAssets = async (dir: string, prefix: string) => { + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = `${prefix}/${entry.name}` + if (entry.isDirectory()) await readAssets(join(dir, entry.name), path) + else if (/\.(html|js|css)$/.test(entry.name)) + assets.set(path, { + data: await readFile(join(dir, entry.name)), + type: entry.name.endsWith('.js') + ? 'text/javascript' + : entry.name.endsWith('.css') + ? 'text/css' + : 'text/html' + }) + } + } + await readAssets(directory, '') + if (!assets.has('/index.html')) + throw new Error('The spatial viewer is missing from this app build.') + const response = (await sendDataProcessRequest('library:graph', {}, 600_000)) as { + value: string + } + const graph = JSON.parse(response.value) as LibraryGraph + const chosen = new Set(input.ids) + const scope = new Set( + graph.nodes.flatMap((node, i) => (node.kind === 'link' && chosen.has(node.id) ? [i] : [])) + ) + const edges = graph.edges.filter((edge) => scope.has(edge.source)) + edges.forEach((edge) => scope.add(edge.target)) + const remap = new Map() + const nodes = graph.nodes.filter((_node, i) => { + if (!scope.has(i)) return false + remap.set(i, remap.size) + return true + }) + const scoped = { + ...graph, + nodes, + edges: edges.map((edge) => ({ + ...edge, + source: remap.get(edge.source)!, + target: remap.get(edge.target)! + })), + linkCount: nodes.filter((node) => node.kind === 'link').length, + resourceCount: nodes.filter((node) => node.kind === 'link').length, + warnings: [] + } + if (!scoped.linkCount) throw new Error('The selected saved links are no longer available.') + const bridge = createSpatialBridge( + { + snapshot: createSpatialSnapshot(scoped, randomUUID()), + async detail(id, text): Promise { + const item = await resource(id) + if (!item) return null + return { + id, + title: item.metadata?.title || item.title, + description: (item.metadata?.description ?? '').slice(0, 8000), + platform: item.platform, + author: item.metadata?.author ?? item.actor, + coverage: item.metadata?.fields ?? {}, + transcript: item.transcript + ? { + source: item.transcript.source, + language: item.transcript.language, + cues: item.transcript.cues.length + } + : null, + thumbnail: !!item.thumbnail, + ...(text + ? { + text: ( + item.transcript?.cues.map((cue) => cue.text).join('\n') ?? item.sourceText + ).slice(0, 100_000) + } + : {}) + } + }, + async thumbnail(id) { + const item = await resource(id) + if (!item?.thumbnail || item.thumbnail.bytes > 8 * 1024 * 1024) return null + const blob = (await sendDataProcessRequest( + 'blob:get', + { cid: item.thumbnail.cid }, + 60_000 + )) as { data: number[] | null } + return blob.data + ? { data: new Uint8Array(blob.data), contentType: item.thumbnail.contentType } + : null + } + }, + url.origin + ) + const next = createSpatialServer({ cert, key, origin: url.origin, assets, bridge }) + if (attempt !== generation) { + bridge.revoke() + throw new Error('Pairing was canceled.') + } + await new Promise((done, reject) => { + next.once('error', reject) + next.listen(Number(url.port), '0.0.0.0', () => { + next.removeListener('error', reject) + done() + }) + }) + if (attempt !== generation) { + bridge.revoke() + next.closeAllConnections() + await new Promise((done) => next.close(() => done())) + throw new Error('Pairing was canceled.') + } + server = next + next.on('error', (error: Error) => { + console.error('[Spatial Library] HTTPS listener failed:', error.message) + void stopSpatialLibrary() + }) + revoke = bridge.revoke + pairing = { + url: `${url.origin}/#pair=${bridge.invitation}`, + expiresAt: bridge.expiresAt, + count: scoped.linkCount + } + expiry = setTimeout( + () => { + void stopSpatialLibrary() + }, + Math.max(0, bridge.expiresAt - Date.now()) + ) + clipboard.writeText(pairing.url) + return { ...pairing, url: url.origin } + } finally { + starting = false + } + }) +} diff --git a/apps/electron/src/main/spatial-server.test.ts b/apps/electron/src/main/spatial-server.test.ts new file mode 100644 index 000000000..a47ca9254 --- /dev/null +++ b/apps/electron/src/main/spatial-server.test.ts @@ -0,0 +1,127 @@ +import type { Server } from 'node:https' +import { execFileSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { request } from 'node:https' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { fixtureAdapter } from '../spatial/fixtures' +import { createSpatialBridge } from './spatial-bridge' +import { createSpatialServer } from './spatial-server' + +describe('private HTTPS spatial transport', () => { + let server: Server, certificate: Buffer, port: number + let bridge: ReturnType + let directory: string + const origin = 'https://localhost:8443' + beforeAll(async () => { + directory = mkdtempSync(join(tmpdir(), 'xnet-spatial-tls-')) + execFileSync( + 'openssl', + [ + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-nodes', + '-days', + '1', + '-keyout', + join(directory, 'key.pem'), + '-out', + join(directory, 'cert.pem'), + '-subj', + '/CN=localhost', + '-addext', + 'subjectAltName=DNS:localhost' + ], + { stdio: 'ignore' } + ) + certificate = readFileSync(join(directory, 'cert.pem')) + const adapter = fixtureAdapter(10), + signal = new AbortController().signal + bridge = createSpatialBridge( + { + snapshot: await adapter.snapshot(signal), + detail: (id) => adapter.detail(id, signal), + thumbnail: async () => null + }, + origin + ) + server = createSpatialServer({ + cert: certificate, + key: readFileSync(join(directory, 'key.pem')), + origin, + bridge, + assets: new Map([ + ['/index.html', { data: Buffer.from('

Synthetic viewer

'), type: 'text/html' }] + ]) + }) + await new Promise((done) => server.listen(0, '127.0.0.1', done)) + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Missing TLS port') + port = address.port + }) + afterAll(async () => { + bridge?.revoke() + if (server) { + server.closeAllConnections() + await new Promise((done) => server.close(() => done())) + } + if (directory) rmSync(directory, { recursive: true, force: true }) + }) + const read = (path: string, token = '', method = 'GET', headers: Record = {}) => + new Promise<{ status: number; headers: Record; body: string }>( + (done, reject) => { + // Trust only the temporary test certificate; normal hostname verification stays enabled. + const req = request( + { + hostname: '127.0.0.1', + servername: 'localhost', + port, + path, + method, + ca: certificate, + headers: { + Host: 'localhost:8443', + Origin: origin, + Authorization: `Bearer ${token}`, + ...headers + } + }, + (response) => { + let body = '' + response.setEncoding('utf8') + response.on('data', (chunk: string) => { + body += chunk + }) + response.on('end', () => + done({ status: response.statusCode!, headers: response.headers, body }) + ) + } + ) + req.on('error', reject) + req.end() + } + ) + it('serves compiled assets with secure browser policies and denies other hosts/origins', async () => { + const page = await read('/') + expect(page.status).toBe(200) + expect(page.headers['content-security-policy']).toContain("connect-src 'self'") + expect(page.headers['permissions-policy']).toBe('xr-spatial-tracking=(self)') + expect((await read('/', '', 'GET', { Host: 'evil.example' })).status).toBe(403) + expect((await read('/', '', 'GET', { Origin: 'https://evil.example' })).status).toBe(403) + expect((await read('/src/main/index.ts')).status).toBe(404) + }) + it('exchanges the invitation over verified TLS, then revokes reads', async () => { + expect((await read('/api/snapshot')).status).toBe(401) + const paired = await read('/pair', bridge.invitation, 'POST') + expect(paired.status).toBe(200) + const { token } = JSON.parse(paired.body) as { token: string } + const snapshot = await read('/api/snapshot', token) + expect(snapshot.status).toBe(200) + expect(JSON.parse(snapshot.body).graph.linkCount).toBe(10) + bridge.revoke() + expect((await read('/api/snapshot', token)).status).toBe(401) + }) +}) diff --git a/apps/electron/src/main/spatial-server.ts b/apps/electron/src/main/spatial-server.ts new file mode 100644 index 000000000..27528462f --- /dev/null +++ b/apps/electron/src/main/spatial-server.ts @@ -0,0 +1,79 @@ +import type { createSpatialBridge } from './spatial-bridge' +import { createServer } from 'node:https' + +const lanAddress = (address: string) => { + const value = address.replace(/^::ffff:/, '') + return ( + value === '::1' || + /^(?:127\.|10\.|192\.168\.|169\.254\.|172\.(?:1[6-9]|2\d|3[01])\.)/.test(value) || + /^(?:fc|fd|fe80:)/i.test(value) + ) +} + +export function createSpatialServer({ + cert, + key, + origin, + assets, + bridge +}: { + cert: Buffer + key: Buffer + origin: string + assets: Map + bridge: ReturnType +}) { + const url = new URL(origin) + const next = createServer({ cert, key, minVersion: 'TLSv1.2' }, (request, reply) => { + void (async () => { + const incomingOrigin = request.headers.origin + if ( + !lanAddress(request.socket.remoteAddress ?? '') || + request.headers.host !== url.host || + (incomingOrigin && incomingOrigin !== url.origin) + ) { + reply.writeHead(403) + reply.end() + return + } + const path = new URL(request.url ?? '/', url.origin) + if (path.pathname === '/pair' || path.pathname.startsWith('/api/')) { + const result = await bridge.handle( + new Request(path, { + method: request.method, + headers: { + ...(incomingOrigin ? { origin: incomingOrigin } : {}), + authorization: request.headers.authorization ?? '' + } + }) + ) + reply.writeHead(result.status, Object.fromEntries(result.headers)) + reply.end(Buffer.from(await result.arrayBuffer())) + return + } + const asset = assets.get(path.pathname === '/' ? '/index.html' : path.pathname) + if (!asset || request.method !== 'GET') { + reply.writeHead(404) + reply.end() + return + } + reply.writeHead(200, { + 'Content-Type': asset.type, + 'Cache-Control': 'no-store', + 'X-Content-Type-Options': 'nosniff', + 'Referrer-Policy': 'no-referrer', + 'Permissions-Policy': 'xr-spatial-tracking=(self)', + 'Content-Security-Policy': + "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' blob:; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'" + }) + reply.end(asset.data) + })().catch(() => { + if (!reply.headersSent) reply.writeHead(503) + reply.end('Viewer unavailable') + }) + }) + next.requestTimeout = 150_000 + next.headersTimeout = 10_000 + + return next +} diff --git a/apps/electron/src/preload/index.ts b/apps/electron/src/preload/index.ts index 527220f93..8237c618a 100644 --- a/apps/electron/src/preload/index.ts +++ b/apps/electron/src/preload/index.ts @@ -33,6 +33,11 @@ contextBridge.exposeInMainWorld('xnet', { getRecoveryStatus: () => ipcRenderer.invoke('xnet:recovery:status'), libraryStatus: () => ipcRenderer.invoke('xnet:library:status'), libraryCards: (ids: string[]) => ipcRenderer.invoke('xnet:library:cards', { ids }), + spatialStatus: () => ipcRenderer.invoke('xnet:spatial:status'), + spatialStart: (input: { origin: string; ids: string[] }) => + ipcRenderer.invoke('xnet:spatial:start', input), + spatialStop: () => ipcRenderer.invoke('xnet:spatial:stop'), + spatialCopy: () => ipcRenderer.invoke('xnet:spatial:copy'), libraryGraph: () => ipcRenderer.invoke('xnet:library:graph'), libraryGraphDetail: (id: string) => ipcRenderer.invoke('xnet:library:graph-detail', { id }), libraryHelperStatus: () => ipcRenderer.invoke('xnet:library:helper-status'), @@ -632,6 +637,13 @@ export interface XNetAPI { }): Promise libraryGet(id: string): Promise libraryCards(ids: string[]): Promise<(LibrarySearchResult | null)[]> + spatialStatus(): Promise<{ url: string; expiresAt: number; count: number } | null> + spatialStart(input: { + origin: string + ids: string[] + }): Promise<{ url: string; expiresAt: number; count: number }> + spatialStop(): Promise + spatialCopy(): Promise libraryGraph(): Promise libraryGraphDetail(id: string): Promise libraryScan(): Promise diff --git a/apps/electron/src/renderer/components/LibraryGraphView.tsx b/apps/electron/src/renderer/components/LibraryGraphView.tsx index cdc59cb16..b61a3cb08 100644 --- a/apps/electron/src/renderer/components/LibraryGraphView.tsx +++ b/apps/electron/src/renderer/components/LibraryGraphView.tsx @@ -12,6 +12,7 @@ import { GraphGroups } from './library-graph/GraphGroups' import { GraphSearch } from './library-graph/GraphSearch' import { graphAdjacency, graphColor, relationKinds, selectGraph } from './library-graph/model' import { groupNames } from './library-graph/navigation' +import { SpatialPairing } from './library-graph/SpatialPairing' import { LibraryThumbnail } from './LibraryResourceCard' const button = @@ -386,6 +387,11 @@ export function LibraryGraphView({ {showGroups ? 'Hide groups' : 'Browse groups'} {groups.length ? ` (${groups.length})` : ''} + {visible && ( + node.kind === 'link').map((node) => node.id)} + /> + )} + {open && ( +
+

Explore on a headset

+

+ Pair a read-only viewer with the {ids.length.toLocaleString()} links in this graph view. + It can read their saved metadata, text and thumbnails for one hour. Private + conversations and personal notes are excluded. +

+

+ Use the Mac’s address on your private network and an HTTPS certificate already trusted + by both devices. Next, choose that certificate and its private key on this Mac. No + public tunnel is opened. +

+ + {error && ( +

+ {error} +

+ )} + {pairing ? ( + <> +

+ {pairing.count.toLocaleString()} links · expires{' '} + {new Date(pairing.expiresAt).toLocaleTimeString()}. The single-use pairing link is + on your clipboard; open it in the headset browser within five minutes. +

+ + + + ) : ( + + )} +

+ This is an experiment. Stationary browsing is available first; controller flight depends + on the headset’s measured input capabilities. Exit the immersive view to search or + change filters. +

+
+ )} + + ) +} diff --git a/site/src/content/docs/docs/architecture/decisions.mdx b/site/src/content/docs/docs/architecture/decisions.mdx index 3996d0ba8..6ea038614 100644 --- a/site/src/content/docs/docs/architecture/decisions.mdx +++ b/site/src/content/docs/docs/architecture/decisions.mdx @@ -1471,3 +1471,37 @@ shutdown. The method covers watch-dispatched work, not independent explicit scan **Tripwire:** A caller needs bounded cancellation, parallel scans become necessary, or any consumer disposes storage before awaiting the drain. Revisit ownership and cancellation semantics before changing the shutdown guarantee. + +## ADR-47: Pair spatial Library viewers with an ephemeral read capability + +**Status:** Accepted · 2026-10-06 · exploration 0468 + +**Context:** A headset browser cannot use Electron's preload. Exposing the general +local API or development server would give the viewer capabilities it does not need. + +**Decision:** The owner starts a separate HTTPS listener from the desktop graph, +explicitly choosing the visible saved links and a certificate already trusted by +both devices. Only private-network clients are accepted. The listener serves a +compiled browser viewer and a frozen, revision-matched graph/layout projection. +A random 256-bit invitation in the URL fragment can be redeemed once within five +minutes for an in-memory bearer capability. The capability expires one hour after +creation or immediately when the owner revokes access, the app exits, or the +listener stops. Credentials are never logged or persisted by the viewer. + +The only data routes are snapshot, bounded search, selected-resource details and +thumbnails resolved through those resources. No arbitrary blob IDs, conversations, +personal notes, identity material, SQL, writes or enrichment jobs are exposed. +Details are a bounded projection; longer saved source text or transcript text is +requested explicitly and limited to 100,000 characters. Reads check revocation +again after asynchronous storage access. Exact host and origin checks, no CORS, +no-store responses and a same-origin content policy constrain browser access. + +**Tradeoff:** This is an experimental, session-local transport, not an independent +synced Library or a supported durable public protocol. Users must provision trust +and address resolution on both devices. A copied capability authorizes its holder +until expiry or revocation; it is not a hardware-bound device credential. Revocation +stops future reads but cannot erase content already delivered to a device. + +**Tripwire:** We need internet access, persistent pairing, offline private content, +multiple readers, or native clients consuming a stable protocol. Revisit the +capability and ownership design before extending this experimental transport. From 0892a351ea368bbac0a8b17c0edd73428a3091ed Mon Sep 17 00:00:00 2001 From: xNet Test Date: Tue, 6 Oct 2026 10:29:05 -0700 Subject: [PATCH 3/4] feat(library): add experimental WebXR graph navigation - Provide searchable synthetic and paired snapshots with stable filters and evidence cards. - Add stationary overview, explicit travel, return history and guarded controller flight. - Bound labels, relationship lines and thumbnail loading; export hardware probe observations. - Document desktop verification and mark exploration 0468 partially implemented. - Keep physical device review and full-pose flight gates pending. Signed-off-by: xNet Test --- apps/electron/package.json | 17 +- apps/electron/spatial.vite.config.ts | 7 + apps/electron/src/spatial/README.md | 96 +++ apps/electron/src/spatial/adapter.ts | 59 ++ apps/electron/src/spatial/index.html | 122 ++++ apps/electron/src/spatial/main.ts | 361 ++++++++++++ apps/electron/src/spatial/panel.ts | 67 +++ apps/electron/src/spatial/probe.ts | 134 +++++ apps/electron/src/spatial/scene.ts | 551 ++++++++++++++++++ apps/electron/src/spatial/style.css | 302 ++++++++++ apps/electron/tsconfig.spatial.json | 9 + ...8_[-]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md} | 28 +- pnpm-lock.yaml | 3 + .../src/content/docs/docs/guides/electron.mdx | 30 + ...lore-a-private-spatial-library-previe.json | 8 + 15 files changed, 1781 insertions(+), 13 deletions(-) create mode 100644 apps/electron/spatial.vite.config.ts create mode 100644 apps/electron/src/spatial/README.md create mode 100644 apps/electron/src/spatial/adapter.ts create mode 100644 apps/electron/src/spatial/index.html create mode 100644 apps/electron/src/spatial/main.ts create mode 100644 apps/electron/src/spatial/panel.ts create mode 100644 apps/electron/src/spatial/probe.ts create mode 100644 apps/electron/src/spatial/scene.ts create mode 100644 apps/electron/src/spatial/style.css create mode 100644 apps/electron/tsconfig.spatial.json rename docs/explorations/{0468_[_]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md => 0468_[-]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md} (90%) create mode 100644 site/src/data/changelog/2026-10-06-explore-a-private-spatial-library-previe.json diff --git a/apps/electron/package.json b/apps/electron/package.json index 290e5ef94..9a5d8486f 100644 --- a/apps/electron/package.json +++ b/apps/electron/package.json @@ -10,15 +10,15 @@ "main": "./out/main/index.js", "type": "module", "scripts": { - "dev": "node scripts/rebuild-if-stale.mjs && node scripts/build-workspace-deps.mjs && node scripts/dev-launch.mjs concurrently -k -n hub,electron -c blue,green \"pnpm run dev:hub\" \"pnpm run dev:electron:raw\"", - "dev:electron": "node scripts/rebuild-if-stale.mjs && node scripts/build-workspace-deps.mjs && node scripts/dev-launch.mjs electron-vite dev", + "dev": "node scripts/rebuild-if-stale.mjs && node scripts/build-workspace-deps.mjs && pnpm spatial:build && node scripts/dev-launch.mjs concurrently -k -n hub,electron -c blue,green \"pnpm run dev:hub\" \"pnpm run dev:electron:raw\"", + "dev:electron": "node scripts/rebuild-if-stale.mjs && node scripts/build-workspace-deps.mjs && pnpm spatial:build && node scripts/dev-launch.mjs electron-vite dev", "dev:electron:raw": "electron-vite dev", "dev:hub": "HUB_AUTH=false HUB_STORAGE=memory node scripts/run-dev-hub.mjs", "dev:clean": "node scripts/dev-clean.mjs", "dev:scope": "node scripts/dev-scope.mjs", - "dev:user2": "node scripts/rebuild-if-stale.mjs && node scripts/build-workspace-deps.mjs && cross-env XNET_PROFILE=user2 VITE_PORT=5174 ELECTRON_CDP_PORT=9224 node scripts/dev-launch.mjs electron-vite dev", - "dev:both": "node scripts/rebuild-if-stale.mjs && node scripts/build-workspace-deps.mjs && concurrently -k -n hub,user1,user2 -c blue,green,yellow \"pnpm run dev:hub\" \"pnpm run dev:electron\" \"sleep 3 && pnpm run dev:user2\"", - "build": "electron-vite build", + "dev:user2": "node scripts/rebuild-if-stale.mjs && node scripts/build-workspace-deps.mjs && pnpm spatial:build && cross-env XNET_PROFILE=user2 VITE_PORT=5174 ELECTRON_CDP_PORT=9224 node scripts/dev-launch.mjs electron-vite dev", + "dev:both": "node scripts/rebuild-if-stale.mjs && node scripts/build-workspace-deps.mjs && pnpm spatial:build && concurrently -k -n hub,user1,user2 -c blue,green,yellow \"pnpm run dev:hub\" \"pnpm run dev:electron\" \"sleep 3 && pnpm run dev:user2\"", + "build": "pnpm spatial:build && electron-vite build", "deps:electron": "pnpm dlx @electron/rebuild -f -w better-sqlite3,usearch,sharp", "deps:node": "pnpm rebuild better-sqlite3 usearch sharp && node scripts/rebuild-if-stale.mjs --invalidate", "preview": "electron-vite preview", @@ -26,7 +26,11 @@ "dist:mac:self-signed": "./scripts/build-macos-self-signed.sh", "dist:mac:self-signed:arm64": "./scripts/build-macos-self-signed.sh arm64", "test": "pnpm run deps:node && vitest run", - "test:watch": "vitest" + "test:watch": "vitest", + "spatial:dev": "vite --config spatial.vite.config.ts", + "spatial:build": "vite build --config spatial.vite.config.ts", + "spatial:typecheck": "tsc -p tsconfig.spatial.json", + "typecheck": "pnpm spatial:typecheck && tsc -p tsconfig.node.json --noEmit --incremental false --composite false" }, "dependencies": { "@xnetjs/canvas": "workspace:*", @@ -71,6 +75,7 @@ "@types/react": "^18.2.0", "@types/react-dom": "^18.2.0", "@types/three": "0.186.0", + "@types/webxr": "0.5.24", "@types/ws": "^8.5.10", "@vitejs/plugin-react": "^4.3.0", "autoprefixer": "^10.4.20", diff --git a/apps/electron/spatial.vite.config.ts b/apps/electron/spatial.vite.config.ts new file mode 100644 index 000000000..26bac833a --- /dev/null +++ b/apps/electron/spatial.vite.config.ts @@ -0,0 +1,7 @@ +import { resolve } from 'node:path' +import { defineConfig } from 'vite' +export default defineConfig({ + root: resolve(__dirname, 'src/spatial'), + build: { outDir: resolve(__dirname, 'out/spatial'), emptyOutDir: true, target: 'es2022' }, + server: { host: '127.0.0.1', port: 5189, strictPort: true } +}) diff --git a/apps/electron/src/spatial/README.md b/apps/electron/src/spatial/README.md new file mode 100644 index 000000000..5d94a2137 --- /dev/null +++ b/apps/electron/src/spatial/README.md @@ -0,0 +1,96 @@ +# Spatial Library experiment + +The standalone browser viewer shares the desktop Library's graph types, filters, +search and initial layout. It does not import Electron preload or create a second +Library database. The desktop graph's **Spatial viewer** control pairs a frozen +projection of the links in the current view. + +```mermaid +flowchart LR + Desktop[Desktop graph selection] --> Pair[Owner starts private HTTPS listener] + Pair --> Capability[One-use invitation → one-hour read capability] + Capability --> Snapshot[Matching graph and metre-scale layout revision] + Snapshot --> Flat[Browser search and filters] + Snapshot --> XR[Stationary WebXR overview] + XR --> Input[Observed tracked pose and analog range] + Input --> Flight[Explicit arm → released trigger → acceleration] + Flight --> Stop[Brake / inspect / tracking loss / suspension] +``` + +## Try the synthetic probe + +From the repository root: + +```sh +pnpm --filter xnet-desktop spatial:dev +pnpm --filter xnet-desktop spatial:build +pnpm --filter xnet-desktop spatial:typecheck +pnpm exec vitest run --project electron apps/electron/src/shared/library-flight.test.ts apps/electron/src/shared/spatial-library.test.ts apps/electron/src/main/spatial-bridge.test.ts +``` + +The development viewer is loopback-only. It offers 1,000, 10,000 and 56,000 +synthetic links. It is a separate browser entry, not the Electron renderer in a +tab. Do not expose a development server or debugger to the headset. + +## Pair a private Library snapshot + +1. Build or start the desktop app. The viewer assets are built with it. +2. Connect both devices to a trusted private network. Provision a valid HTTPS + certificate for the Mac's address and establish trust on both devices through + the operating systems. Do not bypass certificate warnings. The headset's + `localhost` is not the Mac. +3. In the desktop Library graph, choose filters for the links to expose and open + **Spatial viewer**. Supply `https://your-mac.local:8443`, then choose the matching + certificate and private key. The key stays on the Mac. +4. Open the copied, single-use link in headset Safari within five minutes. The + viewer removes its fragment and holds the exchanged capability only in memory. + Reloading loses the capability; create a new pairing instead of saving it. +5. Use **Revoke access** on the Mac when done. Expiry or quitting also stops the + listener. See ADR-47 for scope, security boundaries and limitations. + +Search addresses every paired link and group, even when relationship lines are +limited. Selecting a result highlights it and brakes without travelling. Choose +**Travel there** separately inside the session. Home and Back retain a bounded +return trail. Exit / search ends immersion so the browser can accept normal text +input; DOM Overlay is not assumed. The same desktop camera and target are restored. + +The scene is stable during immersion. Graph changes are applied only outside the +session. Filtering retains the original resource coordinates. A new revision +resets saved travel positions. One image and card, eight group labels, 20,000 +background lines and 200 selected relationships are provisional limits, not +measured headset budgets. Picking runs on deliberate selection, not every frame. + +## Hardware release review — owned by Chris + +No physical headset validation is recorded yet. Advanced full-pose flight and the +WebXR-versus-native decision remain blocked on these observations, as required by +exploration 0468. Button-only input does not enable flight. + +For each fixture size, enter the stationary overview, then record actual headset, +visionOS, Safari and controller firmware versions in **Hardware observations**. +Pair Sense controllers in system settings. Translate and rotate each independently, +then slowly sweep the trigger from released to fully pressed. The report records +input profiles, handedness, ray mode, grip availability, non-emulated position, +observed position/orientation change and analog range. Those observations gate the +experimental Arm button; they are not a substitute for the review below. + +| Review | Required evidence | +| ---------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| Input | Full position and rotation, trigger range, disconnect/reconnect, hands-only pinch and coexistence of transient input sources | +| Navigation | Find three known resources, inspect coverage and thumbnails, travel explicitly, return home/back | +| Safety | Held-trigger entry, tracking loss, invalid pose, hidden session and reconnect never resume without arming and release | +| Selection | Flight-hand trigger does not select graph nodes during flight; inspection and the squeeze brake stop motion | +| Comfort | Chris's proposed 20-minute browsing trial, including a useful stationary alternative | +| Scale | Delivered cadence and p95/p99 intervals at all three sizes; sustained behavior, actual memory measurement and chosen budgets | +| Access | Trusted HTTPS on the physical headset, pairing, expiry/revocation and loss of network; visible errors and unchanged source Library | +| Lifecycle | Repeated entry/exit restores the flat camera and leaves no animation loop or input listeners behind | + +Download the probe report after each run. Timing percentiles use the last 4,096 +frame intervals; frame counts and interruptions are cumulative. The report includes +Three.js geometry/texture counts, not process memory or GPU duration. Declare a +frame budget from the observed cadence, then measure actual device memory with +available platform tools. Do not report desktop results as stereo performance. + +After the default interaction passes hardware review, decide whether to implement +the advanced full-pose clutch. If Safari lacks tracked Sense input, retain the +hands-based viewer and review a native visionOS client against the same tasks. diff --git a/apps/electron/src/spatial/adapter.ts b/apps/electron/src/spatial/adapter.ts new file mode 100644 index 000000000..1e27be8bd --- /dev/null +++ b/apps/electron/src/spatial/adapter.ts @@ -0,0 +1,59 @@ +import type { + SpatialDetail, + SpatialLibraryAdapter, + SpatialSnapshot +} from '../shared/spatial-library' +import { validateSpatialSnapshot } from '../shared/spatial-library' + +/** Same-origin TLS only; capabilities live in memory, never storage or query strings. */ +export async function pairedAdapter(invitation: string): Promise { + if (location.protocol !== 'https:') + throw new Error('Pairing requires trusted HTTPS on both devices.') + let token = '' + const paired = await fetch('/pair', { + method: 'POST', + headers: { Authorization: `Bearer ${invitation}` }, + cache: 'no-store', + credentials: 'omit', + redirect: 'error' + }) + if (!paired.ok) + throw new Error('This invitation expired or was already used. Create a new pairing on the Mac.') + const grant = (await paired.json()) as { token: string } + token = grant.token + async function read(path: string, signal: AbortSignal) { + const response = await fetch(path, { + signal, + headers: { Authorization: `Bearer ${token}` }, + cache: 'no-store', + credentials: 'omit', + redirect: 'error' + }) + if (response.status === 401) + throw new Error('Pairing expired or was revoked. Reconnect from the Mac.') + if (response.status === 404) return null + if (!response.ok) throw new Error(`The Mac could not read this item (${response.status}).`) + return response + } + return { + async snapshot(signal) { + const response = await read('/api/snapshot', signal) + if (!response) throw new Error('The paired graph is unavailable.') + return validateSpatialSnapshot((await response.json()) as SpatialSnapshot) + }, + async detail(id, signal, text = false) { + const response = await read( + `/api/detail/${encodeURIComponent(id)}${text ? '?text=1' : ''}`, + signal + ) + return response ? ((await response.json()) as SpatialDetail) : null + }, + async thumbnail(id, signal) { + const response = await read(`/api/thumbnail/${encodeURIComponent(id)}`, signal) + return response ? response.blob() : null + }, + close() { + token = '' + } + } +} diff --git a/apps/electron/src/spatial/index.html b/apps/electron/src/spatial/index.html new file mode 100644 index 000000000..389bff26e --- /dev/null +++ b/apps/electron/src/spatial/index.html @@ -0,0 +1,122 @@ + + + + + + + xNet · Spatial Library + + +
+
+ xNet / Spatial LibraryEXPERIMENT 0468 +
+
+ +
+
+
+ Preparing graph… +
+

Loading the synthetic graph…

+ +
+ Drag to orbit · Scroll to zoom · Select to inspect
Edges: imported membership, + metadata or hashtags. Proximity is not evidence. +
+
+ +
+
+ +
+
+ + + diff --git a/apps/electron/src/spatial/main.ts b/apps/electron/src/spatial/main.ts new file mode 100644 index 000000000..61956ec0e --- /dev/null +++ b/apps/electron/src/spatial/main.ts @@ -0,0 +1,361 @@ +import type { SpatialLibraryAdapter, SpatialSnapshot } from '../shared/spatial-library' +import { selectGraph, relationKinds } from '../shared/library-graph/model' +import { graphGroups, graphSearchIndex, searchGraph } from '../shared/library-graph/navigation' +import { projectSpatialSnapshot, validateSpatialSnapshot } from '../shared/spatial-library' +import { pairedAdapter } from './adapter' +import { fixtureAdapter } from './fixtures' +import { createSpatialScene } from './scene' +import './style.css' + +const element = (id: string) => + document.getElementById(id) as T +const message = (error: unknown) => (error instanceof Error ? error.message : String(error)) +const fail = (error: unknown) => { + element('error').hidden = false + element('error').textContent = message(error) +} +let adapter: SpatialLibraryAdapter = fixtureAdapter(1000) +let snapshot: SpatialSnapshot | null = null +let selected: string | null = null +let groups: string[] = [], + focus: string | null = null +let current = new AbortController(), + loading = new AbortController() +let generation = 0, + immersive = false +let thumbnailUrl: string | null = null +let image: ImageBitmap | null = null +let paired = false +let searchIndex: ReturnType = [] +const clearImage = () => { + if (thumbnailUrl) URL.revokeObjectURL(thumbnailUrl) + thumbnailUrl = null + image?.close() + image = null + element('thumbnail').hidden = true +} +const scene = (() => { + try { + return createSpatialScene(element('canvas'), { + select(id) { + void inspect(id) + }, + status(text) { + element('status').textContent = text + }, + session(active) { + immersive = active + element('enter').hidden = active + element('exit').hidden = !active + element('travel').disabled = !active || !selected + for (const id of ['fixture', 'platform', 'clear', 'neighborhood', 'hand']) + (element(id) as HTMLButtonElement).disabled = active || (id === 'fixture' && paired) + } + }) + } catch (error) { + fail(`The graph could not start: ${message(error)}`) + element('enter').disabled = true + throw error + } +})() +async function inspect(id: string) { + current.abort() + current = new AbortController() + const signal = current.signal + clearImage() + selected = id + element('error').hidden = true + element('saved-text').hidden = true + element('travel').disabled = !immersive + element('neighborhood').disabled = immersive + element('text').disabled = true + const index = snapshot?.graph.nodes.findIndex((node) => node.id === id) ?? -1 + const node = snapshot?.graph.nodes[index] + if (!node || !snapshot) return + element('title').textContent = node.label + const edges = snapshot.graph.edges.filter( + (edge) => edge.source === index || edge.target === index + ) + const evidence = [...new Set(edges.map((edge) => `${edge.kind} · ${edge.evidence}`))].join('; ') + element('evidence').textContent = `${edges.length.toLocaleString()} connections. ${evidence}` + element('coverage').textContent = '' + if (node.kind !== 'link') { + element('description').textContent = + 'An imported group of saved links. Use Show neighborhood to explore its members.' + scene.detail([ + node.label, + `${edges.length} connections`, + evidence, + 'Exit / search to filter this neighborhood.' + ]) + return + } + element('description').textContent = 'Reading saved details…' + try { + const detail = await adapter.detail(id, signal) + if (signal.aborted) return + if (!detail) + throw new Error( + 'This resource is no longer available. Its graph selection has been retained.' + ) + const coverage = Object.entries(detail.coverage) + .map( + ([field, value]) => `${field}: ${value.state}${value.reason ? ` (${value.reason})` : ''}` + ) + .join(' · ') + const transcript = detail.transcript + ? `${detail.transcript.cues.toLocaleString()} ${detail.transcript.language} transcript cues · ${detail.transcript.source}` + : 'No spoken transcript saved. Written descriptions are not transcripts.' + element('title').textContent = detail.title + element('description').textContent = detail.description || 'No saved description.' + element('coverage').textContent = `${coverage}\n${transcript}` + element('text').disabled = false + const lines = [ + detail.title, + `${detail.platform} · ${detail.author}`, + detail.description.slice(0, 260) || 'Description unavailable', + evidence, + transcript, + coverage.slice(0, 180) + ] + scene.detail(lines) + if (detail.thumbnail) { + const blob = await adapter.thumbnail(id, signal) + if (signal.aborted) return + if (!blob) { + element('coverage').textContent += ' Thumbnail unavailable.' + return + } + const decoded = await createImageBitmap(blob) + if (signal.aborted) { + decoded.close() + return + } + image = decoded + thumbnailUrl = URL.createObjectURL(blob) + element('thumbnail').src = thumbnailUrl + element('thumbnail').hidden = false + scene.detail(lines, decoded) + } + } catch (error) { + if (!signal.aborted) { + fail(error) + element('description').textContent = 'Saved details unavailable.' + scene.detail([node.label, 'Saved details unavailable.', message(error)]) + } + } +} +function results() { + if (!snapshot) return + const found = searchGraph(searchIndex, element('search').value, 30) + element('matches').textContent = + `${found.total.toLocaleString()} matches across all paired links and groups` + element('results').replaceChildren( + ...found.items.map(({ node, count }) => { + const button = document.createElement('button') + button.textContent = node.label + const small = document.createElement('small') + small.textContent = `${node.platform || node.kind} · ${count} connections` + button.append(small) + button.onclick = () => { + if (!snapshot) return + if (groups.length || focus || element('platform').value) { + groups = [] + focus = null + element('platform').value = '' + apply() + } + scene.select(node.id) + } + return button + }) + ) +} +function groupList() { + if (!snapshot) return + const query = element('group-search').value.toLocaleLowerCase() + const found = graphGroups(snapshot.graph, element('platform').value).filter( + ({ node }) => node.label.toLocaleLowerCase().includes(query) + ) + element('groups').replaceChildren( + ...found.slice(0, 40).map(({ node, count }) => { + const button = document.createElement('button') + button.textContent = `${node.label} · ${count} links` + button.setAttribute('aria-pressed', String(groups.includes(node.id))) + button.onclick = () => { + if (immersive) return + groups = groups.includes(node.id) + ? groups.filter((id) => id !== node.id) + : [...groups, node.id] + focus = null + apply() + } + return button + }) + ) + element('filters').textContent = + `${groups.length} selected groups (match any). ${found.length.toLocaleString()} groups; showing up to 40. Refine to find any group.` +} +function apply() { + if (!snapshot || immersive) return + const graph = selectGraph( + snapshot.graph, + element('platform').value, + relationKinds, + focus, + groups + ) + scene.setSnapshot(projectSpatialSnapshot(snapshot, graph)) + element('counts').textContent = + `${graph.linkCount.toLocaleString()} / ${snapshot.graph.linkCount.toLocaleString()} links · ${graph.edges.length.toLocaleString()} relationships` + element('status').textContent = + `Stable snapshot · ${Math.min(graph.edges.length, 20_000).toLocaleString()} relationship lines shown. All links remain searchable.` + groupList() +} +async function load() { + loading.abort() + loading = new AbortController() + current.abort() + const signal = loading.signal + const version = ++generation + try { + const value = validateSpatialSnapshot(await adapter.snapshot(signal)) + if (signal.aborted || version !== generation) return + snapshot = value + searchIndex = graphSearchIndex(value.graph) + groups = [] + focus = null + const select = element('platform') + select.replaceChildren() + for (const platform of [ + '', + ...new Set( + value.graph.nodes.filter((node) => node.kind === 'link').map((node) => node.platform) + ) + ]) { + const option = document.createElement('option') + option.value = platform + option.textContent = platform || 'All platforms' + select.append(option) + } + apply() + results() + } catch (error) { + if (!signal.aborted) fail(error) + } +} +element('search').oninput = results +element('group-search').oninput = groupList +element('platform').onchange = () => { + focus = null + apply() +} +element('fixture').onchange = async () => { + if (paired || immersive) return + adapter.close() + adapter = fixtureAdapter(Number(element('fixture').value)) + selected = null + await load() +} +element('clear').onclick = () => { + groups = [] + focus = null + element('platform').value = '' + apply() +} +element('neighborhood').onclick = () => { + if (!immersive) { + focus = selected + apply() + } +} +element('home').onclick = scene.home +element('back').onclick = scene.back +element('travel').onclick = scene.destination +element('brake').onclick = scene.brake +element('hand').onchange = () => + scene.setHand(element('hand').value === 'left' ? 'left' : 'right') +element('enter').onclick = () => { + element('enter').disabled = true + void scene + .enter() + .catch(fail) + .finally(() => { + element('enter').disabled = false + }) +} +element('exit').onclick = () => { + void scene.exit().catch(fail) +} +element('text').onclick = async () => { + if (!selected) return + scene.brake() + const id = selected, + signal = current.signal + try { + const detail = await adapter.detail(id, signal, true) + if (!signal.aborted && id === selected) { + element('saved-text').textContent = + detail?.text || 'No saved text available. Local audio transcription is not connected.' + element('saved-text').hidden = false + } + } catch (error) { + if (!signal.aborted) fail(error) + } +} +element('report').onclick = () => { + const report = { + recordedAt: new Date().toISOString(), + deviceVersions: element('versions').value, + ...scene.report() + } + const url = URL.createObjectURL( + new Blob([JSON.stringify(report, null, 2)], { type: 'application/json' }) + ) + const link = document.createElement('a') + link.href = url + link.download = 'xnet-spatial-probe.json' + link.click() + setTimeout(() => URL.revokeObjectURL(url), 1000) +} +const invitation = new URLSearchParams(location.hash.slice(1)).get('pair') +if (invitation) { + history.replaceState(null, '', location.pathname) + try { + adapter.close() + adapter = await pairedAdapter(invitation) + paired = true + element('fixture').disabled = true + element('fixture').options[0].textContent = 'Paired Library · read only' + element('connection').textContent = + 'Private connection to your Mac. This snapshot expires within one hour; revoke it on the Mac at any time.' + } catch (error) { + fail(error) + element('connection').textContent = 'Pairing failed. Synthetic data is shown.' + } +} +await load() +try { + const supported = + isSecureContext && !!navigator.xr && (await navigator.xr.isSessionSupported('immersive-vr')) + element('enter').disabled = !supported + element('enter').textContent = supported + ? 'Enter stationary overview' + : 'Immersive view unavailable' + element('support').textContent = supported + ? 'WebXR session support detected. Controller tracking is tested inside the session.' + : 'Use a supported headset browser over trusted HTTPS. Desktop search and inspection remain available.' +} catch (error) { + fail(error) +} +window.addEventListener( + 'pagehide', + () => { + current.abort() + loading.abort() + adapter.close() + clearImage() + void scene.dispose().catch(fail) + }, + { once: true } +) diff --git a/apps/electron/src/spatial/panel.ts b/apps/electron/src/spatial/panel.ts new file mode 100644 index 000000000..4aaee1f3f --- /dev/null +++ b/apps/electron/src/spatial/panel.ts @@ -0,0 +1,67 @@ +import { CanvasTexture, Mesh, MeshBasicMaterial, PlaneGeometry, SRGBColorSpace } from 'three' + +/** One bounded scene-resident card; HTML controls are not visible in immersive WebXR. */ +export function spatialPanel(width = 1.2, height = 0.75) { + const canvas = document.createElement('canvas') + canvas.width = 1024 + canvas.height = Math.round((1024 * height) / width) + const context = canvas.getContext('2d')! + const texture = new CanvasTexture(canvas) + texture.colorSpace = SRGBColorSpace + const material = new MeshBasicMaterial({ map: texture, depthTest: false }) + const mesh = new Mesh(new PlaneGeometry(width, height), material) + mesh.renderOrder = 10 + return { + mesh, + draw(lines: string[], image?: CanvasImageSource) { + context.fillStyle = '#101c30' + context.fillRect(0, 0, canvas.width, canvas.height) + context.strokeStyle = '#56779f' + context.strokeRect(2, 2, canvas.width - 4, canvas.height - 4) + context.fillStyle = '#f1f5f9' + const fontSize = lines.length === 1 ? 60 : 28 + context.font = `${fontSize}px system-ui` + let y = lines.length === 1 ? canvas.height / 2 + 18 : 48 + for (const text of lines) { + const words = text.split(/\s+/u) + let line = '' + for (const word of words) { + if (context.measureText(line + word).width > canvas.width - 60) { + context.fillText(line, 28, y) + y += 36 + line = '' + } + line += `${word} ` + if (y > canvas.height - 40) break + } + if (y > canvas.height - 40) break + context.fillText(line, 28, y) + y += 42 + } + if (image) context.drawImage(image, canvas.width - 244, canvas.height - 154, 216, 126) + texture.needsUpdate = true + }, + buttons(labels: string[], caption: string) { + context.fillStyle = '#101c30' + context.fillRect(0, 0, canvas.width, canvas.height) + context.font = '28px system-ui' + context.textAlign = 'center' + const cell = canvas.width / labels.length + for (const [i, label] of labels.entries()) { + context.strokeStyle = '#56779f' + context.strokeRect(i * cell + 2, 2, cell - 4, canvas.height * 0.62) + context.fillStyle = '#f1f5f9' + context.fillText(label, (i + 0.5) * cell, canvas.height * 0.34) + } + context.fillStyle = '#a8b8ce' + context.fillText(caption, canvas.width / 2, canvas.height * 0.86) + context.textAlign = 'start' + texture.needsUpdate = true + }, + dispose() { + mesh.geometry.dispose() + material.dispose() + texture.dispose() + } + } +} diff --git a/apps/electron/src/spatial/probe.ts b/apps/electron/src/spatial/probe.ts new file mode 100644 index 000000000..fd553eb7e --- /dev/null +++ b/apps/electron/src/spatial/probe.ts @@ -0,0 +1,134 @@ +export type InputObservation = { + id: number + profiles: string[] + hand: string + ray: string + mapping: string + grip: boolean + positionTracked: boolean + positionChanged: boolean + rotationChanged: boolean + triggerMin: number + triggerMax: number + analogObserved: boolean + selections: number + connected: boolean +} +export function createProbe() { + const observations = new Map() + const origins = new Map() + const intervals: number[] = [] + let previous = 0 + let frames = 0 + let interruptions = 0 + let features: string[] = [] + const observe = (source: XRInputSource) => { + let item = observations.get(source) + if (!item) { + if (observations.size >= 32) { + const oldest = [...observations].find(([, value]) => !value.connected) + if (oldest) { + observations.delete(oldest[0]) + origins.delete(oldest[0]) + } + } + item = { + id: frames + observations.size, + profiles: [...source.profiles], + hand: source.handedness, + ray: source.targetRayMode, + mapping: source.gamepad?.mapping ?? '', + grip: !!source.gripSpace, + positionTracked: false, + positionChanged: false, + rotationChanged: false, + triggerMin: 1, + triggerMax: 0, + analogObserved: false, + selections: 0, + connected: true + } + observations.set(source, item) + } + return item + } + return { + start(session: XRSession) { + features = [...(session.enabledFeatures ?? [])] + previous = 0 + }, + interrupt() { + previous = 0 + interruptions++ + }, + select(source: XRInputSource) { + observe(source).selections++ + }, + frame(time: number, frame: XRFrame, space: XRReferenceSpace) { + frames++ + if (previous && time > previous) { + intervals.push(time - previous) + if (intervals.length > 4096) intervals.shift() + } + previous = time + const sources = new Set(frame.session.inputSources) + for (const [source, item] of observations) item.connected = sources.has(source) + for (const source of sources) { + const item = observe(source) + item.connected = true + const pose = source.gripSpace ? frame.getPose(source.gripSpace, space) : null + if (pose && !pose.emulatedPosition) { + item.positionTracked = true + const p = pose.transform.position, + q = pose.transform.orientation + const values = [p.x, p.y, p.z, q.x, q.y, q.z, q.w] + const first = origins.get(source) ?? values + origins.set(source, first) + item.positionChanged ||= + Math.hypot(...values.slice(0, 3).map((v, i) => v - first[i])) > 0.02 + item.rotationChanged ||= + Math.abs(values.slice(3).reduce((dot, v, i) => dot + v * first[i + 3], 0)) < 0.995 + } + const trigger = source.gamepad?.buttons[0]?.value + if (trigger !== undefined && Number.isFinite(trigger)) { + item.triggerMin = Math.min(item.triggerMin, trigger) + item.triggerMax = Math.max(item.triggerMax, trigger) + item.analogObserved ||= trigger > 0.1 && trigger < 0.9 + } + } + }, + flightReady(source: XRInputSource) { + const item = observations.get(source) + return ( + !!item && + item.positionChanged && + item.rotationChanged && + item.analogObserved && + item.triggerMin < 0.1 && + item.triggerMax > 0.9 && + item.mapping === 'xr-standard' + ) + }, + report() { + const sorted = [...intervals].sort((a, b) => a - b) + const percentile = (p: number) => sorted[Math.floor((sorted.length - 1) * p)] ?? null + const median = percentile(0.5) + return { + userAgent: navigator.userAgent, + features, + frames, + interruptions, + cadenceHz: median ? 1000 / median : null, + p95Ms: percentile(0.95), + p99Ms: percentile(0.99), + intervalsAboveOneAndHalfMedian: median + ? intervals.filter((n) => n > median * 1.5).length + : null, + samples: intervals.length, + input: [...observations.values()], + limitations: + 'Pose changes and analog samples are observations, not a hardware or comfort certification. GPU and total process memory are not measured.' + } + } + } +} diff --git a/apps/electron/src/spatial/scene.ts b/apps/electron/src/spatial/scene.ts new file mode 100644 index 000000000..f02b0e463 --- /dev/null +++ b/apps/electron/src/spatial/scene.ts @@ -0,0 +1,551 @@ +import type { Vec3, TravelState } from '../shared/library-flight' +import type { SpatialSnapshot } from '../shared/spatial-library' +import { + BufferAttribute, + BufferGeometry, + Color, + Group, + LineBasicMaterial, + LineSegments, + Matrix4, + PerspectiveCamera, + Points, + PointsMaterial, + Quaternion, + Raycaster, + Scene, + SphereGeometry, + Mesh, + MeshBasicMaterial, + Vector2, + Vector3, + WebGLRenderer +} from 'three' +import { OrbitControls } from 'three/addons/controls/OrbitControls.js' +import { + brakeFlight, + initialFlight, + stepFlight, + travelTo, + travelBack +} from '../shared/library-flight' +import { graphAdjacency, graphColor } from '../shared/library-graph/model' +import { spatialPanel } from './panel' +import { createProbe } from './probe' + +export function createSpatialScene( + host: HTMLElement, + callbacks: { + select(id: string): void + status(message: string): void + session(active: boolean): void + } +) { + const renderer = new WebGLRenderer({ antialias: true }) + renderer.setClearColor('#080f1d') + renderer.setPixelRatio(Math.min(devicePixelRatio, 2)) + renderer.xr.enabled = true + renderer.xr.setReferenceSpaceType('local-floor') + const canvas = renderer.domElement + canvas.tabIndex = 0 + canvas.setAttribute('aria-label', 'Spatial Library graph. Search the list to select a resource.') + host.append(canvas) + const world = new Scene() + const rig = new Group() + const camera = new PerspectiveCamera(55, 1, 0.02, 100) + camera.position.set(0, 1.6, 2) + rig.add(camera) + world.add(rig) + const controls = new OrbitControls(camera, canvas) + controls.target.set(0, 1.4, -3) + controls.minDistance = 0.1 + controls.maxDistance = 30 + const root = new Group() + world.add(root) + const card = spatialPanel() + card.mesh.visible = false + world.add(card.mesh) + const navigation = spatialPanel(1.5, 0.25) + navigation.buttons( + ['Home', 'Back', 'Travel there', 'Brake', 'Exit / search'], + 'Stationary overview · Select a link to inspect' + ) + navigation.mesh.visible = false + world.add(navigation.mesh) + const flightButton = spatialPanel(0.8, 0.16) + flightButton.draw(['Arm experimental flight']) + flightButton.mesh.visible = false + world.add(flightButton.mesh) + const labels: ReturnType[] = [] + const marker = new Mesh( + new SphereGeometry(0.025, 12, 8), + new MeshBasicMaterial({ color: '#ffffff', wireframe: true }) + ) + marker.visible = false + world.add(marker) + const probe = createProbe() + const raycaster = new Raycaster() + raycaster.params.Points = { threshold: 0.025 } + let snapshot: SpatialSnapshot | null = null + let points: Points | null = null + let edges: LineSegments | null = null + let highlight: LineSegments | null = null + let selected: string | null = null + let adjacency: number[][] = [] + let flight = initialFlight() + let travel: TravelState = { revision: '', position: [0, 0, 0], trail: [] } + let hand: XRHandedness = 'right' + let session: XRSession | null = null + let arm = false + let lastTime = 0 + let disposed = false + let frameHandle = 0 + let desktop: { position: Vector3; quaternion: Quaternion; target: Vector3 } | null = null + let sourceInUse: XRInputSource | null = null + const matrix = new Matrix4(), + vector = new Vector3() + const halt = () => { + flight = brakeFlight(flight) + arm = false + sourceInUse = null + } + const render = () => { + frameHandle = 0 + if (!session && !disposed) { + controls.update() + renderer.render(world, camera) + } + } + const invalidate = () => { + if (!frameHandle && !session && !disposed) frameHandle = requestAnimationFrame(render) + } + const panelAtViewer = () => { + rig.updateMatrixWorld(true) + if (renderer.xr.isPresenting) renderer.xr.updateCamera(camera) + const viewer = renderer.xr.isPresenting ? renderer.xr.getCamera() : camera + viewer.updateWorldMatrix(true, false) + const origin = viewer.getWorldPosition(new Vector3()) + const forward = viewer.getWorldDirection(new Vector3()) + forward.y = 0 + if (forward.lengthSq() < 0.01) forward.set(0, 0, -1) + forward.normalize() + navigation.mesh.position.copy(origin).addScaledVector(forward, 1.6) + navigation.mesh.position.y = origin.y - 0.5 + navigation.mesh.lookAt(origin) + flightButton.mesh.position.copy(navigation.mesh.position) + flightButton.mesh.position.y -= 0.24 + flightButton.mesh.lookAt(origin) + } + function move(position: Vec3) { + halt() + flight = initialFlight(position) + rig.position.fromArray(position) + panelAtViewer() + invalidate() + } + const home = () => { + travel = travelTo({ ...travel, position: flight.position }, [0, 0, 0]) + move(travel.position) + } + const back = () => { + travel = travelBack({ ...travel, position: flight.position }) + move(travel.position) + } + function destination() { + if (!snapshot || !selected || !session) return + const index = snapshot.graph.nodes.findIndex((node) => node.id === selected) + if (index < 0) return + const point = new Vector3().fromArray(snapshot.positions, index * 3) + // Move only after the explicit action, preserving the runtime's tracked head offset. + const head = renderer.xr.getCamera().getWorldPosition(new Vector3()).sub(rig.position) + travel = travelTo({ ...travel, position: flight.position }, [ + point.x - head.x, + point.y - head.y, + point.z + 0.8 - head.z + ]) + move(travel.position) + } + const disposeGeometry = () => { + for (const object of [points, edges, highlight]) + if (object) { + root.remove(object) + object.geometry.dispose() + const materials = Array.isArray(object.material) ? object.material : [object.material] + materials.forEach((material) => material.dispose()) + } + for (const label of labels) { + world.remove(label.mesh) + label.dispose() + } + labels.length = 0 + } + const select = (id: string) => { + halt() + selected = id + if (!snapshot) return + const index = snapshot.graph.nodes.findIndex((node) => node.id === id) + if (index < 0) { + marker.visible = false + card.mesh.visible = false + return + } + marker.position.fromArray(snapshot.positions, index * 3) + marker.visible = true + card.mesh.position.copy(marker.position).add(new Vector3(0.7, 0.35, 0)) + card.mesh.lookAt((session ? renderer.xr.getCamera() : camera).getWorldPosition(new Vector3())) + card.mesh.visible = true + const node = snapshot.graph.nodes[index] + const evidence = [ + ...new Set( + snapshot.graph.edges + .filter((e) => e.source === index || e.target === index) + .map((e) => `${e.kind}: ${e.evidence}`) + ) + ] + card.draw([ + node.label, + `${node.platform || node.kind} · ${adjacency[index].length} connections`, + evidence.join(' · '), + 'Loading saved details…' + ]) + if (highlight) { + root.remove(highlight) + highlight.geometry.dispose() + ;(highlight.material as LineBasicMaterial).dispose() + } + const lines = adjacency[index] + .slice(0, 200) + .flatMap((other) => [ + ...snapshot!.positions.slice(index * 3, index * 3 + 3), + ...snapshot!.positions.slice(other * 3, other * 3 + 3) + ]) + highlight = new LineSegments( + new BufferGeometry().setAttribute( + 'position', + new BufferAttribute(new Float32Array(lines), 3) + ), + new LineBasicMaterial({ color: '#ffffff' }) + ) + root.add(highlight) + panelAtViewer() + callbacks.select(id) + invalidate() + } + function pick() { + if (!snapshot || !points) return + // Intentional selection only, never a full-cloud per-frame hover raycast. + const hit = raycaster.intersectObject(points)[0] + if (hit?.index !== undefined) select(snapshot.graph.nodes[hit.index].id) + } + let pointerStart = [0, 0] + const pointerDown = (event: PointerEvent) => { + pointerStart = [event.clientX, event.clientY] + } + const pointer = (event: PointerEvent) => { + if ( + session || + event.button !== 0 || + Math.hypot(event.clientX - pointerStart[0], event.clientY - pointerStart[1]) > 5 + ) + return + const bounds = canvas.getBoundingClientRect() + raycaster.setFromCamera( + new Vector2( + ((event.clientX - bounds.left) / bounds.width) * 2 - 1, + (-(event.clientY - bounds.top) / bounds.height) * 2 + 1 + ), + camera + ) + pick() + } + const onSelect = (event: XRInputSourceEvent) => { + probe.select(event.inputSource) + const space = renderer.xr.getReferenceSpace() + if (!space || !session) return + const pose = event.frame.getPose(event.inputSource.targetRaySpace, space) + if (!pose) { + halt() + return + } + matrix.fromArray(pose.transform.matrix).premultiply(rig.matrixWorld) + raycaster.ray.origin.setFromMatrixPosition(matrix) + raycaster.ray.direction.set(0, 0, -1).transformDirection(matrix) + const nav = raycaster.intersectObject(navigation.mesh)[0] + if (nav?.uv && nav.uv.y > 0.38) { + halt() + const action = Math.min(4, Math.floor(nav.uv.x * 5)) + if (action === 0) home() + if (action === 1) back() + if (action === 2) destination() + if (action === 4) + void session.end().catch((error: unknown) => callbacks.status(String(error))) + return + } + if (flightButton.mesh.visible && raycaster.intersectObject(flightButton.mesh).length) { + halt() + arm = true + callbacks.status('Release the flight trigger, then squeeze to accelerate.') + return + } + if (flight.mode === 'armed' && event.inputSource.handedness === hand) return + pick() + } + const onVisibility = () => { + halt() + probe.interrupt() + lastTime = 0 + } + const onSources = () => { + halt() + probe.interrupt() + } + const end = () => { + const old = session + session = null + old?.removeEventListener('select', onSelect) + old?.removeEventListener('visibilitychange', onVisibility) + old?.removeEventListener('inputsourceschange', onSources) + old?.removeEventListener('end', end) + renderer.setAnimationLoop(null) + halt() + rig.position.set(0, 0, 0) + if (desktop) { + camera.position.copy(desktop.position) + camera.quaternion.copy(desktop.quaternion) + controls.target.copy(desktop.target) + } + controls.enabled = true + navigation.mesh.visible = false + flightButton.mesh.visible = false + callbacks.session(false) + callbacks.status('Session ended. Desktop viewpoint restored.') + // Three.js releases its XR framebuffer in another listener for this event. + queueMicrotask(resize) + } + async function enter() { + if (session || disposed) return + if (!isSecureContext || !navigator.xr) + throw new Error('WebXR requires a supported browser and a trusted secure connection.') + const next = await navigator.xr.requestSession('immersive-vr', { + requiredFeatures: ['local-floor'], + optionalFeatures: ['hand-tracking'] + }) + if (disposed) { + await next.end() + return + } + desktop = { + position: camera.position.clone(), + quaternion: camera.quaternion.clone(), + target: controls.target.clone() + } + session = next + controls.enabled = false + cancelAnimationFrame(frameHandle) + frameHandle = 0 + camera.position.set(0, 0, 0) + camera.quaternion.identity() + move([0, 0, 0]) + lastTime = 0 + next.addEventListener('select', onSelect) + next.addEventListener('end', end) + next.addEventListener('visibilitychange', onVisibility) + next.addEventListener('inputsourceschange', onSources) + try { + await renderer.xr.setSession(next) + navigation.mesh.visible = true + panelAtViewer() + probe.start(next) + callbacks.session(true) + renderer.setAnimationLoop((time, frame) => { + if (!frame || !session) return + const space = renderer.xr.getReferenceSpace() + if (!space) { + halt() + return + } + if (!lastTime) panelAtViewer() + probe.frame(time, frame, space) + const source = Array.from(session.inputSources).find( + (s) => + s.handedness === hand && + s.targetRayMode === 'tracked-pointer' && + s.gripSpace && + s.gamepad?.mapping === 'xr-standard' + ) + const pose = source?.gripSpace ? frame.getPose(source.gripSpace, space) : null + const ready = !!source && probe.flightReady(source) + if (sourceInUse && sourceInUse !== source) halt() + sourceInUse = source ?? null + const q = pose?.transform.orientation + vector.set(0, 0, -1) + if (q) vector.applyQuaternion(new Quaternion(q.x, q.y, q.z, q.w)) + // Default mode translates the rig only; the head quaternion is never a flight input. + const trigger = source?.gamepad?.buttons[0]?.value ?? 0 + flight = stepFlight( + flight, + { + tracked: + !!pose && + !pose.emulatedPosition && + [ + pose.transform.position.x, + pose.transform.position.y, + pose.transform.position.z, + pose.transform.orientation.x, + pose.transform.orientation.y, + pose.transform.orientation.z, + pose.transform.orientation.w + ].every(Number.isFinite) && + ready, + visible: session.visibilityState === 'visible' && !!frame.getViewerPose(space), + trigger, + direction: [vector.x, vector.y, vector.z], + arm: arm && ready, + brake: !!source?.gamepad?.buttons[1]?.pressed + }, + lastTime ? (time - lastTime) / 1000 : 0 + ) + arm = false + lastTime = time + rig.position.fromArray(flight.position) + flightButton.mesh.visible = ready && flight.mode !== 'armed' + const viewerPosition = renderer.xr.getCamera().getWorldPosition(new Vector3()) + labels.forEach((label) => label.mesh.lookAt(viewerPosition)) + if (card.mesh.visible) card.mesh.lookAt(viewerPosition) + renderer.render(world, camera) + }) + } catch (error) { + await next.end().catch(() => undefined) + if (session) end() + throw error + } + } + const resize = () => { + const { width, height } = host.getBoundingClientRect() + if (!session) renderer.setSize(width, height) + camera.aspect = width / Math.max(1, height) + camera.updateProjectionMatrix() + invalidate() + } + const observer = new ResizeObserver(resize) + observer.observe(host) + controls.addEventListener('change', invalidate) + canvas.addEventListener('click', pointer) + canvas.addEventListener('pointerdown', pointerDown) + const lost = (event: Event) => { + event.preventDefault() + halt() + callbacks.status('Graphics context lost. Exit the session and reload the viewer.') + void session?.end().catch((error: unknown) => callbacks.status(String(error))) + } + canvas.addEventListener('webglcontextlost', lost) + resize() + return { + enter, + select, + home, + back, + destination, + brake: halt, + setHand(value: 'left' | 'right') { + halt() + hand = value + }, + async exit() { + if (session) await session.end() + }, + setSnapshot(value: SpatialSnapshot) { + if (session) throw new Error('Exit to apply graph changes.') + disposeGeometry() + snapshot = value + adjacency = graphAdjacency(value.graph) + if (travel.revision !== value.revision) { + travel = { revision: value.revision, position: [0, 0, 0], trail: [] } + move([0, 0, 0]) + } + const colors = new Float32Array(value.positions.length) + value.graph.nodes.forEach((node, i) => + new Color(graphColor(node.kind, node.platform)).toArray(colors, i * 3) + ) + const geometry = new BufferGeometry() + .setAttribute('position', new BufferAttribute(new Float32Array(value.positions), 3)) + .setAttribute('color', new BufferAttribute(colors, 3)) + geometry.computeBoundingSphere() + points = new Points( + geometry, + new PointsMaterial({ size: 0.015, sizeAttenuation: true, vertexColors: true }) + ) + root.add(points) + const lines = value.graph.edges + .slice(0, 20_000) + .flatMap((edge) => [ + ...value.positions.slice(edge.source * 3, edge.source * 3 + 3), + ...value.positions.slice(edge.target * 3, edge.target * 3 + 3) + ]) + edges = new LineSegments( + new BufferGeometry().setAttribute( + 'position', + new BufferAttribute(new Float32Array(lines), 3) + ), + new LineBasicMaterial({ color: '#243850' }) + ) + root.add(edges) + const hubs = value.graph.nodes + .map((node, i) => ({ node, i })) + .filter(({ node }) => node.kind !== 'link') + .sort((a, b) => adjacency[b.i].length - adjacency[a.i].length) + .slice(0, 8) + for (const { node, i } of hubs) { + const label = spatialPanel(0.45, 0.09) + label.draw([`${node.label} (${adjacency[i].length})`]) + label.mesh.position.fromArray(value.positions, i * 3) + label.mesh.position.y += 0.08 + label.mesh.lookAt(camera.position) + labels.push(label) + world.add(label.mesh) + } + marker.visible = false + card.mesh.visible = false + if (selected) select(selected) + invalidate() + }, + detail(lines: string[], image?: CanvasImageSource) { + card.draw(lines, image) + invalidate() + }, + report() { + return { + ...probe.report(), + graphRevision: snapshot?.revision, + nodes: snapshot?.graph.nodes.length, + drawnEdges: Math.min(snapshot?.graph.edges.length ?? 0, 20_000), + labels: labels.length, + geometries: renderer.info.memory.geometries, + textures: renderer.info.memory.textures, + calls: renderer.info.render.calls, + note: 'Rendering caps are provisional. Headset frame-time and memory budgets remain unverified.' + } + }, + async dispose() { + disposed = true + if (session) await session.end() + cancelAnimationFrame(frameHandle) + renderer.setAnimationLoop(null) + observer.disconnect() + controls.removeEventListener('change', invalidate) + controls.dispose() + canvas.removeEventListener('click', pointer) + canvas.removeEventListener('pointerdown', pointerDown) + canvas.removeEventListener('webglcontextlost', lost) + disposeGeometry() + card.dispose() + navigation.dispose() + flightButton.dispose() + marker.geometry.dispose() + ;(marker.material as MeshBasicMaterial).dispose() + renderer.dispose() + canvas.remove() + } + } +} diff --git a/apps/electron/src/spatial/style.css b/apps/electron/src/spatial/style.css new file mode 100644 index 000000000..1559433a4 --- /dev/null +++ b/apps/electron/src/spatial/style.css @@ -0,0 +1,302 @@ +* { + box-sizing: border-box; +} +body { + margin: 0; + background: #080f1d; + color: #e6edf8; + font: + 14px/1.55 system-ui, + -apple-system, + sans-serif; +} +button, +input, +select, +textarea { + font: inherit; +} +button, +select, +input, +textarea { + border: 1px solid #32435f; + border-radius: 7px; + background: #152139; + color: #e6edf8; + padding: 9px 11px; +} +button { + cursor: pointer; +} +button:hover { + background: #233958; + border-color: #7da9be; +} +button:disabled { + opacity: 0.45; + cursor: default; +} +button:focus-visible, +input:focus-visible, +select:focus-visible { + outline: 2px solid #6dd5bd; + outline-offset: 3px; +} +header { + height: 64px; + display: flex; + align-items: center; + justify-content: space-between; + padding: 0 28px; + border-bottom: 1px solid #24324a; +} +.brand { + font-size: 24px; + color: #eff6ff; + text-decoration: none; + letter-spacing: -1px; +} +.brand span { + font-size: 14px; + color: #91a6bf; + letter-spacing: 0; + margin-left: 8px; +} +.badge { + font-size: 10px; + letter-spacing: 2px; + color: #82bbaa; +} +main { + display: grid; + grid-template-columns: 300px minmax(300px, 1fr) 280px; + height: calc(100vh - 64px); +} +.sidebar, +.inspector { + padding: 28px 22px; + overflow: auto; +} +.sidebar { + border-right: 1px solid #24324a; +} +.inspector { + border-left: 1px solid #24324a; + background: #0d1728; +} +h1 { + font-size: 30px; + line-height: 1.12; + letter-spacing: -1.2px; + margin: 10px 0 15px; +} +h2 { + font-size: 18px; + line-height: 1.4; + font-weight: 550; +} +.eyebrow { + color: #83c9b6; + font-size: 10px; + font-weight: 600; + letter-spacing: 2px; + margin: 0; +} +.intro { + color: #a8b8ce; + margin-bottom: 26px; +} +.muted { + font-size: 12px; + color: #92a7c2; +} +.stage { + position: relative; + min-width: 0; + overflow: hidden; +} +#canvas { + position: absolute; + inset: 0; +} +#canvas canvas { + display: block; + width: 100%; + height: 100%; +} +.stage-heading { + position: absolute; + left: 25px; + top: 24px; + display: flex; + gap: 10px; + align-items: center; + font-size: 12px; + pointer-events: none; +} +.live-dot { + height: 6px; + width: 6px; + border-radius: 50%; + background: #7bd2b9; +} +#status { + position: absolute; + top: 48px; + left: 25px; + right: 20px; + color: #90a9c7; + font-size: 12px; + pointer-events: none; +} +#error { + position: absolute; + top: 100px; + left: 25px; + right: 25px; + padding: 16px; + border: 1px solid #bf6969; + background: #321e2b; + color: #ffcfca; +} +.legend { + position: absolute; + bottom: 90px; + left: 25px; + right: 20px; + color: #6f8bad; + font-size: 11px; + pointer-events: none; +} +.toolbar { + position: absolute; + bottom: 25px; + left: 25px; + right: 20px; + display: flex; + flex-wrap: wrap; + gap: 8px; +} +.toolbar button { + background: #10203be8; + font-size: 12px; +} +label { + display: block; + color: #aebfd4; + font-size: 12px; + margin: 15px 0 7px; +} +label input, +label select, +label textarea { + display: block; + width: 100%; + margin-top: 7px; +} +details { + border-top: 1px solid #24324a; + padding: 15px 0; + margin-top: 15px; +} +summary { + cursor: pointer; + font-size: 12px; + color: #b7c7dc; +} +.results { + max-height: 220px; + overflow: auto; +} +.results button { + display: block; + width: 100%; + text-align: left; + font-size: 12px; + margin: 5px 0; + border-color: transparent; + background: transparent; +} +.results button:hover, +.results button[aria-pressed='true'] { + background: #203750; +} +.results small { + display: block; + color: #83a0be; +} +.session { + margin-top: 25px; +} +.primary { + width: 100%; + background: #a9e9d6; + color: #102b2b; + border: 0; + font-weight: 600; + margin: 10px 0; +} +.primary:hover { + background: #c5f4e5; +} +.inspector img { + width: 100%; + max-height: 190px; + object-fit: contain; + border-radius: 7px; + background: #16243b; +} +.inspector button { + margin: 6px 0; + width: 100%; +} +.inspector pre { + white-space: pre-wrap; + overflow-wrap: anywhere; + font: 12px/1.6 system-ui; + border-top: 1px solid #32435f; + padding-top: 15px; +} +#description { + white-space: pre-wrap; + overflow-wrap: anywhere; +} +#title { + overflow-wrap: anywhere; +} +@media (max-width: 1050px) { + main { + grid-template-columns: 260px minmax(250px, 1fr); + } + .inspector { + grid-column: 1 / -1; + max-height: 350px; + border-top: 1px solid #24324a; + } + main { + height: auto; + } + .stage { + height: 75vh; + } + .sidebar { + max-height: 75vh; + } +} +@media (max-width: 620px) { + main { + display: flex; + flex-direction: column; + } + .sidebar { + max-height: none; + } + .stage { + min-height: 500px; + order: -1; + } + .badge { + display: none; + } +} diff --git a/apps/electron/tsconfig.spatial.json b/apps/electron/tsconfig.spatial.json new file mode 100644 index 000000000..d5de224b2 --- /dev/null +++ b/apps/electron/tsconfig.spatial.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { "types": ["webxr", "vite/client"], "noEmit": true }, + "include": [ + "src/spatial/**/*.ts", + "src/shared/spatial-library.ts", + "src/shared/library-flight.ts" + ] +} diff --git a/docs/explorations/0468_[_]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md b/docs/explorations/0468_[-]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md similarity index 90% rename from docs/explorations/0468_[_]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md rename to docs/explorations/0468_[-]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md index fc5a274eb..c80ec0d42 100644 --- a/docs/explorations/0468_[_]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md +++ b/docs/explorations/0468_[-]_VISION_PRO_SPATIAL_LIBRARY_GRAPH.md @@ -1,7 +1,7 @@ --- title: Exploring the Library graph on Apple Vision Pro -status: draft -last_updated: 2026-10-03 +status: partially implemented +last_updated: 2026-10-06 review: 2026-10-24 decider: Chris Smothers door: two-way @@ -21,7 +21,21 @@ Useful moments include finding a saved video, seeing its title and thumbnail, fo This builds on the [personal Library](./0466_[-]_PERSONAL_LIBRARY_FOR_LEARNING_AND_SHARING.md), the [durable content proposal](./0467_[_]_DURABLE_LIBRARY_CONTENT_AS_XNET_NODES.md), and the earlier [immersive recommendation space exploration](./0151_[_]_SELF_ORGANIZING_SOCIAL_GRAPH_IMMERSIVE_RECOMMENDATION_SPACE.md). The earlier immersive proposal supplies product ideas; the code inventory below describes today's implementation. -**Status: research and design only.** No XR renderer, headset connection, or controller implementation is added here. No physical Vision Pro test has been performed for this exploration. +**Status: software experiment implemented; physical review pending.** The standalone browser viewer now shares graph filtering, search and initial layout with the desktop app. It includes a WebXR probe, stationary overview, explicit destination travel, guarded controller flight and an expiring read-only HTTPS pairing bridge. No physical Vision Pro or Sense controller result has been recorded. Full-pose flight, the native fallback decision and headset performance budgets remain gated on that review. + +The [runbook](../../apps/electron/src/spatial/README.md) explains how to build the viewer, establish trusted HTTPS and record device evidence. The implementation is stacked on the unfinished Library work in PR #721; it is not a production release claim. + +### Implementation evidence — 2026-10-06 + +| Area | Evidence | Limit | +| -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------ | +| Pure logic and access boundaries | 66 focused tests pass: deterministic flight, matched snapshot revisions, filtering, graph privacy, capability scope, expiry/revocation and verified TLS transport. The full suite passes 12,749 tests with four skipped. | Synthetic inputs do not prove physical controller behavior. | +| Browser viewer | Drove 1,000 and 56,000 synthetic link fixtures; searched the last link, inspected evidence and coverage, and applied platform/group/neighborhood filters with visible counts. No browser errors observed. | Desktop rendering does not establish stereo timing, readability or comfort. | +| Native desktop entry | Drove the real Electron Library graph and Spatial viewer control in an isolated profile; verified preload status and rejection of insecure HTTP. | No personal identity was replaced, no test-auth bypass was enabled, and no real Library was exposed. | +| Build and types | Spatial production bundle and Electron main/spatial typechecks pass. | Five existing renderer type errors reproduce at the base commit; headset execution remains unverified. | +| Rendering limits | One image/card, eight labels, 20,000 background edges, 200 selected edges; picking only on intentional selection. | Provisional limits. Cluster summaries, picking acceleration and resolution tuning await measurements. | + +Unchecked combined implementation/validation items below retain their physical-device requirements even where the software exists. The release review must decide whether the web path is viable before full-pose flight or a native client is implemented. The review date gives three weeks to decide whether the hardware experiment justifies further work. It is a decision checkpoint, not a delivery promise. The initial read-only viewer is reversible, so this exploration is `two-way`. A durable public pairing protocol or schema commitment would need an ADR with a `Tripwire:` in the [decision log](../../site/src/content/docs/docs/architecture/decisions.mdx) before implementation. @@ -38,7 +52,7 @@ We already own a Three.js renderer. We are not using `3d-force-graph` or `react- | Graph data | ✅ Reusable shape | [library-graph.ts](../../apps/electron/src/shared/library-graph.ts) defines nodes, indexed edges, relationship evidence, counts, and warnings. Positions are not currently part of that snapshot. | | Data access | 🚧 Electron dependency | The view calls `window.xnet.libraryGraph()` and related native APIs. [library-ipc.ts](../../apps/electron/src/main/library-ipc.ts) wires desktop access; Safari cannot use Electron's preload. | | Relationship construction | ✅ Source evidence | [graph.ts](../../apps/electron/src/library/graph.ts) derives the graph from Library records. Preserve resource identities and evidence when presenting it elsewhere. | -| XR input and presentation | ⬜ Proposed | No headset session, spatial panels, controller flight, or native visionOS target is established by these files. | +| XR input and presentation | 🚧 Experimental | A separate [browser scene](../../apps/electron/src/spatial/scene.ts) provides WebXR lifecycle, spatial panels and guarded flight. Hardware validation and a native visionOS target remain pending. | The desktop [package manifest](../../apps/electron/package.json) pins Three.js `0.186.1` and `d3-force-3d` `3.0.6`. Test changes against those versions before assuming an upgrade is necessary. @@ -252,7 +266,7 @@ Use the existing desktop interface as the accessible fallback. In-headset contro ### 1. Establish the hardware facts -- [ ] Build a minimal secure WebXR probe using synthetic geometry and explicit session entry/exit. +- [x] Build a minimal secure WebXR probe using synthetic geometry and explicit session entry/exit. - [ ] Record actual device/software versions, `immersive-vr` support, granted features, and session cadence. - [ ] Pair PS VR2 Sense controllers through the system and inspect input-source profiles, handedness, ray mode, grip pose, and gamepad mapping. - [ ] Verify independent position and rotation tracking, analog trigger values across the squeeze range, and disconnect/reconnect behavior. @@ -271,8 +285,8 @@ Use the existing desktop interface as the accessible fallback. In-headset contro ### 3. Connect the real Library and measure scale - [ ] Design and verify trusted, paired read-only access for snapshots, search, details, and thumbnails; keep private data off public hosting. -- [ ] Use stable IDs and revision-matched positions; handle stale responses, disconnection, and missing enrichment explicitly. -- [ ] Add playlist, platform, tag, and category filtering with visible counts and evidence labels. +- [x] Use stable IDs and revision-matched positions; handle stale responses, disconnection, and missing enrichment explicitly. +- [x] Add playlist, platform, tag, and category filtering with visible counts and evidence labels. - [ ] Measure 1,000-link, 10,000-link, and representative full-Library fixtures on the headset; document the chosen frame and memory budgets. - [ ] Add bounded labels, edge detail, thumbnail loading, and picking acceleration where measurements require them. - [ ] Decide whether native visionOS or Mac-assisted rendering is justified by an observed capability or performance gap. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 388bc9023..c828af6e5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -365,6 +365,9 @@ importers: '@types/three': specifier: 0.186.0 version: 0.186.0 + '@types/webxr': + specifier: 0.5.24 + version: 0.5.24 '@types/ws': specifier: ^8.5.10 version: 8.18.1 diff --git a/site/src/content/docs/docs/guides/electron.mdx b/site/src/content/docs/docs/guides/electron.mdx index c08f96901..fa8fa8ad4 100644 --- a/site/src/content/docs/docs/guides/electron.mdx +++ b/site/src/content/docs/docs/guides/electron.mdx @@ -180,6 +180,36 @@ Enable verbose sync logging: localStorage.setItem('xnet:sync:debug', 'true') ``` +## Experimental spatial Library viewer + +The Library graph's **Spatial viewer** control shares the saved links in the current +view with a separate, read-only browser viewer. Search and source/group filters +remain available in its flat view. A supported headset can request a stationary +WebXR overview with a selected-resource card, Home, Back and explicit travel. + +To connect a headset, put it and the Mac on a trusted private network. Provide the +Mac's HTTPS address with a port above 1023, then select a valid certificate for that +address and its private key. Establish certificate trust on both devices through +their operating systems; do not bypass browser warnings. The key stays on the Mac. +Open the copied pairing link within five minutes. It can be used once, expires +within one hour, and loses its capability on reload. **Revoke access** or quitting +the desktop app stops access. Previously viewed content cannot be recalled. + +The viewer exposes scoped graph data, saved details and thumbnails. It does not +expose private conversation records, the development debugger or the general Local +API. A preview or written caption remains distinct from a spoken transcript. + +Vision Pro and PlayStation VR2 Sense support, headset performance and comfort are +**not yet physically verified**. Experimental flight requires observed tracked +position, rotation and analog trigger range, explicit arming and trigger release. +Full-pose flight is not enabled. Choose **Exit / search** to leave immersion and +use normal browser text input; an immersive DOM overlay is not assumed. + +Developers can run the synthetic 1,000, 10,000 and 56,000-link fixtures with +`pnpm --filter xnet-desktop spatial:dev`. The development server is loopback-only; +use the paired HTTPS path for a headset. The device review procedure is in +`apps/electron/src/spatial/README.md` in the repository. + ## Building for production ```bash diff --git a/site/src/data/changelog/2026-10-06-explore-a-private-spatial-library-previe.json b/site/src/data/changelog/2026-10-06-explore-a-private-spatial-library-previe.json new file mode 100644 index 000000000..738768fc5 --- /dev/null +++ b/site/src/data/changelog/2026-10-06-explore-a-private-spatial-library-previe.json @@ -0,0 +1,8 @@ +{ + "id": "2026-10-06-explore-a-private-spatial-library-previe", + "date": "October 6, 2026", + "title": "Explore a private spatial Library preview", + "summary": "Pair selected saved links with an experimental read-only browser viewer, search their neighborhoods, and inspect saved details; Vision Pro controller support and comfort still need device testing.", + "highlights": [], + "tags": ["app"] +} From 4ef29b5e6a6b3cd72ad6a946ed9cf2823cf6dca4 Mon Sep 17 00:00:00 2001 From: xNet Test Date: Tue, 6 Oct 2026 13:00:20 -0700 Subject: [PATCH 4/4] docs(spatial): include pairing in generated documentation - Regenerate the documentation export with the spatial pairing guide and ADR-47. - Verify the generated text matches all 63 source documentation pages. Signed-off-by: xNet Test --- site/public/llms-full.txt | 64 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/site/public/llms-full.txt b/site/public/llms-full.txt index f8d11b59a..f6b449ea8 100644 --- a/site/public/llms-full.txt +++ b/site/public/llms-full.txt @@ -8825,6 +8825,36 @@ Enable verbose sync logging: localStorage.setItem('xnet:sync:debug', 'true') ``` +## Experimental spatial Library viewer + +The Library graph's **Spatial viewer** control shares the saved links in the current +view with a separate, read-only browser viewer. Search and source/group filters +remain available in its flat view. A supported headset can request a stationary +WebXR overview with a selected-resource card, Home, Back and explicit travel. + +To connect a headset, put it and the Mac on a trusted private network. Provide the +Mac's HTTPS address with a port above 1023, then select a valid certificate for that +address and its private key. Establish certificate trust on both devices through +their operating systems; do not bypass browser warnings. The key stays on the Mac. +Open the copied pairing link within five minutes. It can be used once, expires +within one hour, and loses its capability on reload. **Revoke access** or quitting +the desktop app stops access. Previously viewed content cannot be recalled. + +The viewer exposes scoped graph data, saved details and thumbnails. It does not +expose private conversation records, the development debugger or the general Local +API. A preview or written caption remains distinct from a spoken transcript. + +Vision Pro and PlayStation VR2 Sense support, headset performance and comfort are +**not yet physically verified**. Experimental flight requires observed tracked +position, rotation and analog trigger range, explicit arming and trigger release. +Full-pose flight is not enabled. Choose **Exit / search** to leave immersion and +use normal browser text input; an immersive DOM overlay is not assumed. + +Developers can run the synthetic 1,000, 10,000 and 56,000-link fixtures with +`pnpm --filter xnet-desktop spatial:dev`. The development server is loopback-only; +use the paired HTTPS path for a headset. The device review procedure is in +`apps/electron/src/spatial/README.md` in the repository. + ## Building for production ```bash @@ -11674,6 +11704,40 @@ shutdown. The method covers watch-dispatched work, not independent explicit scan or any consumer disposes storage before awaiting the drain. Revisit ownership and cancellation semantics before changing the shutdown guarantee. +## ADR-47: Pair spatial Library viewers with an ephemeral read capability + +**Status:** Accepted · 2026-10-06 · exploration 0468 + +**Context:** A headset browser cannot use Electron's preload. Exposing the general +local API or development server would give the viewer capabilities it does not need. + +**Decision:** The owner starts a separate HTTPS listener from the desktop graph, +explicitly choosing the visible saved links and a certificate already trusted by +both devices. Only private-network clients are accepted. The listener serves a +compiled browser viewer and a frozen, revision-matched graph/layout projection. +A random 256-bit invitation in the URL fragment can be redeemed once within five +minutes for an in-memory bearer capability. The capability expires one hour after +creation or immediately when the owner revokes access, the app exits, or the +listener stops. Credentials are never logged or persisted by the viewer. + +The only data routes are snapshot, bounded search, selected-resource details and +thumbnails resolved through those resources. No arbitrary blob IDs, conversations, +personal notes, identity material, SQL, writes or enrichment jobs are exposed. +Details are a bounded projection; longer saved source text or transcript text is +requested explicitly and limited to 100,000 characters. Reads check revocation +again after asynchronous storage access. Exact host and origin checks, no CORS, +no-store responses and a same-origin content policy constrain browser access. + +**Tradeoff:** This is an experimental, session-local transport, not an independent +synced Library or a supported durable public protocol. Users must provision trust +and address resolution on both devices. A copied capability authorizes its holder +until expiry or revocation; it is not a hardware-bound device credential. Revocation +stops future reads but cannot erase content already delivered to a device. + +**Tripwire:** We need internet access, persistent pairing, offline private content, +multiple readers, or native clients consuming a stable protocol. Revisit the +capability and ownership design before extending this experimental transport. + --- ## Package Graph