From 688f2cbbab8e43260350fd676f6e5cd0a292a527 Mon Sep 17 00:00:00 2001 From: Zach Wilke Date: Mon, 7 Sep 2026 19:11:22 -0500 Subject: [PATCH 1/7] time: stamps cross the bridge as UTC; local time is a display concern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stamps arrived as the Mac's naive wall clock ("2026-09-07 14:33:12") and were compared against the Linux clock — the same string only while both machines sat in one timezone. A Mac an hour ahead put every read mark ahead of every message, so nothing ever counted as unread; an hour behind and the backlog re-toasted. And once a year, in the DST fall-back hour, the Mac's own clock repeats: two messages an hour apart carried the SAME stamp, so ordering, the watermark and "newer than the mark" stopped meaning anything for that hour. Every `a.ts < b.ts` in the collector was already assuming lexical order equals chronological order; only UTC actually provides it. imsg's JSON now emits ISO-8601 UTC (fmt_ts); the plain-text renders keep the Mac's clock (fmt_ts_local), because a person reading `imsg recent` wants the time they remember. On the Linux side localNowTs() became nowTs() in the same format, and every label converts to the READER's zone at render: localDay() and formatStamp() in thread.ts, stampMs()/localDay()/fmtTime() in BlipView. Day dividers no longer slice the date out of the string — that is UTC's day, and the divider belongs at your midnight. Upgrading is covered from both ends, and both halves are load-bearing together: loadState() migrates the marks a pre-UTC release wrote, and stamps are normalised at the two fetch doors so a Mac still on the old bridge keeps working. Migrating the marks alone would have sorted every message below every mark (" " < "T") and silently emptied the badge and the toasts. The suite runs pinned to TZ=UTC, where all of this is invisible; the new timezone.test.ts and bridge/mac/test_wire_time.py set their own zones and are what actually cover it. Verified live against the gateway Mac on both the old and the new bridge: 298 threads, unread unchanged, no toast storm on migration, and 23:47:54Z rendering as 6:47 PM in CDT. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015xxS9WCT7ifHhbDjYyGnHS --- .github/workflows/ci.yml | 2 + BlipView.qml | 44 +++-- CHANGELOG.md | 19 ++ CLAUDE.md | 25 +++ ROADMAP.md | 12 +- bridge/mac/imsg | 29 +++- bridge/mac/test_wire_time.py | 100 +++++++++++ collector.test.ts | 324 +++++++++++++++++------------------ collector.ts | 101 ++++++++--- test-setup.ts | 5 + thread.test.ts | 162 +++++++++--------- thread.ts | 89 +++++++--- tier2.test.ts | 32 ++-- timezone.test.ts | 168 ++++++++++++++++++ 14 files changed, 788 insertions(+), 324 deletions(-) create mode 100644 bridge/mac/test_wire_time.py create mode 100644 timezone.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d9dbf6..d9756b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,8 @@ jobs: run: python3 bridge/mac/test_unsent.py - name: conversation cluster pins (no macOS needed) run: python3 bridge/mac/test_cluster.py + - name: stamps cross the bridge as UTC (no macOS needed) + run: python3 bridge/mac/test_wire_time.py - name: shell scripts (bash -n + shellcheck) run: | bash -n bridge/linux/blip-shim scripts/blip-setup scripts/sync-bridge.sh scripts/demo/blip-shots bridge/mac/install.sh diff --git a/BlipView.qml b/BlipView.qml index f7771bd..70221f5 100644 --- a/BlipView.qml +++ b/BlipView.qml @@ -1251,7 +1251,7 @@ FocusScope { // writing the row) happens behind it. The field clears at once, so a // second message can follow without waiting — sends queue in order. var chat = String(root.active.chat) - var stamp = root.localStamp() + var stamp = root.wireStamp() var target = root.activeIsGroup ? ["--chat-id", String(root.active.guid)] : ["--to", chat] @@ -1267,9 +1267,27 @@ FocusScope { pumpSend() } - /** Local wall clock as a chat.db-style stamp; the pending bubble's ts. */ - function localStamp() { - return Qt.formatDateTime(new Date(), "yyyy-MM-dd HH:mm:ss") + /** Now in the bridge's WIRE format (UTC ISO-8601): the pending bubble's ts. + * It is compared against real message stamps — which are UTC — so a local + * wall clock here would put every in-flight bubble hours out of order. */ + function wireStamp() { + return new Date().toISOString().replace(/\.\d{3}Z$/, "Z") + } + + /** Wire stamp → epoch ms. Mirrors stampMs() in thread.ts, including the + * fallback that reads a pre-UTC stamp as local. */ + function stampMs(ts) { + var t = String(ts || "") + if (t === "") return NaN + return Date.parse(t.indexOf("T") >= 0 ? t : t.replace(" ", "T")) + } + + /** The LOCAL calendar date a wire stamp falls on — day dividers break at + * the reader's midnight, never UTC's. Mirrors localDay() in thread.ts. */ + function localDay(ts) { + var ms = stampMs(ts) + if (isNaN(ms)) return "" + return Qt.formatDate(new Date(ms), "yyyy-MM-dd") } /** The instant echo: thread.ts's pendingBubble() in miniature — enough to @@ -1278,8 +1296,8 @@ FocusScope { function appendPendingBubble(list, text, stamp) { var out = (list || []).slice() var prev = out.length ? out[out.length - 1] : null - var newDay = !prev || String(prev.ts || "").slice(0, 10) !== stamp.slice(0, 10) - var gapMin = prev ? (Date.parse(stamp.replace(" ", "T")) - Date.parse(String(prev.ts || "").replace(" ", "T"))) / 60000 : Infinity + var newDay = !prev || root.localDay(prev.ts) !== root.localDay(stamp) + var gapMin = prev ? (root.stampMs(stamp) - root.stampMs(prev.ts)) / 60000 : Infinity var start = !prev || newDay || prev.from_me !== true || !(gapMin <= 15) if (!start) { var p = Object.assign({}, prev); p.groupEnd = false; p.time = ""; out[out.length - 1] = p } out.push({ ts: stamp, from_me: true, name: "", text: String(text).trim(), day: newDay ? "Today" : "", @@ -1331,13 +1349,15 @@ FocusScope { // The same patterns as the bubbles: today the time, older rows the date and // the time, with the year once it is not this year. function fmtTime(ts) { - var s = String(ts || "") - if (s.length < 16) return s + var ms = root.stampMs(ts) + if (isNaN(ms)) return String(ts || "") var now = new Date() - var at = new Date(+s.substring(0, 4), +s.substring(5, 7) - 1, +s.substring(8, 10), - +s.substring(11, 13), +s.substring(14, 16)) + // The stamp is UTC; every label below is the reader's local time. Reading + // the digits out of the string (as this did) showed a Mac's clock, and + // compared it against a Linux date. + var at = new Date(ms) var clock = Qt.formatTime(at, root.timeFormat) - if (s.substring(0, 10) === Qt.formatDate(now, "yyyy-MM-dd")) return clock + if (Qt.formatDate(at, "yyyy-MM-dd") === Qt.formatDate(now, "yyyy-MM-dd")) return clock // Messages stamps a row "Yesterday", then the weekday for the last week, // then a date — never a date-plus-clock, which is what a mail client does. var midnight = new Date(now.getFullYear(), now.getMonth(), now.getDate()) @@ -2256,7 +2276,7 @@ FocusScope { font.bold: true } Text { - text: String(modelData.ts || "").slice(0, 16) + text: root.fmtTime(modelData.ts) textFormat: Text.PlainText color: root.dim font.family: root.fontFamily diff --git a/CHANGELOG.md b/CHANGELOG.md index 64a98d0..ec96e91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## Unreleased +- **Time crosses the bridge as UTC.** Stamps used to arrive as the Mac's naive + wall clock ("2026-09-07 14:33:12") and were compared against the Linux + clock — the same string only while both machines sat in one timezone. A Mac + an hour ahead put every read mark ahead of every message, so nothing ever + counted as unread; an hour behind and the backlog re-toasted. And once a + year, in the DST fall-back hour, the Mac's own clock repeated itself: two + messages an hour apart carried the SAME stamp, so ordering, the watermark + and "newer than the mark" all quietly stopped meaning anything for that + hour. The bridge now emits ISO-8601 UTC (`2026-09-07T18:33:12Z`), which is + monotonic and whose lexical order is chronological order — the property + every ledger, sort and watermark in Blip was already assuming. Local time + became a display concern: bubbles, day dividers, "Today"/"Yesterday" and + read receipts are rendered in the READER's zone, so a day still breaks at + your midnight and not at Greenwich's. `imsg`'s own plain-text output keeps + the Mac's clock — a person reading `imsg recent` wants the time they + remember. Upgrading migrates the marks in `state.json`, and stamps from a + Mac still on the old bridge are normalised as they come in, so a + half-upgraded pair keeps working instead of silently going quiet. + - **Pinned avatars no longer grow on the first cursor move.** The tiles were measured before the grid had its width, and the layout kept that small size until the next relayout; the size is now a layout hint, so they render at diff --git a/CLAUDE.md b/CLAUDE.md index 06f247e..2f00cba 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,6 +52,31 @@ what it is handed. Keep it that way. can carry a FUTURE timestamp (tz skew); a mark taken from the global max once suppressed unrelated threads until "tomorrow". The panel passes the newest VISIBLE ts (`--seen`) so mid-round-trip arrivals stay unread. +- **Every stamp inside Blip is UTC; local time is a DISPLAY concern.** The + bridge emits ISO-8601 UTC to the second (`2026-09-07T18:33:12Z`, `fmt_ts`), + because fixed-width UTC is the one format whose LEXICAL order is + chronological order — which every watermark, ledger, `maxTs` and `a.ts < + b.ts` in the collector silently assumes. Naive Mac wall clock broke that + twice: against a Linux clock in another zone every mark sat ahead of every + message (nothing unread), and in the DST fall-back hour the Mac's own clock + repeats, so two messages an hour apart carried the same string. Convert to + the reader's zone only when rendering — `localDay()`/`formatStamp()` in + thread.ts, `stampMs()`/`localDay()`/`fmtTime()` in BlipView. NEVER slice a + date out of a stamp (`ts.slice(0, 10)`) to find its day: that is UTC's day, + and the divider belongs at the reader's midnight. `imsg`'s plain-text + renders keep `fmt_ts_local` — a human reading `imsg recent` wants the time + they remember; every JSON field is `fmt_ts`. + **Stamps are normalised at the two fetch doors** (`fetchMessages` in + collector.ts, `loadThread` in thread.ts) via `toUtcStamp`, and `loadState` + migrates the marks a pre-UTC release wrote. Both halves are load-bearing + together: migrating the marks while the bridge still emitted naive stamps + would sort every message BELOW every mark (`" "` < `"T"`) and silently + empty the badge and the toasts. Legacy stamps are read as Linux-local — + exact whenever the Mac shared the zone, which is every setup where the old + format looked right. The suite runs pinned to `TZ=UTC` (test-setup.ts), + where all of this is invisible; `timezone.test.ts` and + `bridge/mac/test_wire_time.py` set their own zones and are what actually + cover it. - **Reads are optimistic-with-suppression.** Persistent read state moves only via collector runs (~1 s), so BarWidget applies reads to the local model IMMEDIATELY and remembers them in `localReads[chat]` (thread last_ts at diff --git a/ROADMAP.md b/ROADMAP.md index 6f69113..609ab18 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -180,9 +180,15 @@ panel (3 lenses × finding, against current main) confirmed 11 closed and digits-only id shorter than 5 as "not a phone", so a carrier-style sender opens read-only with "group id unknown". Widen to `{3,15}` (E.164 max is 15) once tested against a real one. 5+ digit codes load correctly since 2.2.1. -- [ ] **Timestamps are Mac-local wall-clock strings** compared against the - Linux clock (DST fall-back hour, a UTC Mac). Move the bridge to epoch/UTC - and convert on display. Cross-cutting; do it as its own release. +- [x] **Timestamps are Mac-local wall-clock strings** compared against the + Linux clock (DST fall-back hour, a UTC Mac) — fixed: the bridge emits + ISO-8601 UTC (`fmt_ts`), `fmt_ts_local` keeps the human CLI renders on the + Mac's clock, and every label is converted to the reader's zone at display + (`localDay`/`formatStamp` in thread.ts, `fmtTime` in BlipView). Stamps are + normalised at the two fetch doors, so a Mac on the old bridge still works; + `state.json` marks migrate on load. Covered by `timezone.test.ts` and + `bridge/mac/test_wire_time.py` (both pin a zone; the rest of the suite runs + at UTC, where the bug is invisible). - [ ] **One never-opened unread pins the catch-up loop** — every poll walks 150→8192 rows across sequential ssh calls. Cache the reconciliation boundary per chat. diff --git a/bridge/mac/imsg b/bridge/mac/imsg index 53226af..88271e8 100755 --- a/bridge/mac/imsg +++ b/bridge/mac/imsg @@ -442,6 +442,27 @@ def decode_attributed_body(blob: bytes | None) -> str | None: def fmt_ts(apple_ns: int | None) -> str: + """WIRE format: UTC, ISO-8601, second precision ("2026-09-07T18:33:12Z"). + + Stamps used to cross the bridge as NAIVE Mac-local wall clock, and the + Linux side compared them against its OWN clock. Two ways that lied: a Mac + in another timezone shifted every read mark by the offset (marks ahead of + every message = nothing ever unread), and on the DST fall-back the Mac's + own wall clock repeats an hour, so for that hour string order stopped + matching real order. Every `a.ts < b.ts` in the client depends on those + being the same thing. UTC is monotonic and sorts lexicographically in + chronological order; clients convert to local for DISPLAY only. + """ + if not apple_ns: + return "?" + return dt.datetime.fromtimestamp( + apple_ns / 1e9 + APPLE_EPOCH, dt.timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def fmt_ts_local(apple_ns: int | None) -> str: + """Mac-local wall clock, for the HUMAN plain-text renders only. A person + reading `imsg recent` wants the time they remember, not UTC.""" if not apple_ns: return "?" return dt.datetime.fromtimestamp(apple_ns / 1e9 + APPLE_EPOCH).strftime( @@ -908,7 +929,7 @@ def render(rows, as_json: bool, with_names: bool = True, con=None, rich: bool = text = message_text(r).replace("\n", " | ") chat = r["room_name"] or r["chat_identifier"] or "" chat_tag = f" [{chat}]" if chat and chat != (r["handle"] or "") else "" - print(f"{fmt_ts(r['date'])} {who:>30}{chat_tag}: {text}") + print(f"{fmt_ts_local(r['date'])} {who:>30}{chat_tag}: {text}") # Two kinds of rows live in the message table without being messages. Both are @@ -1083,7 +1104,7 @@ def cmd_contacts(con, args): for r in rows: nm = name_for(r["handle"]) if not args.no_names else None suffix = f" ({nm})" if nm else "" - print(f"{fmt_ts(r['last_date'])} {r['n']:>6} {r['service']:>8} {r['handle']}{suffix}") + print(f"{fmt_ts_local(r['last_date'])} {r['n']:>6} {r['service']:>8} {r['handle']}{suffix}") def cmd_chats(con, args): @@ -1165,7 +1186,7 @@ def cmd_chats(con, args): return for r in rows: name = r["display_name"] or r["room_name"] or r["chat_identifier"] - print(f"{fmt_ts(r['last_date'])} {r['n']:>5} {r['service_name']:>8} {name}") + print(f"{fmt_ts_local(r['last_date'])} {r['n']:>5} {r['service_name']:>8} {name}") def cmd_groups(con, args): @@ -1235,7 +1256,7 @@ def cmd_groups(con, args): for h in ((r["participants"] or "").split(",") if r["participants"] else []) ) name = r["display_name"] or members or r["chat_identifier"] - print(f"{fmt_ts(r['last_date']) if r['last_date'] else '':19} {r['guid']} {name}") + print(f"{fmt_ts_local(r['last_date']) if r['last_date'] else '':19} {r['guid']} {name}") ATTACHMENTS_ROOT = os.path.expanduser("~/Library/Messages/Attachments") diff --git a/bridge/mac/test_wire_time.py b/bridge/mac/test_wire_time.py new file mode 100644 index 0000000..d7acc8d --- /dev/null +++ b/bridge/mac/test_wire_time.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Stamps cross the bridge as UTC. + +The naive Mac-local wall clock they used to carry was only comparable to the +Linux side's clock while both machines sat in one timezone, and stopped being +ordered at all during the DST fall-back hour. `fmt_ts` is the wire; the plain +text renders keep `fmt_ts_local`, because a person reading `imsg recent` wants +the time they remember.""" +from __future__ import annotations + +import datetime as dt +import os +import sys +import time +import unittest +from importlib.machinery import SourceFileLoader +from importlib.util import module_from_spec, spec_from_loader +from pathlib import Path + +HERE = Path(__file__).parent + + +def load(tool: str): + loader = SourceFileLoader(f"blip_{tool}", str(HERE / tool)) + spec = spec_from_loader(loader.name, loader) + assert spec is not None + mod = module_from_spec(spec) + sys.modules[loader.name] = mod + loader.exec_module(mod) + return mod + + +imsg = load("imsg") +APPLE_EPOCH = 978307200 + + +def apple_ns(*, year, month, day, hour=0, minute=0, second=0) -> int: + """Apple's nanoseconds-since-2001 for a UTC instant.""" + when = dt.datetime(year, month, day, hour, minute, second, tzinfo=dt.timezone.utc) + return int((when.timestamp() - APPLE_EPOCH) * 1e9) + + +class InZone: + """Run a block with the process in `zone` — fmt_ts must not care.""" + + def __init__(self, zone: str) -> None: + self.zone = zone + + def __enter__(self): + self.prev = os.environ.get("TZ") + os.environ["TZ"] = self.zone + time.tzset() + return self + + def __exit__(self, *exc): + if self.prev is None: + os.environ.pop("TZ", None) + else: + os.environ["TZ"] = self.prev + time.tzset() + return False + + +class WireFormat(unittest.TestCase): + def test_utc_regardless_of_the_macs_timezone(self): + ns = apple_ns(year=2026, month=9, day=7, hour=18, minute=33, second=12) + for zone in ("UTC", "America/New_York", "Asia/Tokyo", "Australia/Adelaide"): + with InZone(zone): + self.assertEqual(imsg.fmt_ts(ns), "2026-09-07T18:33:12Z") + + def test_human_render_follows_the_macs_clock(self): + ns = apple_ns(year=2026, month=9, day=7, hour=18, minute=33, second=12) + with InZone("America/New_York"): + self.assertEqual(imsg.fmt_ts_local(ns), "2026-09-07 14:33:12") + with InZone("UTC"): + self.assertEqual(imsg.fmt_ts_local(ns), "2026-09-07 18:33:12") + + def test_sorts_lexicographically_in_time_order(self): + earlier = imsg.fmt_ts(apple_ns(year=2026, month=9, day=7, hour=23, minute=59)) + later = imsg.fmt_ts(apple_ns(year=2026, month=9, day=8, hour=0, minute=1)) + self.assertLess(earlier, later) + + def test_the_dst_fall_back_hour_stays_ordered(self): + # America/New_York, 2026-11-01: 01:00-01:59 runs twice, EDT then EST. + edt = apple_ns(year=2026, month=11, day=1, hour=5) # 01:00 EDT + est = apple_ns(year=2026, month=11, day=1, hour=6) # 01:00 EST, an hour later + with InZone("America/New_York"): + # The wall clock cannot tell these apart — the whole reason the + # wire is UTC. Every client orders messages by comparing stamps. + self.assertEqual(imsg.fmt_ts_local(edt), imsg.fmt_ts_local(est)) + self.assertLess(imsg.fmt_ts(edt), imsg.fmt_ts(est)) + + def test_missing_date_is_still_a_question_mark(self): + for empty in (0, None): + self.assertEqual(imsg.fmt_ts(empty), "?") + self.assertEqual(imsg.fmt_ts_local(empty), "?") + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/collector.test.ts b/collector.test.ts index ff17f11..e234349 100644 --- a/collector.test.ts +++ b/collector.test.ts @@ -43,7 +43,7 @@ const tmp = () => mkdtempSync(join(tmpdir(), "blip-test-")); function msg(over: Partial = {}): ImsgMessage { return { - ts: "2026-08-30 12:00:00", + ts: "2026-08-30T12:00:00Z", from_me: false, handle: "+15551234567", name: "Test Person", @@ -58,9 +58,9 @@ describe("buildThreads", () => { test("groups by chat and keeps the newest message as the preview", () => { const threads = buildThreads( [ - msg({ chat: "A", ts: "2026-08-30 10:00:00", text: "older" }), - msg({ chat: "A", ts: "2026-08-30 11:00:00", text: "newer" }), - msg({ chat: "B", ts: "2026-08-30 09:00:00", text: "b only" }), + msg({ chat: "A", ts: "2026-08-30T10:00:00Z", text: "older" }), + msg({ chat: "A", ts: "2026-08-30T11:00:00Z", text: "newer" }), + msg({ chat: "B", ts: "2026-08-30T09:00:00Z", text: "b only" }), ], "", ); @@ -73,9 +73,9 @@ describe("buildThreads", () => { test("orders threads newest-first regardless of input order", () => { const threads = buildThreads( [ - msg({ chat: "old", ts: "2026-01-01 00:00:00" }), - msg({ chat: "new", ts: "2026-08-30 23:59:59" }), - msg({ chat: "mid", ts: "2026-05-05 12:00:00" }), + msg({ chat: "old", ts: "2026-01-01T00:00:00Z" }), + msg({ chat: "new", ts: "2026-08-30T23:59:59Z" }), + msg({ chat: "mid", ts: "2026-05-05T12:00:00Z" }), ], "", ); @@ -83,24 +83,24 @@ describe("buildThreads", () => { }); test("first run reports zero unread — an empty watermark must not flag the backlog", () => { - const threads = buildThreads([msg({ ts: "2026-08-30 10:00:00" })], ""); + const threads = buildThreads([msg({ ts: "2026-08-30T10:00:00Z" })], ""); expect(threads[0]!.unread).toBe(0); }); test("counts only inbound messages newer than the watermark", () => { const threads = buildThreads( [ - msg({ ts: "2026-08-30 09:00:00" }), // old inbound - msg({ ts: "2026-08-30 11:00:00" }), // new inbound ✓ - msg({ ts: "2026-08-30 12:00:00", from_me: true }), // new outbound ✗ + msg({ ts: "2026-08-30T09:00:00Z" }), // old inbound + msg({ ts: "2026-08-30T11:00:00Z" }), // new inbound ✓ + msg({ ts: "2026-08-30T12:00:00Z", from_me: true }), // new outbound ✗ ], - "2026-08-30 10:00:00", + "2026-08-30T10:00:00Z", ); expect(threads[0]!.unread).toBe(1); }); test("a message exactly at the watermark is not unread", () => { - const threads = buildThreads([msg({ ts: "2026-08-30 10:00:00" })], "2026-08-30 10:00:00"); + const threads = buildThreads([msg({ ts: "2026-08-30T10:00:00Z" })], "2026-08-30T10:00:00Z"); expect(threads[0]!.unread).toBe(0); }); @@ -123,17 +123,17 @@ describe("buildThreads", () => { }); test("handles an empty window", () => { - expect(buildThreads([], "2026-01-01 00:00:00")).toEqual([]); + expect(buildThreads([], "2026-01-01T00:00:00Z")).toEqual([]); }); test("a per-thread read mark clears only that thread", () => { const threads = buildThreads( [ - msg({ chat: "A", handle: "A", ts: "2026-08-30 11:00:00" }), - msg({ chat: "B", handle: "B", ts: "2026-08-30 11:00:00" }), + msg({ chat: "A", handle: "A", ts: "2026-08-30T11:00:00Z" }), + msg({ chat: "B", handle: "B", ts: "2026-08-30T11:00:00Z" }), ], - "2026-08-30 10:00:00", - { A: "2026-08-30 11:00:00" }, + "2026-08-30T10:00:00Z", + { A: "2026-08-30T11:00:00Z" }, ); const byChat = Object.fromEntries(threads.map((t) => [t.chat, t.unread])); expect(byChat).toEqual({ A: 0, B: 1 }); @@ -141,9 +141,9 @@ describe("buildThreads", () => { test("a stale per-thread mark never resurrects unread below the global mark", () => { const threads = buildThreads( - [msg({ chat: "A", handle: "A", ts: "2026-08-30 09:30:00" })], - "2026-08-30 10:00:00", - { A: "2026-08-30 09:00:00" }, + [msg({ chat: "A", handle: "A", ts: "2026-08-30T09:30:00Z" })], + "2026-08-30T10:00:00Z", + { A: "2026-08-30T09:00:00Z" }, ); expect(threads[0]!.unread).toBe(0); }); @@ -180,26 +180,26 @@ describe("self-echo in the thread list", () => { test("a message Fred sends himself does not count as unread", () => { // Without this every panel send to the self-thread re-lit the dot. const msgs = dedupeSelfEcho([ - msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30 11:00:00", from_me: true, text: "note" }), - msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30 11:00:00", from_me: false, text: "note" }), + msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30T11:00:00Z", from_me: true, text: "note" }), + msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30T11:00:00Z", from_me: false, text: "note" }), ], ["+15550100001"]); - const threads = buildThreads(msgs, "2026-08-30 10:00:00"); + const threads = buildThreads(msgs, "2026-08-30T10:00:00Z"); expect(threads[0]!.unread).toBe(0); expect(threads[0]!.last_from_me).toBe(true); }); test("the same text in two different chats at one ts is two messages", () => { const msgs = dedupeSelfEcho([ - msg({ chat: "A", handle: "A", ts: "2026-08-30 11:00:00", text: "ok" }), - msg({ chat: "B", handle: "B", ts: "2026-08-30 11:00:00", text: "ok" }), + msg({ chat: "A", handle: "A", ts: "2026-08-30T11:00:00Z", text: "ok" }), + msg({ chat: "B", handle: "B", ts: "2026-08-30T11:00:00Z", text: "ok" }), ]); expect(msgs).toHaveLength(2); }); test("an empty self row cannot reclassify another chat at the same second", () => { const msgs = dedupeSelfEcho([ - msg({ chat: "SELF", handle: "SELF", from_me: true, text: "", ts: "2026-08-30 11:00:00" }), - msg({ chat: "OTHER", handle: "OTHER", from_me: false, text: "urgent", ts: "2026-08-30 11:00:00" }), + msg({ chat: "SELF", handle: "SELF", from_me: true, text: "", ts: "2026-08-30T11:00:00Z" }), + msg({ chat: "OTHER", handle: "OTHER", from_me: false, text: "urgent", ts: "2026-08-30T11:00:00Z" }), ], ["SELF"]); expect(msgs).toHaveLength(1); expect(msgs[0]!.chat).toBe("OTHER"); @@ -277,8 +277,8 @@ describe("selectToasts", () => { test("a null-chat toast carries the handle, never the string null", () => { const out = selectToasts( - [msg({ chat: null as unknown as string, handle: "+15550100002", ts: "2026-08-30 11:00:00" })], - "2026-08-30 10:00:00", + [msg({ chat: null as unknown as string, handle: "+15550100002", ts: "2026-08-30T11:00:00Z" })], + "2026-08-30T10:00:00Z", allow, [], ); @@ -287,8 +287,8 @@ describe("selectToasts", () => { test("toasts an allowlisted inbound message", () => { const out = selectToasts( - [msg({ chat: "+15550100002", handle: "+15550100002", name: "Alex Rivera", ts: "2026-08-30 11:00:00" })], - "2026-08-30 10:00:00", + [msg({ chat: "+15550100002", handle: "+15550100002", name: "Alex Rivera", ts: "2026-08-30T11:00:00Z" })], + "2026-08-30T10:00:00Z", allow, [], ); @@ -299,8 +299,8 @@ describe("selectToasts", () => { test("drops senders that are not allowlisted", () => { // Bank alerts and 2FA codes are the reason this gate exists. const out = selectToasts( - [msg({ chat: "878478", handle: "878478", ts: "2026-08-30 11:00:00" })], - "2026-08-30 10:00:00", + [msg({ chat: "878478", handle: "878478", ts: "2026-08-30T11:00:00Z" })], + "2026-08-30T10:00:00Z", allow, [], ); @@ -309,8 +309,8 @@ describe("selectToasts", () => { test("never toasts outbound", () => { const out = selectToasts( - [msg({ chat: "+15550100002", handle: "+15550100002", from_me: true, ts: "2026-08-30 11:00:00" })], - "2026-08-30 10:00:00", + [msg({ chat: "+15550100002", handle: "+15550100002", from_me: true, ts: "2026-08-30T11:00:00Z" })], + "2026-08-30T10:00:00Z", allow, [], ); @@ -319,7 +319,7 @@ describe("selectToasts", () => { test("never toasts the backlog on first run", () => { const out = selectToasts( - [msg({ chat: "+15550100002", handle: "+15550100002", ts: "2026-08-30 11:00:00" })], + [msg({ chat: "+15550100002", handle: "+15550100002", ts: "2026-08-30T11:00:00Z" })], "", allow, [], @@ -330,20 +330,20 @@ describe("selectToasts", () => { test("suppresses a message already toasted — the self-thread echo guard", () => { // In the self-thread, the user's sent replies come back as from_me=false. // Without this dedupe the loop would notify on its own output forever. - const m = msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30 11:00:00", text: "echo" }); - const out = selectToasts([m], "2026-08-30 10:00:00", ["+15550100001"], [toastKey(m)]); + const m = msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30T11:00:00Z", text: "echo" }); + const out = selectToasts([m], "2026-08-30T10:00:00Z", ["+15550100001"], [toastKey(m)]); expect(out).toEqual([]); }); test("deduplicates identical messages within a single batch", () => { - const m = msg({ chat: "+15550100002", handle: "+15550100002", ts: "2026-08-30 11:00:00" }); - const out = selectToasts([m, { ...m }], "2026-08-30 10:00:00", allow, []); + const m = msg({ chat: "+15550100002", handle: "+15550100002", ts: "2026-08-30T11:00:00Z" }); + const out = selectToasts([m, { ...m }], "2026-08-30T10:00:00Z", allow, []); expect(out).toHaveLength(1); }); test("persisted toast keys are opaque and distinguish group senders", () => { - const a = msg({ chat: "group", handle: "ALICE", text: "yes", ts: "2026-08-30 11:00:00" }); - const b = msg({ chat: "group", handle: "BOB", text: "yes", ts: "2026-08-30 11:00:00" }); + const a = msg({ chat: "group", handle: "ALICE", text: "yes", ts: "2026-08-30T11:00:00Z" }); + const b = msg({ chat: "group", handle: "BOB", text: "yes", ts: "2026-08-30T11:00:00Z" }); expect(toastKey(a)).toMatch(/^sha256:[0-9a-f]{64}$/); expect(toastKey(a)).not.toContain("yes"); expect(toastKey(a)).not.toBe(toastKey(b)); @@ -351,8 +351,8 @@ describe("selectToasts", () => { test("matches on handle when the chat id is an opaque GUID", () => { const out = selectToasts( - [msg({ chat: "053856bb0d9a40e392db59eace1c56d1", handle: "+15550100004", ts: "2026-08-30 11:00:00" })], - "2026-08-30 10:00:00", + [msg({ chat: "053856bb0d9a40e392db59eace1c56d1", handle: "+15550100004", ts: "2026-08-30T11:00:00Z" })], + "2026-08-30T10:00:00Z", ["+15550100004"], [], ); @@ -361,8 +361,8 @@ describe("selectToasts", () => { test("an empty allowlist toasts nothing", () => { const out = selectToasts( - [msg({ ts: "2026-08-30 11:00:00" })], - "2026-08-30 10:00:00", + [msg({ ts: "2026-08-30T11:00:00Z" })], + "2026-08-30T10:00:00Z", [], [], ); @@ -372,19 +372,19 @@ describe("selectToasts", () => { describe("maxTs", () => { test("returns the highest timestamp", () => { - expect(maxTs([msg({ ts: "2026-08-30 09:00:00" }), msg({ ts: "2026-08-30 11:00:00" })], "")).toBe( - "2026-08-30 11:00:00", + expect(maxTs([msg({ ts: "2026-08-30T09:00:00Z" }), msg({ ts: "2026-08-30T11:00:00Z" })], "")).toBe( + "2026-08-30T11:00:00Z", ); }); test("never moves the watermark backwards on a short window", () => { - expect(maxTs([msg({ ts: "2026-01-01 00:00:00" })], "2026-08-30 10:00:00")).toBe( - "2026-08-30 10:00:00", + expect(maxTs([msg({ ts: "2026-01-01T00:00:00Z" })], "2026-08-30T10:00:00Z")).toBe( + "2026-08-30T10:00:00Z", ); }); test("an empty fetch leaves the watermark untouched", () => { - expect(maxTs([], "2026-08-30 10:00:00")).toBe("2026-08-30 10:00:00"); + expect(maxTs([], "2026-08-30T10:00:00Z")).toBe("2026-08-30T10:00:00Z"); }); }); @@ -393,19 +393,19 @@ describe("state and allowlist I/O", () => { const p = join(tmp(), "state.json"); const opaque = `sha256:${"a".repeat(64)}`; expect(saveState({ - watermark: "2026-08-30 10:00:00", readMark: "2026-08-30 09:00:00", - unreadCounts: { A: 2 }, unreadOldest: { A: "2026-08-30 09:01:00" }, + watermark: "2026-08-30T10:00:00Z", readMark: "2026-08-30T09:00:00Z", + unreadCounts: { A: 2 }, unreadOldest: { A: "2026-08-30T09:01:00Z" }, unreadInitialized: true, selfChats: ["SELF"], - readMarks: { A: "x" }, groups: {}, chatAliases: { OLD: "A" }, pins: { A: 0 }, toasted: [opaque], + readMarks: { A: "2026-08-30T09:30:00Z" }, groups: {}, chatAliases: { OLD: "A" }, pins: { A: 0 }, toasted: [opaque], }, p)).toBe(true); expect(loadState(p)).toEqual({ - watermark: "2026-08-30 10:00:00", - readMark: "2026-08-30 09:00:00", + watermark: "2026-08-30T10:00:00Z", + readMark: "2026-08-30T09:00:00Z", unreadCounts: { A: 2 }, - unreadOldest: { A: "2026-08-30 09:01:00" }, + unreadOldest: { A: "2026-08-30T09:01:00Z" }, unreadInitialized: true, selfChats: ["SELF"], - readMarks: { A: "x" }, + readMarks: { A: "2026-08-30T09:30:00Z" }, groups: {}, chatAliases: { OLD: "A" }, pins: { A: 0 }, @@ -462,16 +462,16 @@ describe("state and allowlist I/O", () => { // Migration guard: an old {watermark, toasted} file must report zero unread, // not a fabricated backlog, the first time the new collector reads it. const p = join(tmp(), "legacy.json"); - writeFileSync(p, JSON.stringify({ watermark: "2026-08-30 10:00:00", toasted: [] })); - expect(loadState(p).readMark).toBe("2026-08-30 10:00:00"); + writeFileSync(p, JSON.stringify({ watermark: "2026-08-30T10:00:00Z", toasted: [] })); + expect(loadState(p).readMark).toBe("2026-08-30T10:00:00Z"); expect(loadState(p).readMarks).toEqual({}); }); test("a count-only ledger is reseeded so deletes can be reconciled", () => { const p = join(tmp(), "count-only.json"); writeFileSync(p, JSON.stringify({ - watermark: "2026-08-30 12:00:00", - readMark: "2026-08-30 10:00:00", + watermark: "2026-08-30T12:00:00Z", + readMark: "2026-08-30T10:00:00Z", unreadCounts: { A: 2 }, unreadInitialized: true, })); @@ -561,10 +561,10 @@ describe("fetchMessages", () => { describe("adaptive unread catch-up", () => { test("expands until the previous watermark is inside the fetched window", () => { const all = [ - msg({ ts: "2026-08-30 12:04:00" }), - msg({ ts: "2026-08-30 12:03:00" }), - msg({ ts: "2026-08-30 12:02:00" }), - msg({ ts: "2026-08-30 11:59:00" }), + msg({ ts: "2026-08-30T12:04:00Z" }), + msg({ ts: "2026-08-30T12:03:00Z" }), + msg({ ts: "2026-08-30T12:02:00Z" }), + msg({ ts: "2026-08-30T11:59:00Z" }), ]; const limits: number[] = []; const fake = ((_cmd: string, args: string[]) => { @@ -572,7 +572,7 @@ describe("adaptive unread catch-up", () => { limits.push(limit); return { status: 0, stdout: JSON.stringify(all.slice(0, limit)), stderr: "" }; }) as never; - const result = fetchMessagesAfter("2026-08-30 12:00:00", 2, fake); + const result = fetchMessagesAfter("2026-08-30T12:00:00Z", 2, fake); expect(result.ok).toBe(true); expect(result.msgs).toHaveLength(4); expect(limits).toEqual([2, 4]); @@ -583,10 +583,10 @@ describe("adaptive unread catch-up", () => { // neither chat nor handle and is dropped. Counting survivors read that as // "the bridge ran out" and stopped before the older unread. const all = [ - msg({ ts: "2026-08-30 12:03:00", handle: "+15550000001" }), - msg({ ts: "2026-08-30 12:02:00", chat: null as unknown as string, handle: null as unknown as string }), - msg({ ts: "2026-08-30 12:01:00", handle: "+15550000002" }), - msg({ ts: "2026-08-30 11:59:00", handle: "+15550000003" }), + msg({ ts: "2026-08-30T12:03:00Z", handle: "+15550000001" }), + msg({ ts: "2026-08-30T12:02:00Z", chat: null as unknown as string, handle: null as unknown as string }), + msg({ ts: "2026-08-30T12:01:00Z", handle: "+15550000002" }), + msg({ ts: "2026-08-30T11:59:00Z", handle: "+15550000003" }), ]; const limits: number[] = []; const fake = ((_cmd: string, args: string[]) => { @@ -594,7 +594,7 @@ describe("adaptive unread catch-up", () => { limits.push(limit); return { status: 0, stdout: JSON.stringify(all.slice(0, limit)), stderr: "" }; }) as never; - const r = fetchMessagesAfter("2026-08-30 12:00:00", 2, fake); + const r = fetchMessagesAfter("2026-08-30T12:00:00Z", 2, fake); expect(r.ok).toBe(true); expect(limits).toEqual([2, 4]); expect(r.fetchedCount).toBe(4); @@ -606,10 +606,10 @@ describe("adaptive unread catch-up", () => { const fake = ((_cmd: string, args: string[]) => { const limit = Number(args[2]); limits.push(limit); - const rows = Array.from({ length: limit }, (_, i) => msg({ ts: "2026-08-30 12:01:00", handle: "H" + i })); + const rows = Array.from({ length: limit }, (_, i) => msg({ ts: "2026-08-30T12:01:00Z", handle: "H" + i })); return { status: 0, stdout: JSON.stringify(rows), stderr: "" }; }) as never; - const r = fetchMessagesAfter("2026-08-30 12:00:00", 2, fake); + const r = fetchMessagesAfter("2026-08-30T12:00:00Z", 2, fake); expect(r.ok).toBe(true); expect(limits[limits.length - 1]).toBeLessThanOrEqual(CATCHUP_MAX_ROWS); expect(limits.length).toBeLessThan(20); @@ -617,9 +617,9 @@ describe("adaptive unread catch-up", () => { test("expands past an equal timestamp boundary", () => { const all = [ - msg({ ts: "2026-08-30 12:01:00", handle: "A" }), - msg({ ts: "2026-08-30 12:00:00", handle: "B" }), - msg({ ts: "2026-08-30 12:00:00", handle: "C" }), + msg({ ts: "2026-08-30T12:01:00Z", handle: "A" }), + msg({ ts: "2026-08-30T12:00:00Z", handle: "B" }), + msg({ ts: "2026-08-30T12:00:00Z", handle: "C" }), ]; const limits: number[] = []; const fake = ((_cmd: string, args: string[]) => { @@ -627,30 +627,30 @@ describe("adaptive unread catch-up", () => { limits.push(limit); return { status: 0, stdout: JSON.stringify(all.slice(0, limit)), stderr: "" }; }) as never; - expect(fetchMessagesAfter("2026-08-30 12:00:00", 2, fake).msgs).toHaveLength(3); + expect(fetchMessagesAfter("2026-08-30T12:00:00Z", 2, fake).msgs).toHaveLength(3); expect(limits).toEqual([2, 4]); }); test("the unread ledger records counts and its reconciliation boundary", () => { const rows = [ - msg({ chat: "A", ts: "2026-08-30 11:00:00" }), - msg({ chat: "A", ts: "2026-08-30 10:30:00" }), - msg({ chat: "A", ts: "2026-08-30 11:01:00", from_me: true }), + msg({ chat: "A", ts: "2026-08-30T11:00:00Z" }), + msg({ chat: "A", ts: "2026-08-30T10:30:00Z" }), + msg({ chat: "A", ts: "2026-08-30T11:01:00Z", from_me: true }), ]; - expect(unreadCounts(rows, "2026-08-30 10:00:00", {})).toEqual({ A: 2 }); - expect(unreadOldest(rows, "2026-08-30 10:00:00", {})).toEqual({ A: "2026-08-30 10:30:00" }); + expect(unreadCounts(rows, "2026-08-30T10:00:00Z", {})).toEqual({ A: 2 }); + expect(unreadOldest(rows, "2026-08-30T10:00:00Z", {})).toEqual({ A: "2026-08-30T10:30:00Z" }); }); test("rebuilding the covered unread range removes deleted rows", () => { - const rowsAfterDelete = [msg({ chat: "A", ts: "2026-08-30 11:00:00" })]; - expect(unreadCounts(rowsAfterDelete, "2026-08-30 10:00:00", {})).toEqual({ A: 1 }); - expect(unreadOldest(rowsAfterDelete, "2026-08-30 10:00:00", {})).toEqual({ A: "2026-08-30 11:00:00" }); + const rowsAfterDelete = [msg({ chat: "A", ts: "2026-08-30T11:00:00Z" })]; + expect(unreadCounts(rowsAfterDelete, "2026-08-30T10:00:00Z", {})).toEqual({ A: 1 }); + expect(unreadOldest(rowsAfterDelete, "2026-08-30T10:00:00Z", {})).toEqual({ A: "2026-08-30T11:00:00Z" }); }); test("exact ledger counts override the bounded thread window", () => { const threads = buildThreads( - [msg({ chat: "A", ts: "2026-08-30 11:00:00" })], - "2026-08-30 10:00:00", {}, {}, { A: 151 }, + [msg({ chat: "A", ts: "2026-08-30T11:00:00Z" })], + "2026-08-30T10:00:00Z", {}, {}, { A: 151 }, ); expect(threads[0]!.unread).toBe(151); }); @@ -659,8 +659,8 @@ describe("adaptive unread catch-up", () => { describe("readMarks pruning", () => { test("a per-thread mark at or below the global mark is dropped from state", () => { // collect() prunes; emulate its rule directly. - const readMarks: Record = { A: "2026-08-30 09:00:00", B: "2026-08-30 12:00:00" }; - const readMark = "2026-08-30 10:00:00"; + const readMarks: Record = { A: "2026-08-30T09:00:00Z", B: "2026-08-30T12:00:00Z" }; + const readMark = "2026-08-30T10:00:00Z"; for (const [chat, ts] of Object.entries(readMarks)) if (ts <= readMark) delete readMarks[chat]; expect(Object.keys(readMarks)).toEqual(["B"]); }); @@ -670,8 +670,8 @@ describe("text-bearing self twins", () => { test("a same-chat same-handle same-second text pair marks the self chat", () => { // imsg decodes attributedBody: the outbound twin is rarely empty. const msgs = [ - msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30 21:45:00", from_me: true, text: "note" }), - msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30 21:45:00", from_me: false, text: "note" }), + msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30T21:45:00Z", from_me: true, text: "note" }), + msg({ chat: "+15550100001", handle: "+15550100001", ts: "2026-08-30T21:45:00Z", from_me: false, text: "note" }), ]; expect(detectSelfChats(msgs)).toEqual(["+15550100001"]); const out = dedupeSelfEcho(msgs); @@ -681,8 +681,8 @@ describe("text-bearing self twins", () => { test("two different senders with the same text in one second stay distinct", () => { const msgs = [ - msg({ chat: "g", handle: "+15550100004", ts: "2026-08-30 21:45:00", from_me: false, text: "lol" }), - msg({ chat: "g", handle: "+15550100005", ts: "2026-08-30 21:45:00", from_me: false, text: "lol" }), + msg({ chat: "g", handle: "+15550100004", ts: "2026-08-30T21:45:00Z", from_me: false, text: "lol" }), + msg({ chat: "g", handle: "+15550100005", ts: "2026-08-30T21:45:00Z", from_me: false, text: "lol" }), ]; expect(detectSelfChats(msgs)).toEqual([]); expect(dedupeSelfEcho(msgs)).toHaveLength(2); @@ -704,25 +704,25 @@ describe("fetchGroups", () => { describe("phone-synced read state (imsg ≥1.9.0 `read`)", () => { const m = (over: Record) => ({ - ts: "2026-08-31 12:00:00", from_me: false, handle: "+15551234567", + ts: "2026-08-31T12:00:00Z", from_me: false, handle: "+15551234567", name: null, service: "iMessage", chat: "+15551234567", text: "x", ...over, }) as never; test("a message read on the PHONE stops counting even past the local mark", () => { const counts = unreadCounts( - [m({ read: true }), m({ ts: "2026-08-31 12:01:00", read: false })], - "2026-08-31 00:00:00", {}, + [m({ read: true }), m({ ts: "2026-08-31T12:01:00Z", read: false })], + "2026-08-31T00:00:00Z", {}, ); expect(counts["+15551234567"]).toBe(1); }); test("rows without the read field fall back to local-only semantics", () => { - const counts = unreadCounts([m({})], "2026-08-31 00:00:00", {}); + const counts = unreadCounts([m({})], "2026-08-31T00:00:00Z", {}); expect(counts["+15551234567"]).toBe(1); }); test("locally-read messages stay read regardless of Apple state", () => { - const counts = unreadCounts([m({ read: false })], "2026-08-31 23:00:00", {}); + const counts = unreadCounts([m({ read: false })], "2026-08-31T23:00:00Z", {}); expect(counts["+15551234567"]).toBeUndefined(); }); }); @@ -730,27 +730,27 @@ describe("phone-synced read state (imsg ≥1.9.0 `read`)", () => { describe("future-dated messages must not poison read marks", () => { test("mark-all clamps the global mark to now; the future chat gets a per-chat mark", () => { // Simulated via collect()'s pieces: verify the clamp math directly. - const { localNowTs } = require("./collector") as typeof import("./collector"); - const now = localNowTs(); - const future = "2099-01-01 00:00:00"; + const { nowTs } = require("./collector") as typeof import("./collector"); + const now = nowTs(); + const future = "2099-01-01T00:00:00Z"; expect(future > now).toBe(true); const readMark = future <= now ? future : now; expect(readMark).toBe(now); // global mark never exceeds the clock }); test("a chat read now stays readable for messages arriving later today", () => { - const { isUnread, localNowTs } = require("./collector") as typeof import("./collector"); - const mark = localNowTs(new Date(Date.now() - 60000)); // read a minute ago - const arriving = { ts: localNowTs(), from_me: false, read: false } as never; + const { isUnread, nowTs } = require("./collector") as typeof import("./collector"); + const mark = nowTs(new Date(Date.now() - 60000)); // read a minute ago + const arriving = { ts: nowTs(), from_me: false, read: false } as never; expect(isUnread(arriving, mark)).toBe(true); // new arrival still badges }); }); describe("failed-delivery detection", () => { const { selectFailures } = require("./collector") as typeof import("./collector"); - const now = "2026-08-31 20:30:00"; + const now = "2026-08-31T20:30:00Z"; const mine = (over: Record) => ({ - ts: "2026-08-31 20:25:00", from_me: true, handle: "+15551234567", name: "Pat", + ts: "2026-08-31T20:25:00Z", from_me: true, handle: "+15551234567", name: "Pat", service: "iMessage", chat: "+15551234567", text: "photo", ...over, }) as never; @@ -765,7 +765,7 @@ describe("failed-delivery detection", () => { test("error 0, inbound rows, and old failures are ignored", () => { expect(selectFailures([mine({ error: 0 })], [], now).length).toBe(0); expect(selectFailures([mine({ error: 25, from_me: false })], [], now).length).toBe(0); - expect(selectFailures([mine({ error: 25, ts: "2026-08-31 19:00:00" })], [], now).length).toBe(0); + expect(selectFailures([mine({ error: 25, ts: "2026-08-31T19:00:00Z" })], [], now).length).toBe(0); }); test("dedupes through the toasted ring — interrupts exactly once", () => { @@ -777,7 +777,7 @@ describe("failed-delivery detection", () => { describe("a link that just arrived opens the share sheet", () => { const { firstUrl, selectIncomingLinks } = require("./collector") as typeof import("./collector"); - const WM = "2026-09-02 12:00:00"; + const WM = "2026-09-02T12:00:00Z"; test("firstUrl finds one http(s) url and drops sentence punctuation", () => { expect(firstUrl("see https://gpc.sc/occ/pay to view bill.")).toBe("https://gpc.sc/occ/pay"); @@ -790,11 +790,11 @@ describe("a link that just arrived opens the share sheet", () => { test("only NEW inbound links, newest last, one key each", () => { const out = selectIncomingLinks([ - msg({ ts: "2026-09-02 12:30:00", from_me: false, text: "read https://a.test/1" }), - msg({ ts: "2026-09-02 11:00:00", from_me: false, text: "old https://b.test/2" }), // before watermark - msg({ ts: "2026-09-02 12:40:00", from_me: true, text: "mine https://c.test/3" }), // outbound - msg({ ts: "2026-09-02 12:50:00", from_me: false, text: "no url" }), - msg({ ts: "2026-09-02 12:55:00", from_me: false, text: "later https://d.test/4" }), + msg({ ts: "2026-09-02T12:30:00Z", from_me: false, text: "read https://a.test/1" }), + msg({ ts: "2026-09-02T11:00:00Z", from_me: false, text: "old https://b.test/2" }), // before watermark + msg({ ts: "2026-09-02T12:40:00Z", from_me: true, text: "mine https://c.test/3" }), // outbound + msg({ ts: "2026-09-02T12:50:00Z", from_me: false, text: "no url" }), + msg({ ts: "2026-09-02T12:55:00Z", from_me: false, text: "later https://d.test/4" }), ], WM, []); expect(out.map((l) => l.url)).toEqual(["https://a.test/1", "https://d.test/4"]); expect(out[out.length - 1]!.url).toBe("https://d.test/4"); // the caller shows the newest @@ -810,14 +810,14 @@ describe("a link that just arrived opens the share sheet", () => { }); test("a link fires once — its key suppresses the next poll", () => { - const m = msg({ ts: "2026-09-02 12:30:00", from_me: false, text: "https://a.test/1" }); + const m = msg({ ts: "2026-09-02T12:30:00Z", from_me: false, text: "https://a.test/1" }); const first = selectIncomingLinks([m], WM, []); expect(first.length).toBe(1); expect(selectIncomingLinks([m], WM, [first[0]!.key])).toEqual([]); }); test("never the backlog on first run, never the self-thread", () => { - const m = msg({ ts: "2026-09-02 12:30:00", from_me: false, text: "https://a.test/1" }); + const m = msg({ ts: "2026-09-02T12:30:00Z", from_me: false, text: "https://a.test/1" }); expect(selectIncomingLinks([m], "", [])).toEqual([]); expect(selectIncomingLinks([m], WM, [], [String(m.chat || m.handle)])).toEqual([]); }); @@ -827,7 +827,7 @@ describe("a re-keyed group is ONE conversation", () => { const { aliasesFromChats, foldThreadAliases, foldChatRecord } = require("./collector") as typeof import("./collector"); const chat = (id: string, aliases: string[] = []) => ({ - id, name: "Sportsball!", service: "iMessage", messages: 3, last: "2026-08-09 21:50:59", + id, name: "Sportsball!", service: "iMessage", messages: 3, last: "2026-08-09T21:50:59Z", last_text: "", last_from_me: false, last_handle: "", last_name: null, pinned: false, pin_order: null, aliases, }); @@ -846,24 +846,24 @@ describe("a re-keyed group is ONE conversation", () => { test("two rows of one group become a single thread, counts summed", () => { const out = foldThreadAliases( - [thread("chat6703", "2026-02-26 22:26:56", 5263, 1), thread("chat2244", "2026-08-09 21:50:59", 3477, 2)], + [thread("chat6703", "2026-02-26T22:26:56Z", 5263, 1), thread("chat2244", "2026-08-09T21:50:59Z", 3477, 2)], { chat6703: "chat2244" }, ); expect(out.length).toBe(1); expect(out[0]!.chat).toBe("chat2244"); - expect(out[0]!.last_ts).toBe("2026-08-09 21:50:59"); // the LIVE row's preview + expect(out[0]!.last_ts).toBe("2026-08-09T21:50:59Z"); // the LIVE row's preview expect(out[0]!.count).toBe(5263 + 3477); expect(out[0]!.unread).toBe(3); }); test("the older row alone still shows, under the live id", () => { - const out = foldThreadAliases([thread("chat6703", "2026-02-26 22:26:56", 5, 0)], { chat6703: "chat2244" }); + const out = foldThreadAliases([thread("chat6703", "2026-02-26T22:26:56Z", 5, 0)], { chat6703: "chat2244" }); expect(out.length).toBe(1); expect(out[0]!.chat).toBe("chat2244"); }); test("no aliases is a no-op (same array back)", () => { - const t = [thread("chat2244", "2026-08-09 21:50:59", 1, 0)]; + const t = [thread("chat2244", "2026-08-09T21:50:59Z", 1, 0)]; expect(foldThreadAliases(t, {})).toBe(t); }); @@ -871,10 +871,10 @@ describe("a re-keyed group is ONE conversation", () => { expect(foldChatRecord({ chat6703: 1, chat2244: 2, "+1555": 4 }, { chat6703: "chat2244" }, (a, b) => a + b)) .toEqual({ chat2244: 3, "+1555": 4 }); expect(foldChatRecord( - { chat6703: "2026-02-26 22:26:56", chat2244: "2026-08-09 21:50:59" }, + { chat6703: "2026-02-26T22:26:56Z", chat2244: "2026-08-09T21:50:59Z" }, { chat6703: "chat2244" }, (a, b) => (a < b ? a : b), - )).toEqual({ chat2244: "2026-02-26 22:26:56" }); + )).toEqual({ chat2244: "2026-02-26T22:26:56Z" }); }); test("a merged DM (phone + email) folds onto the live handle", () => { @@ -896,15 +896,15 @@ describe("complete conversation list (mergeChats)", () => { const { mergeChats } = require("./collector") as typeof import("./collector"); const windowThread = { chat: "+15551234567", guid: "", name: "Pat", handle: "+15551234567", service: "iMessage", - last_ts: "2026-08-31 20:00:00", last_text: "hi", last_from_me: false, count: 3, unread: 1, + last_ts: "2026-08-31T20:00:00Z", last_text: "hi", last_from_me: false, count: 3, unread: 1, pinned: false, pin_order: null, }; const chats = [ - { id: "+15551234567", name: null, service: "iMessage", last: "2026-08-31 20:00:00", + { id: "+15551234567", name: null, service: "iMessage", last: "2026-08-31T20:00:00Z", last_text: "hi", last_from_me: false, last_handle: "+15551234567", last_name: "Pat", pinned: false, pin_order: null }, - { id: "ce5a593a78af408282d61461ade89135", name: "Lunch Crew", service: "iMessage", last: "2026-08-31 19:00:00", + { id: "ce5a593a78af408282d61461ade89135", name: "Lunch Crew", service: "iMessage", last: "2026-08-31T19:00:00Z", last_text: "Nice", last_from_me: false, last_handle: "+15550001111", last_name: "Sam", pinned: false, pin_order: null }, - { id: "+15559990000", name: null, service: "SMS", last: "2026-08-20 09:00:00", + { id: "+15559990000", name: null, service: "SMS", last: "2026-08-20T09:00:00Z", last_text: "old news", last_from_me: true, last_handle: "+15559990000", last_name: "Quiet Q", pinned: false, pin_order: null }, ]; @@ -932,8 +932,8 @@ describe("complete conversation list (mergeChats)", () => { test("mirrored pins sort first in Messages pin order, ahead of activity", () => { const out = mergeChats( [ - { ...windowThread, last_ts: "2026-08-31 22:00:00" }, - { ...windowThread, chat: "+15557770000", last_ts: "2026-08-31 21:00:00", pinned: false, pin_order: null }, + { ...windowThread, last_ts: "2026-08-31T22:00:00Z" }, + { ...windowThread, chat: "+15557770000", last_ts: "2026-08-31T21:00:00Z", pinned: false, pin_order: null }, ], [ { ...chats[0]!, pinned: false, pin_order: null }, @@ -958,12 +958,12 @@ describe("pinned conversation metadata", () => { status: 0, stdout: JSON.stringify([ { - id: "+15551234567", name: "Pat", service: "iMessage", last: "2026-08-31 20:00:00", + id: "+15551234567", name: "Pat", service: "iMessage", last: "2026-08-31T20:00:00Z", last_text: "hi", last_from_me: false, last_handle: "+15551234567", last_name: "Pat", pinned: true, pin_order: 0, }, { - id: "+15550001111", name: null, service: "SMS", last: "2026-08-31 19:00:00", + id: "+15550001111", name: null, service: "SMS", last: "2026-08-31T19:00:00Z", last_text: "old", last_from_me: true, last_handle: "+15550001111", last_name: null, }, ]), @@ -1000,17 +1000,17 @@ describe("explainBridgeError — a dim icon is not a diagnosis", () => { describe("toast identity is stable across polls (2.1.6)", () => { test("the same message with its text decoded on the second poll toasts once", () => { - const first = { ts: "2026-09-01 20:00:05", from_me: false, handle: "+15550001111", name: "T", service: "iMessage", chat: "+15550001111", text: "" } as ImsgMessage; + const first = { ts: "2026-09-01T20:00:05Z", from_me: false, handle: "+15550001111", name: "T", service: "iMessage", chat: "+15550001111", text: "" } as ImsgMessage; const second = { ...first, text: "Ok, I will be there" }; const allow = ["+15550001111"]; - const t1 = selectToasts([first], "2026-09-01 20:00:00", allow, []); + const t1 = selectToasts([first], "2026-09-01T20:00:00Z", allow, []); expect(t1).toHaveLength(1); - const t2 = selectToasts([second], "2026-09-01 20:00:00", allow, [t1[0]!.key]); + const t2 = selectToasts([second], "2026-09-01T20:00:00Z", allow, [t1[0]!.key]); expect(t2).toHaveLength(0); }); test("a bridge ROWID wins over the ts/chat/handle fallback", () => { - const a = { id: 42, ts: "2026-09-01 20:00:05", from_me: false, handle: "h", name: null, service: "iMessage", chat: "h", text: "x" } as ImsgMessage; - const b = { ...a, ts: "2026-09-01 20:00:09", text: "y" }; + const a = { id: 42, ts: "2026-09-01T20:00:05Z", from_me: false, handle: "h", name: null, service: "iMessage", chat: "h", text: "x" } as ImsgMessage; + const b = { ...a, ts: "2026-09-01T20:00:09Z", text: "y" }; expect(toastKey(a)).toBe(toastKey(b)); }); }); @@ -1019,13 +1019,13 @@ describe("self-chat promotion is not persisted on one coincidence (2.2.0)", () = const dm = (ts: string, from_me: boolean, text: string) => ({ ts, from_me, handle: "+15550002222", name: "B", service: "iMessage", chat: "+15550002222", text } as ImsgMessage); test("a single same-second 'ok' pair dedupes for display but does not promote for persistence", () => { - const pair = [dm("2026-09-01 20:00:05", true, "ok"), dm("2026-09-01 20:00:05", false, "ok")]; + const pair = [dm("2026-09-01T20:00:05Z", true, "ok"), dm("2026-09-01T20:00:05Z", false, "ok")]; expect(detectSelfChats(pair)).toEqual(["+15550002222"]); // display-time dedupe still works expect(detectSelfChats(pair, 2)).toEqual([]); // persistence needs a second twin }); test("two twins at different seconds do promote", () => { - const rows = [dm("2026-09-01 20:00:05", true, "a"), dm("2026-09-01 20:00:05", false, "a"), - dm("2026-09-01 20:00:09", true, "b"), dm("2026-09-01 20:00:09", false, "b")]; + const rows = [dm("2026-09-01T20:00:05Z", true, "a"), dm("2026-09-01T20:00:05Z", false, "a"), + dm("2026-09-01T20:00:09Z", true, "b"), dm("2026-09-01T20:00:09Z", false, "b")]; expect(detectSelfChats(rows, 2)).toEqual(["+15550002222"]); }); }); @@ -1033,12 +1033,12 @@ describe("self-chat promotion is not persisted on one coincidence (2.2.0)", () = describe("failure-toast keys survive the ring normalizer (2.2.0)", () => { const { selectFailures } = require("./collector") as typeof import("./collector"); test("a fail: key is kept verbatim on load, so a failed send toasts once", () => { - const m = { id: 9, ts: "2026-09-01 20:00:05", from_me: true, handle: "h", name: "H", service: "iMessage", chat: "h", text: "x", error: 25 } as ImsgMessage; - const first = selectFailures([m], [], "2026-09-01 20:05:00"); + const m = { id: 9, ts: "2026-09-01T20:00:05Z", from_me: true, handle: "h", name: "H", service: "iMessage", chat: "h", text: "x", error: 25 } as ImsgMessage; + const first = selectFailures([m], [], "2026-09-01T20:05:00Z"); expect(first).toHaveLength(1); const tmp = `${process.env.XDG_CACHE_HOME}/state-${process.pid}.json`; saveState({ ...loadState(tmp), toasted: [first[0]!.key] }, tmp); - const again = selectFailures([m], loadState(tmp).toasted, "2026-09-01 20:05:00"); + const again = selectFailures([m], loadState(tmp).toasted, "2026-09-01T20:05:00Z"); expect(again).toHaveLength(0); }); }); @@ -1089,7 +1089,7 @@ describe("which service a DM sends on (@lukejmorrison, PR #4)", () => { const { normalizeSendService, sendServiceForMessages } = require("./collector") as typeof import("./collector"); const at = (min: number, over: Partial = {}) => - msg({ ts: `2026-09-03 1${String(min).padStart(2, "0")}:00:00`, ...over }); + msg({ ts: `2026-09-03T1${String(min).padStart(2, "0")}:00:00Z`, ...over }); test("normalizes what chat.db says into what imsg-send accepts", () => { expect(normalizeSendService("SMS")).toBe("SMS"); @@ -1136,8 +1136,8 @@ describe("which service a DM sends on (@lukejmorrison, PR #4)", () => { test("a group keeps the raw service — it sends by chat-id, not by service", () => { const threads = buildThreads([ msg({ chat: "chat900001", handle: "+15550100011", from_me: false, service: "RCS", - ts: "2026-09-03 10:00:00" }), - ], "2026-09-03 09:00:00"); + ts: "2026-09-03T10:00:00Z" }), + ], "2026-09-03T09:00:00Z"); expect(threads[0]!.service).toBe("RCS"); }); }); @@ -1148,7 +1148,7 @@ describe("pins survive shallow polls", () => { last_from_me: false, count: 1, unread: 0, pinned: false, pin_order: null, ...extra, }); const chat = (id: string, pinned: boolean, pin_order: number | null): ChatInfo => ({ - id, name: id, service: "iMessage", last: "2026-09-03 10:00:00", last_text: "", last_from_me: false, + id, name: id, service: "iMessage", last: "2026-09-03T10:00:00Z", last_text: "", last_from_me: false, last_handle: id, last_name: null, pinned, pin_order, aliases: [], }); @@ -1158,7 +1158,7 @@ describe("pins survive shallow polls", () => { }); test("applyPins re-pins a shallow poll's rows and sorts them first", () => { - const shallow = [thread("NEW", "2026-09-03 12:00:00"), thread("MOM", "2026-09-03 11:00:00"), thread("OLD", "2026-09-03 10:00:00")]; + const shallow = [thread("NEW", "2026-09-03T12:00:00Z"), thread("MOM", "2026-09-03T11:00:00Z"), thread("OLD", "2026-09-03T10:00:00Z")]; const out = applyPins(shallow, { MOM: 0, QUIET: 1 }); expect(out.map((t) => t.chat)).toEqual(["MOM", "NEW", "OLD"]); expect(out[0]).toMatchObject({ pinned: true, pin_order: 0 }); @@ -1168,12 +1168,12 @@ describe("pins survive shallow polls", () => { }); test("applyPins un-pins a row whose pin was removed on the Mac", () => { - const stale = [thread("X", "2026-09-03 12:00:00", { pinned: true, pin_order: 0 }), thread("Y", "2026-09-03 13:00:00")]; + const stale = [thread("X", "2026-09-03T12:00:00Z", { pinned: true, pin_order: 0 }), thread("Y", "2026-09-03T13:00:00Z")]; expect(applyPins(stale, {}).map((t) => [t.chat, t.pinned])).toEqual([["Y", false], ["X", false]]); }); test("applyPins with nothing pinned anywhere is a no-op", () => { - const list = [thread("A", "2026-09-03 12:00:00")]; + const list = [thread("A", "2026-09-03T12:00:00Z")]; expect(applyPins(list, {})).toBe(list); }); @@ -1246,8 +1246,8 @@ describe("mute list", () => { test("one match mutes the whole conversation, not just that message", () => { const window = [ - blast({ ts: "2026-08-30 09:00:00" }), - blast({ ts: "2026-08-30 10:00:00", text: "Are you still with us?" }), + blast({ ts: "2026-08-30T09:00:00Z" }), + blast({ ts: "2026-08-30T10:00:00Z", text: "Are you still with us?" }), msg({ chat: "+15550100002", handle: "+15550100002", text: "lunch?" }), ]; const muted = mutedChats(window, ["Stop2End"]); @@ -1259,7 +1259,7 @@ describe("mute list", () => { test("the chat list drops muted rows too, by id, alias, or preview text", () => { const row = (id: string, last_text: string, aliases: string[] = []): ChatInfo => ({ - id, name: id, service: "SMS", last: "2026-08-30 10:00:00", last_text, + id, name: id, service: "SMS", last: "2026-08-30T10:00:00Z", last_text, last_from_me: false, last_handle: id, last_name: null, pinned: false, pin_order: null, aliases, }); const chats = [ @@ -1273,7 +1273,7 @@ describe("mute list", () => { test("your own reply keeps a chat row alive", () => { const row: ChatInfo = { - id: "+15550100002", name: "Alex Rivera", service: "iMessage", last: "2026-08-30 10:00:00", + id: "+15550100002", name: "Alex Rivera", service: "iMessage", last: "2026-08-30T10:00:00Z", last_text: "ugh, ActBlue again", last_from_me: true, last_handle: "+15550100002", last_name: null, pinned: false, pin_order: null, aliases: [], }; @@ -1285,9 +1285,9 @@ describe("mute list", () => { }); test("a muted conversation never toasts, because it never reaches selectToasts", () => { - const window = [blast({ ts: "2026-08-30 11:00:00" })]; + const window = [blast({ ts: "2026-08-30T11:00:00Z" })]; const kept = dropMuted(window, mutedChats(window, ["ActBlue"])); - expect(selectToasts(kept, "2026-08-30 10:00:00", ["78462"], [])).toEqual([]); + expect(selectToasts(kept, "2026-08-30T10:00:00Z", ["78462"], [])).toEqual([]); }); }); @@ -1341,7 +1341,7 @@ describe("pushRead breadcrumb", () => { describe("security codes: detect, hold once, never from a group", () => { const { extractCode, selectCodes } = require("./collector") as typeof import("./collector"); - const WM = "2026-09-02 12:00:00"; + const WM = "2026-09-02T12:00:00Z"; const code = (text: string) => extractCode(text)?.code ?? null; test("the usual shapes", () => { @@ -1383,13 +1383,13 @@ describe("security codes: detect, hold once, never from a group", () => { }); test("selectCodes: inbound, new, DM only, once", () => { - const m = msg({ ts: "2026-09-02 12:30:00", from_me: false, chat: "77029", handle: "77029", name: null, text: "Your code is 483920" }); + const m = msg({ ts: "2026-09-02T12:30:00Z", from_me: false, chat: "77029", handle: "77029", name: null, text: "Your code is 483920" }); const out = selectCodes([ m, - msg({ ts: "2026-09-02 11:00:00", from_me: false, text: "old code 111111" }), // before watermark - msg({ ts: "2026-09-02 12:40:00", from_me: true, text: "my code is 222222" }), // outbound - msg({ ts: "2026-09-02 12:45:00", from_me: false, chat: "e98633ecd4e84723b69d142cd721b2b9", text: "code 333333" }), // group - msg({ ts: "2026-09-02 12:50:00", from_me: false, chat: "+15550001111", handle: "+15550001111", name: "Eli", text: "Enter passcode 444444" }), + msg({ ts: "2026-09-02T11:00:00Z", from_me: false, text: "old code 111111" }), // before watermark + msg({ ts: "2026-09-02T12:40:00Z", from_me: true, text: "my code is 222222" }), // outbound + msg({ ts: "2026-09-02T12:45:00Z", from_me: false, chat: "e98633ecd4e84723b69d142cd721b2b9", text: "code 333333" }), // group + msg({ ts: "2026-09-02T12:50:00Z", from_me: false, chat: "+15550001111", handle: "+15550001111", name: "Eli", text: "Enter passcode 444444" }), ], WM, []); expect(out.map((c) => [c.code, c.name])).toEqual([["483920", "77029"], ["444444", "Eli"]]); expect(out.every((c) => c.key.startsWith("code:"))).toBe(true); diff --git a/collector.ts b/collector.ts index 79208f1..06b13d8 100644 --- a/collector.ts +++ b/collector.ts @@ -53,7 +53,10 @@ export interface ImsgMessage { id?: number | string; /** Stable Messages GUID when available. */ guid?: string; - ts: string; // "YYYY-MM-DD HH:MM:SS" — lexically sortable, which we rely on + // UTC, ISO-8601, to the second: "2026-09-07T18:33:12Z". Fixed width, so + // lexical order IS chronological order — which every ledger, watermark and + // sort in this file relies on. Local time is a DISPLAY concern (thread.ts). + ts: string; from_me: boolean; /** True only when the Mac bridge knows this is the configured self chat. */ self_chat?: boolean; @@ -210,7 +213,11 @@ export function normalizeGroups(raw: unknown): Record { export function loadState(path = STATE_PATH): BlipState { try { const s = JSON.parse(readFileSync(path, "utf8")) as Partial; - const watermark = typeof s.watermark === "string" ? s.watermark : ""; + // Every persisted stamp predating the UTC wire format is naive Mac-local + // wall clock. Left alone it would sort BELOW every new stamp on the same + // day (" " < "T"), so the whole preview window would read as newer than + // the mark. toUtcStamp() re-anchors it; see there for the exactness. + const watermark = toUtcStamp(typeof s.watermark === "string" ? s.watermark : ""); const unreadCounts = s.unreadCounts && typeof s.unreadCounts === "object" ? Object.fromEntries( Object.entries(s.unreadCounts).filter(([, n]) => typeof n === "number" && Number.isFinite(n) && n >= 0), @@ -218,7 +225,9 @@ export function loadState(path = STATE_PATH): BlipState { : {}; const unreadOldest = s.unreadOldest && typeof s.unreadOldest === "object" ? Object.fromEntries( - Object.entries(s.unreadOldest).filter(([, ts]) => typeof ts === "string" && ts !== ""), + Object.entries(s.unreadOldest) + .filter(([, ts]) => typeof ts === "string" && ts !== "") + .map(([chat, ts]) => [chat, toUtcStamp(ts as string)]), ) : {}; // A count-only ledger from an interrupted/experimental build cannot be @@ -229,14 +238,20 @@ export function loadState(path = STATE_PATH): BlipState { watermark, // Pre-two-mark state files have no readMark. Inheriting the watermark is // the safe migration: it reports zero unread rather than a fake backlog. - readMark: typeof s.readMark === "string" ? s.readMark : watermark, + readMark: toUtcStamp(typeof s.readMark === "string" ? s.readMark : watermark), unreadCounts, unreadOldest, unreadInitialized: s.unreadInitialized === true && ledgerComplete, selfChats: Array.isArray(s.selfChats) ? s.selfChats.filter((chat): chat is string => typeof chat === "string") : [], - readMarks: s.readMarks && typeof s.readMarks === "object" ? { ...s.readMarks } : {}, + readMarks: s.readMarks && typeof s.readMarks === "object" + ? Object.fromEntries( + Object.entries(s.readMarks) + .filter(([, ts]) => typeof ts === "string" && ts !== "") + .map(([chat, ts]) => [chat, toUtcStamp(ts as string)]), + ) + : {}, // Every cached group goes through the same shape fetchGroups() enforces: // a participants OBJECT in state.json threw inside groupName() on every // poll until the next deep refresh (Astra B#6). @@ -937,11 +952,11 @@ export const FAILURE_TOAST_WINDOW_MS = 15 * 60 * 1000; export function selectFailures( msgs: ImsgMessage[], toasted: string[], - nowTs = localNowTs(), + now = nowTs(), ): Toast[] { const seen = new Set(toasted); const out: Toast[] = []; - const cutoff = localNowTs(new Date(Date.parse(nowTs.replace(" ", "T")) - FAILURE_TOAST_WINDOW_MS)); + const cutoff = nowTs(new Date(Date.parse(now) - FAILURE_TOAST_WINDOW_MS)); for (const m of msgs) { if (!m.from_me || typeof m.error !== "number" || m.error === 0) continue; if (m.ts < cutoff) continue; @@ -1041,7 +1056,10 @@ export function fetchMessages(limit: number, runner = spawnSync): FetchResult { try { const parsed = JSON.parse(res.stdout as string); if (!Array.isArray(parsed)) throw new Error("not an array"); - return { ok: true, online: true, error: "", msgs: (parsed as ImsgMessage[]).filter(hasIdentity), fetchedCount: parsed.length }; + // Normalise stamps HERE, the one door messages come through, so nothing + // downstream has to know which bridge version produced them. + const msgs = (parsed as ImsgMessage[]).filter(hasIdentity).map(normalizeMsgStamps); + return { ok: true, online: true, error: "", msgs, fetchedCount: parsed.length }; } catch (e) { return { ok: false, online: true, error: `bad JSON from imsg: ${e}`, msgs: [] }; } @@ -1083,11 +1101,47 @@ export function fetchMessagesAfter( * impossible). * A row without the `read` field (older imsg) falls back to local-only. */ -/** Local wall clock as a chat.db-style "YYYY-MM-DD HH:MM:SS" stamp. */ -export function localNowTs(now = new Date()): string { - const p = (n: number, w = 2) => String(n).padStart(w, "0"); - return `${now.getFullYear()}-${p(now.getMonth() + 1)}-${p(now.getDate())} ` + - `${p(now.getHours())}:${p(now.getMinutes())}:${p(now.getSeconds())}`; +/** + * Now, in the bridge's wire format: UTC ISO-8601 to the second. + * + * This used to be the LINUX wall clock compared against MAC wall-clock + * stamps — the same numbers only while both machines sat in one timezone. + * A Mac an hour ahead put every read mark ahead of every message (nothing + * ever unread); an hour behind and the backlog re-toasted. Both clocks are + * UTC now, so the comparison means what it reads as. + */ +export function nowTs(now = new Date()): string { + return now.toISOString().replace(/\.\d{3}Z$/, "Z"); +} + +/** + * Re-anchor a pre-UTC stamp ("2026-09-07 14:33:12") to the wire format, + * reading it as THIS machine's local time. A stamp already in the wire + * format is returned untouched; junk becomes "". + * + * Two callers, one rule — every stamp inside Blip is UTC: + * - loadState(), for marks written by an older release; + * - the fetch boundary, for a Mac still running the pre-UTC bridge. + * Both matter TOGETHER: migrating the marks while the bridge still emitted + * naive stamps would sort every message below every mark (" " < "T") and + * silently empty the badge and the toasts. + * + * Exact whenever the Mac shares the Linux timezone — every setup in which + * the naive format looked correct in the first place. Where they differ a + * mark lands off by the offset for one poll; the toast ring keys on message + * identity rather than time, so that cannot re-toast a backlog. + */ +export function toUtcStamp(ts: string): string { + if (!ts || ts.includes("T")) return ts; // already UTC, or unset + const ms = Date.parse(ts.replace(" ", "T")); // naive → this machine's local + return Number.isNaN(ms) ? "" : nowTs(new Date(ms)); +} + +/** Every stamp a message carries, normalised to the wire format. */ +export function normalizeMsgStamps(m: T): T { + if (typeof m.ts === "string" && !m.ts.includes("T")) m = { ...m, ts: toUtcStamp(m.ts) }; + if (typeof m.read_at === "string" && !m.read_at.includes("T")) m = { ...m, read_at: toUtcStamp(m.read_at) }; + return m; } export function isUnread(m: ImsgMessage, mark: string): boolean { @@ -1175,7 +1229,7 @@ export function fetchChats(runner = spawnSync): ChatInfo[] | null { id: String(r.id), name: typeof r.name === "string" ? r.name : null, service: String(r.service ?? ""), - last: String(r.last ?? ""), + last: toUtcStamp(String(r.last ?? "")), last_text: String(r.last_text ?? ""), last_from_me: r.last_from_me === true, last_handle: String(r.last_handle ?? ""), @@ -1352,7 +1406,10 @@ export function fetchGroups(runner = spawnSync): Record | nul // ---------------------------------------------------------------- main export function collect(deep: boolean, markRead = false, readChat = "", seenTs = ""): BlipOutput { - const now = new Date().toISOString(); + // When this POLL ran — the output's own metadata, not a message stamp. + // Distinct from nowTs() below, which is the clock message stamps are + // measured against and therefore has to be in the wire format. + const producedAt = new Date().toISOString(); const state = loadState(); // On migration, seed the ledger all the way back to what the user last read. // Thereafter cover both new arrivals and every outstanding unread row. That @@ -1371,7 +1428,7 @@ export function collect(deep: boolean, markRead = false, readChat = "", seenTs = ok: false, online: fetched.online, error: fetched.error, - ts: now, + ts: producedAt, unread: 0, threads: [], toast: [], @@ -1389,7 +1446,7 @@ export function collect(deep: boolean, markRead = false, readChat = "", seenTs = // clamped to the local clock, and each chat that reaches past it gets a // PER-CHAT mark at its own max — every visible message is covered, // nothing beyond now leaks onto other threads. - const nowTs = localNowTs(); + const now = nowTs(); const chatMax: Record = {}; for (const m of fetched.msgs) { const c = chatKey(m); @@ -1397,7 +1454,7 @@ export function collect(deep: boolean, markRead = false, readChat = "", seenTs = } // Badge counts against readMark (what the user has seen); toasts fire against // watermark (what the collector has seen). See BlipState. - const readMark = markRead ? (highest <= nowTs ? highest : nowTs) : state.readMark; + const readMark = markRead ? (highest <= now ? highest : now) : state.readMark; // Opening one conversation clears only that thread's dot — marked with // THAT chat's newest ts, never the global max. const readMarks = { ...state.readMarks }; @@ -1413,7 +1470,7 @@ export function collect(deep: boolean, markRead = false, readChat = "", seenTs = // ts > seen and stays unread. Fallback without --seen: through now, or // the chat's own future row. const own = chatMax[readChat] ?? ""; - readMarks[readChat] = seenTs !== "" ? seenTs : (own > nowTs ? own : nowTs); + readMarks[readChat] = seenTs !== "" ? seenTs : (own > now ? own : now); } const readSeen = readChat ? readMarks[readChat]! : ""; // Group metadata is ~1000 rows; refresh it only on a deep (panel) fetch and @@ -1477,7 +1534,7 @@ export function collect(deep: boolean, markRead = false, readChat = "", seenTs = const foldedWindow = foldThreadAliases(windowThreads, chatAliases); const threads = chats ? mergeChats(foldedWindow, chats, groups, exactCounts) : applyPins(foldedWindow, pins); const toast = selectToasts(msgs, state.watermark, loadAllowlist(), state.toasted); - const failures = selectFailures(fetched.msgs, state.toasted, nowTs); + const failures = selectFailures(fetched.msgs, state.toasted, now); const links = selectIncomingLinks(msgs, state.watermark, state.toasted, selfChats); const codes = selectCodes(msgs, state.watermark, state.toasted, selfChats); const unread = Object.values(exactCounts).reduce((n, count) => n + count, 0); @@ -1486,7 +1543,7 @@ export function collect(deep: boolean, markRead = false, readChat = "", seenTs = // swallow the messages that arrived during it. // A row dated tomorrow (tz skew) must not become the mark everything is // measured against — nothing would badge or toast until "tomorrow" (Astra B#4). - const highestNow = highest <= nowTs ? highest : nowTs; + const highestNow = highest <= now ? highest : now; const persisted = saveState({ watermark: highestNow, // First ever run: adopt the current high-water rather than reporting the @@ -1515,7 +1572,7 @@ export function collect(deep: boolean, markRead = false, readChat = "", seenTs = ok: true, online: true, error: warning, - ts: now, + ts: producedAt, unread, threads, // Never emit notifications that could not be committed to the dedupe ring. diff --git a/test-setup.ts b/test-setup.ts index ea03640..ee6e7d5 100644 --- a/test-setup.ts +++ b/test-setup.ts @@ -4,3 +4,8 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; process.env.XDG_CACHE_HOME = mkdtempSync(join(tmpdir(), "blip-test-cache-")); + +// Labels are rendered in LOCAL time from UTC wire stamps, so an unpinned TZ +// would make every clock/day assertion depend on where the test ran. Pin it; +// the tests that are ABOUT the conversion set their own zone explicitly. +process.env.TZ ||= "UTC"; diff --git a/thread.test.ts b/thread.test.ts index 19900d1..ea6aa03 100644 --- a/thread.test.ts +++ b/thread.test.ts @@ -24,7 +24,7 @@ import { cliChatArg } from "./thread"; function msg(over: Partial = {}): ImsgMessage { return { - ts: "2026-08-30 12:00:00", + ts: "2026-08-30T12:00:00Z", from_me: false, handle: "+15551234567", name: "Test Person", @@ -38,7 +38,7 @@ function msg(over: Partial = {}): ImsgMessage { const DOTTED: Formats = { time: "HH:mm", date: "dd.MM", dateWithYear: "dd.MM.yyyy" }; describe("formatStamp", () => { - const ts = "2026-08-30 21:08:22"; // a Sunday + const ts = "2026-08-30T21:08:22Z"; // a Sunday test.each([ ["HH:mm", "21:08"], @@ -63,8 +63,8 @@ describe("formatStamp", () => { }); test("midnight and noon on a 12-hour clock", () => { - expect(formatStamp("2026-08-30 00:30:00", "h:mm AP")).toBe("12:30 AM"); - expect(formatStamp("2026-08-30 12:00:00", "h:mm AP")).toBe("12:00 PM"); + expect(formatStamp("2026-08-30T00:30:00Z", "h:mm AP")).toBe("12:30 AM"); + expect(formatStamp("2026-08-30T12:00:00Z", "h:mm AP")).toBe("12:00 PM"); }); test("a date-only stamp formats at midnight", () => { @@ -92,23 +92,23 @@ describe("formatsFromArgv", () => { describe("clockLabel", () => { test("formats afternoon as 12-hour with PM", () => { - expect(clockLabel("2026-08-30 21:08:22")).toBe("9:08 PM"); + expect(clockLabel("2026-08-30T21:08:22Z")).toBe("9:08 PM"); }); test("formats morning as AM", () => { - expect(clockLabel("2026-08-30 09:05:00")).toBe("9:05 AM"); + expect(clockLabel("2026-08-30T09:05:00Z")).toBe("9:05 AM"); }); test("midnight is 12 AM, not 0 AM", () => { - expect(clockLabel("2026-08-30 00:30:00")).toBe("12:30 AM"); + expect(clockLabel("2026-08-30T00:30:00Z")).toBe("12:30 AM"); }); test("noon is 12 PM, not 0 PM", () => { - expect(clockLabel("2026-08-30 12:00:00")).toBe("12:00 PM"); + expect(clockLabel("2026-08-30T12:00:00Z")).toBe("12:00 PM"); }); test("follows the time pattern it is given", () => { - expect(clockLabel("2026-08-30 21:08:22", "HH:mm")).toBe("21:08"); + expect(clockLabel("2026-08-30T21:08:22Z", "HH:mm")).toBe("21:08"); }); test("garbage in yields an empty label, not a crash", () => { @@ -120,30 +120,30 @@ describe("dayLabel", () => { const today = "2026-08-30"; test("same date reads Today", () => { - expect(dayLabel("2026-08-30 09:00:00", today)).toBe("Today"); + expect(dayLabel("2026-08-30T09:00:00Z", today)).toBe("Today"); }); test("one day back reads Yesterday", () => { - expect(dayLabel("2026-08-29 09:00:00", today)).toBe("Yesterday"); + expect(dayLabel("2026-08-29T09:00:00Z", today)).toBe("Yesterday"); }); test("earlier this year omits the year", () => { - expect(dayLabel("2026-08-28 09:00:00", today)).toBe("Aug 28"); + expect(dayLabel("2026-08-28T09:00:00Z", today)).toBe("Aug 28"); }); test("a previous year includes it", () => { - expect(dayLabel("2025-12-24 09:00:00", today)).toBe("Dec 24, 2025"); + expect(dayLabel("2025-12-24T09:00:00Z", today)).toBe("Dec 24, 2025"); }); test("dates follow the patterns, Today and Yesterday stay words", () => { - expect(dayLabel("2026-08-30 09:00:00", today, DOTTED)).toBe("Today"); - expect(dayLabel("2026-08-29 09:00:00", today, DOTTED)).toBe("Yesterday"); - expect(dayLabel("2026-08-28 09:00:00", today, DOTTED)).toBe("28.08"); - expect(dayLabel("2025-12-24 09:00:00", today, DOTTED)).toBe("24.12.2025"); + expect(dayLabel("2026-08-30T09:00:00Z", today, DOTTED)).toBe("Today"); + expect(dayLabel("2026-08-29T09:00:00Z", today, DOTTED)).toBe("Yesterday"); + expect(dayLabel("2026-08-28T09:00:00Z", today, DOTTED)).toBe("28.08"); + expect(dayLabel("2025-12-24T09:00:00Z", today, DOTTED)).toBe("24.12.2025"); }); test("crossing a month boundary still resolves Yesterday", () => { - expect(dayLabel("2026-07-31 09:00:00", "2026-08-01")).toBe("Yesterday"); + expect(dayLabel("2026-07-31T09:00:00Z", "2026-08-01")).toBe("Yesterday"); }); test("malformed input yields an empty label", () => { @@ -153,15 +153,15 @@ describe("dayLabel", () => { describe("minutesBetween", () => { test("measures a simple gap", () => { - expect(minutesBetween("2026-08-30 12:00:00", "2026-08-30 12:20:00")).toBe(20); + expect(minutesBetween("2026-08-30T12:00:00Z", "2026-08-30T12:20:00Z")).toBe(20); }); test("is order-independent", () => { - expect(minutesBetween("2026-08-30 12:20:00", "2026-08-30 12:00:00")).toBe(20); + expect(minutesBetween("2026-08-30T12:20:00Z", "2026-08-30T12:00:00Z")).toBe(20); }); test("unparseable stamps read as infinitely far apart, forcing a new group", () => { - expect(minutesBetween("junk", "2026-08-30 12:00:00")).toBe(Number.POSITIVE_INFINITY); + expect(minutesBetween("junk", "2026-08-30T12:00:00Z")).toBe(Number.POSITIVE_INFINITY); }); }); @@ -169,7 +169,7 @@ describe("decorate", () => { const today = "2026-08-30"; test("a lone message is both group start and end", () => { - const b = decorate([msg({ ts: "2026-08-30 12:00:00" })], today); + const b = decorate([msg({ ts: "2026-08-30T12:00:00Z" })], today); expect(b[0]!.groupStart).toBe(true); expect(b[0]!.groupEnd).toBe(true); expect(b[0]!.time).toBe("12:00 PM"); @@ -177,7 +177,7 @@ describe("decorate", () => { test("the patterns reach the divider, the timestamp and the read receipt", () => { const b = decorate( - [msg({ ts: "2026-08-28 16:42:00", from_me: true, read_at: "2026-08-28 16:45:00" })], + [msg({ ts: "2026-08-28T16:42:00Z", from_me: true, read_at: "2026-08-28T16:45:00Z" })], today, DOTTED, ); @@ -189,9 +189,9 @@ describe("decorate", () => { test("consecutive messages from one sender form a single group", () => { const b = decorate( [ - msg({ ts: "2026-08-30 12:00:00", text: "one" }), - msg({ ts: "2026-08-30 12:01:00", text: "two" }), - msg({ ts: "2026-08-30 12:02:00", text: "three" }), + msg({ ts: "2026-08-30T12:00:00Z", text: "one" }), + msg({ ts: "2026-08-30T12:01:00Z", text: "two" }), + msg({ ts: "2026-08-30T12:02:00Z", text: "three" }), ], today, ); @@ -201,7 +201,7 @@ describe("decorate", () => { test("only the last bubble of a group carries a timestamp", () => { const b = decorate( - [msg({ ts: "2026-08-30 12:00:00" }), msg({ ts: "2026-08-30 12:01:00" })], + [msg({ ts: "2026-08-30T12:00:00Z" }), msg({ ts: "2026-08-30T12:01:00Z" })], today, ); expect(b[0]!.time).toBe(""); @@ -211,8 +211,8 @@ describe("decorate", () => { test("a sender change breaks the group", () => { const b = decorate( [ - msg({ ts: "2026-08-30 12:00:00", from_me: false }), - msg({ ts: "2026-08-30 12:01:00", from_me: true }), + msg({ ts: "2026-08-30T12:00:00Z", from_me: false }), + msg({ ts: "2026-08-30T12:01:00Z", from_me: true }), ], today, ); @@ -223,8 +223,8 @@ describe("decorate", () => { test("in a group, a different member breaks the run even seconds apart", () => { const b = decorate( [ - msg({ ts: "2026-08-30 12:00:00", handle: "+1111111111", name: "Jordan" }), - msg({ ts: "2026-08-30 12:00:30", handle: "+2222222222", name: "Casey" }), + msg({ ts: "2026-08-30T12:00:00Z", handle: "+1111111111", name: "Jordan" }), + msg({ ts: "2026-08-30T12:00:30Z", handle: "+2222222222", name: "Casey" }), ], today, ); @@ -234,7 +234,7 @@ describe("decorate", () => { test("a gap longer than 15 minutes breaks the group", () => { const b = decorate( - [msg({ ts: "2026-08-30 12:00:00" }), msg({ ts: "2026-08-30 12:20:00" })], + [msg({ ts: "2026-08-30T12:00:00Z" }), msg({ ts: "2026-08-30T12:20:00Z" })], today, ); expect(b[1]!.groupStart).toBe(true); @@ -242,7 +242,7 @@ describe("decorate", () => { test("a gap under 15 minutes does not", () => { const b = decorate( - [msg({ ts: "2026-08-30 12:00:00" }), msg({ ts: "2026-08-30 12:14:00" })], + [msg({ ts: "2026-08-30T12:00:00Z" }), msg({ ts: "2026-08-30T12:14:00Z" })], today, ); expect(b[1]!.groupStart).toBe(false); @@ -251,9 +251,9 @@ describe("decorate", () => { test("the day label appears once, on the first message of that day", () => { const b = decorate( [ - msg({ ts: "2026-08-29 23:00:00" }), - msg({ ts: "2026-08-30 09:00:00" }), - msg({ ts: "2026-08-30 09:01:00" }), + msg({ ts: "2026-08-29T23:00:00Z" }), + msg({ ts: "2026-08-30T09:00:00Z" }), + msg({ ts: "2026-08-30T09:01:00Z" }), ], today, ); @@ -264,7 +264,7 @@ describe("decorate", () => { test("a day change always starts a new group even within 15 minutes", () => { const b = decorate( - [msg({ ts: "2026-08-29 23:59:00" }), msg({ ts: "2026-08-30 00:01:00" })], + [msg({ ts: "2026-08-29T23:59:00Z" }), msg({ ts: "2026-08-30T00:01:00Z" })], today, ); expect(b[1]!.groupStart).toBe(true); @@ -286,8 +286,8 @@ describe("dedupeSelfEcho", () => { // A real send logs twice: from_me=true with empty text, and from_me=false // carrying the text. Rendering both shows your own message as theirs. const out = dedupeSelfEcho([ - msg({ ts: "2026-08-30 21:08:22", from_me: true, text: "" }), - msg({ ts: "2026-08-30 21:08:22", from_me: false, text: "Larry test" }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: true, text: "" }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: false, text: "Larry test" }), ]); expect(out).toHaveLength(1); expect(out[0]!.text).toBe("Larry test"); @@ -295,8 +295,8 @@ describe("dedupeSelfEcho", () => { test("the surviving twin is re-attributed to me — it renders on the right", () => { const out = dedupeSelfEcho([ - msg({ ts: "2026-08-30 21:08:22", from_me: true, text: "" }), - msg({ ts: "2026-08-30 21:08:22", from_me: false, text: "mine" }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: true, text: "" }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: false, text: "mine" }), ]); expect(out[0]!.from_me).toBe(true); }); @@ -305,14 +305,14 @@ describe("dedupeSelfEcho", () => { // imsg decodes attributedBody, so the from_me=true twin has text too. const self = msg().chat; const a = dedupeSelfEcho([ - msg({ ts: "2026-08-30 21:08:22", from_me: false, text: "same" }), - msg({ ts: "2026-08-30 21:08:22", from_me: true, text: "same" }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: false, text: "same" }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: true, text: "same" }), ], [self]); expect(a).toHaveLength(1); expect(a[0]!.from_me).toBe(true); const b = dedupeSelfEcho([ - msg({ ts: "2026-08-30 21:08:22", from_me: true, text: "same" }), - msg({ ts: "2026-08-30 21:08:22", from_me: false, text: "same" }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: true, text: "same" }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: false, text: "same" }), ], [self]); expect(b[0]!.from_me).toBe(true); }); @@ -322,8 +322,8 @@ describe("dedupeSelfEcho", () => { // carries the text. Same shape as the echo case, opposite truth. const self = msg().chat; const out = dedupeSelfEcho([ - msg({ ts: "2026-08-30 21:08:22", from_me: true, text: "", retracted: true }), - msg({ ts: "2026-08-30 21:08:22", from_me: false, text: "taken back" }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: true, text: "", retracted: true }), + msg({ ts: "2026-08-30T21:08:22Z", from_me: false, text: "taken back" }), ], [self]); expect(out).toHaveLength(1); expect(out[0]!.retracted).toBe(true); @@ -332,30 +332,30 @@ describe("dedupeSelfEcho", () => { }); test("a genuine inbound with no empty twin stays theirs", () => { - const out = dedupeSelfEcho([msg({ ts: "2026-08-30 21:08:22", from_me: false, text: "theirs" })]); + const out = dedupeSelfEcho([msg({ ts: "2026-08-30T21:08:22Z", from_me: false, text: "theirs" })]); expect(out[0]!.from_me).toBe(false); }); test("keeps legitimate same-direction duplicates at the same timestamp", () => { const out = dedupeSelfEcho([ - msg({ ts: "2026-08-30 21:08:22", text: "same" }), - msg({ ts: "2026-08-30 21:08:22", text: "same" }), + msg({ ts: "2026-08-30T21:08:22Z", text: "same" }), + msg({ ts: "2026-08-30T21:08:22Z", text: "same" }), ]); expect(out).toHaveLength(2); }); test("keeps identical text sent at different times", () => { const out = dedupeSelfEcho([ - msg({ ts: "2026-08-30 21:08:22", text: "ok" }), - msg({ ts: "2026-08-30 21:09:00", text: "ok" }), + msg({ ts: "2026-08-30T21:08:22Z", text: "ok" }), + msg({ ts: "2026-08-30T21:09:00Z", text: "ok" }), ]); expect(out).toHaveLength(2); }); test("leaves an ordinary conversation untouched", () => { const out = dedupeSelfEcho([ - msg({ ts: "2026-08-30 12:00:00", text: "hi" }), - msg({ ts: "2026-08-30 12:01:00", from_me: true, text: "hey" }), + msg({ ts: "2026-08-30T12:00:00Z", text: "hi" }), + msg({ ts: "2026-08-30T12:01:00Z", from_me: true, text: "hey" }), ]); expect(out).toHaveLength(2); }); @@ -371,7 +371,7 @@ describe("loadThread", () => { 10, "2026-08-30", DEFAULT_FORMATS, - fake({ status: 0, stdout: JSON.stringify([msg({ ts: "2026-08-30 12:00:00" })]) }), + fake({ status: 0, stdout: JSON.stringify([msg({ ts: "2026-08-30T12:00:00Z" })]) }), ); expect(r.ok).toBe(true); expect(r.bubbles).toHaveLength(1); @@ -414,9 +414,9 @@ describe("selectThread", () => { test("a group filters a mixed recent window down to its own chat, oldest-first", () => { const out = selectThread( [ - msg({ chat: guid, ts: "2026-08-30 12:05:00", text: "late" }), - msg({ chat: "+15550000000", ts: "2026-08-30 12:04:00", text: "other chat" }), - msg({ chat: guid, ts: "2026-08-30 12:00:00", text: "early" }), + msg({ chat: guid, ts: "2026-08-30T12:05:00Z", text: "late" }), + msg({ chat: "+15550000000", ts: "2026-08-30T12:04:00Z", text: "other chat" }), + msg({ chat: guid, ts: "2026-08-30T12:00:00Z", text: "early" }), ], guid, true, 80, ); @@ -430,9 +430,9 @@ describe("selectThread", () => { const alias = "a1b2c3d4e5f60718293a4b5c6d7e8f90"; const out = selectThread( [ - msg({ chat: guid, ts: "2026-08-30 12:05:00", text: "new row" }), - msg({ chat: alias, ts: "2026-08-30 12:00:00", text: "old row, before the re-key" }), - msg({ chat: "+15550000000", ts: "2026-08-30 12:04:00", text: "a DM, never" }), + msg({ chat: guid, ts: "2026-08-30T12:05:00Z", text: "new row" }), + msg({ chat: alias, ts: "2026-08-30T12:00:00Z", text: "old row, before the re-key" }), + msg({ chat: "+15550000000", ts: "2026-08-30T12:04:00Z", text: "a DM, never" }), ], guid, true, 80, ); @@ -440,7 +440,7 @@ describe("selectThread", () => { }); test("keeps only the newest `limit` messages", () => { - const raw = Array.from({ length: 5 }, (_, i) => msg({ chat: guid, ts: `2026-08-30 12:0${i}:00`, text: `m${i}` })); + const raw = Array.from({ length: 5 }, (_, i) => msg({ chat: guid, ts: `2026-08-30T12:0${i}:00Z`, text: `m${i}` })); expect(selectThread(raw, guid, true, 2).map((m) => m.text)).toEqual(["m3", "m4"]); }); @@ -458,8 +458,8 @@ describe("selectThread", () => { }); test("a DM window passes through untouched apart from ordering", () => { - const out = selectThread([msg({ ts: "2026-08-30 12:01:00" }), msg({ ts: "2026-08-30 12:00:00" })], "+15551234567", false, 80); - expect(out[0]!.ts).toBe("2026-08-30 12:00:00"); + const out = selectThread([msg({ ts: "2026-08-30T12:01:00Z" }), msg({ ts: "2026-08-30T12:00:00Z" })], "+15551234567", false, 80); + expect(out[0]!.ts).toBe("2026-08-30T12:00:00Z"); }); test("a merged DM keeps rows from every alias handle, in either direction", () => { @@ -530,10 +530,10 @@ describe("rich decoration", () => { test("receipt lands only on the NEWEST read from-me bubble", () => { const out = decorate( [ - msg({ ts: "2026-08-31 10:00:00", from_me: true, read_at: "2026-08-31 10:00:30" }), - msg({ ts: "2026-08-31 10:05:00", from_me: false }), - msg({ ts: "2026-08-31 10:06:00", from_me: true, read_at: "2026-08-31 10:07:12" }), - msg({ ts: "2026-08-31 10:08:00", from_me: true }), // sent, not yet read + msg({ ts: "2026-08-31T10:00:00Z", from_me: true, read_at: "2026-08-31T10:00:30Z" }), + msg({ ts: "2026-08-31T10:05:00Z", from_me: false }), + msg({ ts: "2026-08-31T10:06:00Z", from_me: true, read_at: "2026-08-31T10:07:12Z" }), + msg({ ts: "2026-08-31T10:08:00Z", from_me: true }), // sent, not yet read ], today, ); @@ -541,8 +541,8 @@ describe("rich decoration", () => { }); test("receipt from an earlier day carries the day label", () => { - expect(receiptLabel("2026-08-30 21:03:00", today)).toBe("Read Yesterday 9:03 PM"); - expect(receiptLabel("2026-08-31 16:42:00", today)).toBe("Read 4:42 PM"); + expect(receiptLabel("2026-08-30T21:03:00Z", today)).toBe("Read Yesterday 9:03 PM"); + expect(receiptLabel("2026-08-31T16:42:00Z", today)).toBe("Read 4:42 PM"); }); test("link-preview payload pseudo-attachments are filtered out", () => { @@ -577,7 +577,7 @@ describe("rich decoration", () => { edited: true, effect: "confetti", }), - msg({ ts: "2026-08-31 10:01:00", retracted: true }), + msg({ ts: "2026-08-31T10:01:00Z", retracted: true }), ], today, ); @@ -622,8 +622,8 @@ describe("pending sends (the bubble drawn before the Mac writes the row)", () => const today = "2026-08-30"; const now = Date.parse("2026-08-30T12:00:30"); const real = (over: Partial = {}): Bubble => - ({ ...decorate([msg({ from_me: true, text: "on my way", ts: "2026-08-30 11:59:00" })], today)[0]!, ...over }); - const send = (over: Partial = {}): PendingSend => ({ chat: "+15551234567", text: "see you soon", ts: "2026-08-30 12:00:00", ...over }); + ({ ...decorate([msg({ from_me: true, text: "on my way", ts: "2026-08-30T11:59:00Z" })], today)[0]!, ...over }); + const send = (over: Partial = {}): PendingSend => ({ chat: "+15551234567", text: "see you soon", ts: "2026-08-30T12:00:00Z", ...over }); test("an unresolved send is appended as a pending bubble that joins your run", () => { const r = withPendingSends([real()], [send()], today, DEFAULT_FORMATS, now); @@ -647,14 +647,14 @@ describe("pending sends (the bubble drawn before the Mac writes the row)", () => }); test("a reply from them starts a new run", () => { - const theirs = decorate([msg({ from_me: false, text: "where are you", ts: "2026-08-30 11:59:00" })], today)[0]!; + const theirs = decorate([msg({ from_me: false, text: "where are you", ts: "2026-08-30T11:59:00Z" })], today)[0]!; const r = withPendingSends([theirs], [send()], today, DEFAULT_FORMATS, now); expect(r.bubbles[1]!.groupStart).toBe(true); expect(r.bubbles[0]!.time).toBe("11:59 AM"); }); test("the real row resolves the send and nothing is appended", () => { - const landed = real({ text: "see you soon", ts: "2026-08-30 12:00:01" }); + const landed = real({ text: "see you soon", ts: "2026-08-30T12:00:01Z" }); const r = withPendingSends([real(), landed], [send()], today, DEFAULT_FORMATS, now); expect(r.bubbles).toHaveLength(2); expect(r.bubbles.some((b) => b.pending)).toBe(false); @@ -662,8 +662,8 @@ describe("pending sends (the bubble drawn before the Mac writes the row)", () => }); test("the same text sent twice waits for two rows", () => { - const landed = real({ text: "ok", ts: "2026-08-30 12:00:01" }); - const twice = [send({ text: "ok", ts: "2026-08-30 12:00:00" }), send({ text: "ok", ts: "2026-08-30 12:00:05" })]; + const landed = real({ text: "ok", ts: "2026-08-30T12:00:01Z" }); + const twice = [send({ text: "ok", ts: "2026-08-30T12:00:00Z" }), send({ text: "ok", ts: "2026-08-30T12:00:05Z" })]; const r = withPendingSends([landed], twice, today, DEFAULT_FORMATS, now); expect(r.bubbles).toHaveLength(2); expect(r.bubbles[1]!.pending).toBe(true); @@ -671,27 +671,27 @@ describe("pending sends (the bubble drawn before the Mac writes the row)", () => }); test("an identical message from yesterday does not resolve today's send", () => { - const old = real({ text: "see you soon", ts: "2026-08-29 12:00:00" }); + const old = real({ text: "see you soon", ts: "2026-08-29T12:00:00Z" }); const r = withPendingSends([old], [send()], today, DEFAULT_FORMATS, now); expect(r.bubbles).toHaveLength(2); expect(r.bubbles[1]!.pending).toBe(true); }); test("a Mac timestamp a little behind the Linux clock still resolves", () => { - const landed = real({ text: "see you soon", ts: "2026-08-30 11:58:30" }); + const landed = real({ text: "see you soon", ts: "2026-08-30T11:58:30Z" }); const r = withPendingSends([landed], [send()], today, DEFAULT_FORMATS, now); expect(r.pending).toEqual([]); }); test("a send two minutes old without a row is given up on, quietly", () => { - const stale = send({ ts: "2026-08-30 11:58:00" }); + const stale = send({ ts: "2026-08-30T11:58:00Z" }); const r = withPendingSends([real()], [stale], today, DEFAULT_FORMATS, now); expect(r.bubbles).toHaveLength(1); expect(r.pending).toEqual([]); }); test("a retracted or already-pending bubble never resolves a send", () => { - const gone = real({ text: "see you soon", ts: "2026-08-30 12:00:01", retracted: true }); + const gone = real({ text: "see you soon", ts: "2026-08-30T12:00:01Z", retracted: true }); const r = withPendingSends([gone], [send()], today, DEFAULT_FORMATS, now); expect(r.pending).toHaveLength(1); const again = withPendingSends(r.bubbles, [send()], today, DEFAULT_FORMATS, now); diff --git a/thread.ts b/thread.ts index 194b845..4d1ed8a 100644 --- a/thread.ts +++ b/thread.ts @@ -18,6 +18,7 @@ import { dedupeSelfEcho, isGroupChat, loadState, + normalizeMsgStamps, type AttachmentMeta, type ImsgMessage, type LinkCard, @@ -94,7 +95,8 @@ export interface Bubble { } /** A send in flight: what was typed, where, and when Enter was pressed - * (local "YYYY-MM-DD HH:MM:SS"). Lives in BarWidget memory only. */ + * (the wire format, UTC — it is compared against real message stamps). + * Lives in BarWidget memory only. */ export interface PendingSend { chat: string; text: string; ts: string } export interface ThreadOutput { @@ -140,24 +142,61 @@ const MONTHS = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]; const DAYS = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]; +/** A wire stamp shaped like the bridge emits, or the pre-UTC stamp it used to. */ +const STAMP = /^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2}))?Z?)?$/; + +/** + * A wire stamp as epoch ms. + * + * The bridge emits UTC ("2026-09-07T18:33:12Z"). A stamp without the marker + * is the pre-UTC naive format and is read as LOCAL, so a Mac still running an + * old bridge degrades to the behaviour it always had rather than jumping by + * the whole UTC offset. A bare date is local midnight — parsing it by the + * ISO rule would make it UTC midnight, i.e. the day before, west of Greenwich. + */ +export function stampMs(ts: string): number { + const s = String(ts ?? ""); + if (!STAMP.test(s)) return Number.NaN; + if (s.length === 10) return Date.parse(`${s}T00:00:00`); + return Date.parse(s.includes("T") ? s : s.replace(" ", "T")); +} + +/** A wire stamp's fields in THIS machine's local time — the display side of UTC. */ +function localFields(ts: string) { + const ms = stampMs(ts); + if (Number.isNaN(ms)) return null; + const d = new Date(ms); + const p = (n: number, w = 2) => String(n).padStart(w, "0"); + return { + year: p(d.getFullYear(), 4), month: p(d.getMonth() + 1), day: p(d.getDate()), + hour: p(d.getHours()), minute: p(d.getMinutes()), second: p(d.getSeconds()), + weekday: d.getDay(), + }; +} + +/** The local calendar date a wire stamp falls on, "" when unparseable. */ +export function localDay(ts: string): string { + const f = localFields(ts); + return f ? `${f.year}-${f.month}-${f.day}` : ""; +} + /** - * Qt's QDateTime::toString() for a "YYYY-MM-DD HH:MM:SS" stamp, the tokens - * Blip needs: d dd ddd dddd · M MM MMM MMMM · yy yyyy · h hh H HH · m mm · s ss · + * Qt's QDateTime::toString() for a wire stamp, the tokens Blip needs: + * d dd ddd dddd · M MM MMM MMMM · yy yyyy · h hh H HH · m mm · s ss · * AP ap A a, and 'quoted literals' with '' as an apostrophe. h/hh are 12-hour - * when an AM/PM token is present, 24-hour otherwise, as in Qt. Works on the - * string so a Mac-local stamp never meets the Linux time zone. Malformed - * input formats to "". + * when an AM/PM token is present, 24-hour otherwise, as in Qt. The stamp is + * UTC on the wire and every label is LOCAL, so this converts first — a + * message is shown at the time the reader's own clock said. Malformed input + * formats to "". */ export function formatStamp(ts: string, pattern: string): string { - const m = /^(\d{4})-(\d{2})-(\d{2})(?: (\d{2}):(\d{2})(?::(\d{2}))?)?/.exec(ts); - if (!m) return ""; - const [year, month, day, hour = "00", minute = "00", second = "00"] = - m.slice(1) as [string, string, string, string?, string?, string?]; + const f = localFields(ts); + if (!f) return ""; + const { year, month, day, hour, minute, second, weekday } = f; const h24 = Number(hour); const h12 = h24 % 12 || 12; // 0 and 12 read as 12 const ampm = h24 < 12 ? "am" : "pm"; const twelveHour = /[Aa]/.test(pattern.replace(/'(?:[^']|'')*'/g, "")); - const weekday = new Date(Date.UTC(Number(year), Number(month) - 1, Number(day))).getUTCDay(); const field = (v: string, n: number) => (n === 1 ? String(Number(v)) : v); // "08" → "8" | "08" const clock = (h: number, n: number) => (n === 1 ? String(h) : String(h).padStart(2, "0")); @@ -191,8 +230,8 @@ export function clockLabel(ts: string, time = DEFAULT_FORMATS.time): string { /** "Today" / "Yesterday", else the date — with the year when it is not this year. */ export function dayLabel(ts: string, today: string, formats = DEFAULT_FORMATS): string { - const date = ts.slice(0, 10); - if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) return ""; + const date = localDay(ts); // the reader's day, not UTC's + if (!date) return ""; if (date === today) return "Today"; const d = new Date(`${date}T00:00:00`); @@ -203,10 +242,10 @@ export function dayLabel(ts: string, today: string, formats = DEFAULT_FORMATS): return formatStamp(ts, date.slice(0, 4) === today.slice(0, 4) ? formats.date : formats.dateWithYear); } -/** Minutes between two "YYYY-MM-DD HH:MM:SS" stamps. */ +/** Minutes between two wire stamps. */ export function minutesBetween(a: string, b: string): number { - const pa = Date.parse(a.replace(" ", "T")); - const pb = Date.parse(b.replace(" ", "T")); + const pa = stampMs(a); + const pb = stampMs(b); if (Number.isNaN(pa) || Number.isNaN(pb)) return Number.POSITIVE_INFINITY; return Math.abs(pb - pa) / 60000; } @@ -235,7 +274,10 @@ export function decorate(msgs: ImsgMessage[], today: string, formats = DEFAULT_F const prev = i > 0 ? msgs[i - 1]! : null; const next = i < msgs.length - 1 ? msgs[i + 1]! : null; - const newDay = !prev || prev.ts.slice(0, 10) !== m.ts.slice(0, 10); + // Day dividers fall on the READER's midnight: the stamps are UTC, so + // slicing the date off the string would break the day in the wrong place + // for anyone not on UTC. + const newDay = !prev || localDay(prev.ts) !== localDay(m.ts); // In a group, two members' messages must not merge into one run under // the first name — so a run breaks on sender (handle) change, not only // on the from_me flip. @@ -248,7 +290,7 @@ export function decorate(msgs: ImsgMessage[], today: string, formats = DEFAULT_F minutesBetween(prev.ts, m.ts) > GROUP_GAP_MINUTES; const groupEnd = !next || - next.ts.slice(0, 10) !== m.ts.slice(0, 10) || + localDay(next.ts) !== localDay(m.ts) || !sameSender(m, next) || minutesBetween(m.ts, next.ts) > GROUP_GAP_MINUTES; @@ -338,16 +380,12 @@ export const PENDING_MAX_AGE_MS = 2 * 60 * 1000; /** A real row may carry a Mac timestamp a little BEHIND the Linux clock. */ const PENDING_SKEW_MS = 5 * 60 * 1000; -function stampMs(ts: string): number { - return Date.parse(String(ts).replace(" ", "T")); -} - /** The bubble for a send in flight, decorated against the bubble before it * the way decorate() would: same run when it follows one of yours within * the gap (that bubble then loses its time), a day divider when it starts one. */ export function pendingBubble(prev: Bubble | undefined, send: PendingSend, today: string, formats = DEFAULT_FORMATS): { bubble: Bubble; prev: Bubble | undefined } { const ts = send.ts; - const newDay = !prev || prev.ts.slice(0, 10) !== ts.slice(0, 10); + const newDay = !prev || localDay(prev.ts) !== localDay(ts); const groupStart = !prev || newDay || !prev.from_me || minutesBetween(prev.ts, ts) > GROUP_GAP_MINUTES; const bubble: Bubble = { ts, @@ -516,8 +554,11 @@ export function loadThread( const parsed = JSON.parse(res.stdout as string); if (!Array.isArray(parsed)) throw new Error("not an array"); const state = loadState(); + // Same door rule as the collector: stamps become the wire format on the + // way in, so a Mac on the pre-UTC bridge cannot mix formats downstream. + const rows = (parsed as ImsgMessage[]).map(normalizeMsgStamps); const msgs = selectThread( - parsed as ImsgMessage[], chat, group, limit, state.selfChats, state.chatAliases, + rows, chat, group, limit, state.selfChats, state.chatAliases, ); return { ok: true, online: true, error: "", bubbles: decorate(msgs, today, formats) }; } catch (e) { diff --git a/tier2.test.ts b/tier2.test.ts index b65c49d..9007909 100644 --- a/tier2.test.ts +++ b/tier2.test.ts @@ -272,9 +272,9 @@ describe("search shaping", () => { test("attachment-only rows (placeholder char) are dropped", () => { const rows = [ - { ts: "2026-08-31 10:00:00", from_me: false, handle: "+15551234567", name: "A", + { ts: "2026-08-31T10:00:00Z", from_me: false, handle: "+15551234567", name: "A", service: "iMessage", chat: "+15551234567", text: "" }, - { ts: "2026-08-31 10:01:00", from_me: true, handle: "+15551234567", name: "A", + { ts: "2026-08-31T10:01:00Z", from_me: true, handle: "+15551234567", name: "A", service: "iMessage", chat: "+15551234567", text: "real match" }, ] as never[]; const out = shapeResults(rows, "match", 10); @@ -285,7 +285,7 @@ describe("search shaping", () => { test("group hits are flagged and limit respected", () => { const rows = Array.from({ length: 5 }, (_, i) => ({ - ts: `2026-08-31 10:0${i}:00`, from_me: false, handle: "+15551234567", name: "G", + ts: `2026-08-31T10:0${i}:00Z`, from_me: false, handle: "+15551234567", name: "G", service: "iMessage", chat: "abcdef0123456789abcdef0123456789", text: `hit ${i}`, })) as never[]; const out = shapeResults(rows, "hit", 3); @@ -299,9 +299,9 @@ describe("search shaping", () => { messageMatchScore("cat", "A scatter of leaves."), ); const rows = [ - { ts: "2026-08-28 17:29:00", from_me: false, handle: "+15550001111", name: "Alice", + { ts: "2026-08-28T17:29:00Z", from_me: false, handle: "+15550001111", name: "Alice", service: "iMessage", chat: "group-a", text: "A scatter of leaves." }, - { ts: "2026-05-18 16:34:00", from_me: true, handle: "+15550002222", name: "Bob", + { ts: "2026-05-18T16:34:00Z", from_me: true, handle: "+15550002222", name: "Bob", service: "iMessage", chat: "+15550002222", text: "The cat sat down." }, ] as never[]; const out = shape(rows, "cat", 10); @@ -312,23 +312,23 @@ describe("search shaping", () => { test("query case does not change whole-word recency order", () => { const { shapeResults: shape } = require("./search") as typeof import("./search"); const rows = [ - { ts: "2026-05-25 22:20:00", from_me: false, handle: "+15550001111", name: "Alice", + { ts: "2026-05-25T22:20:00Z", from_me: false, handle: "+15550001111", name: "Alice", service: "iMessage", chat: "group-a", text: "Alice thanks Bob." }, - { ts: "2026-08-25 20:45:00", from_me: false, handle: "+15550002222", name: "Bob", + { ts: "2026-08-25T20:45:00Z", from_me: false, handle: "+15550002222", name: "Bob", service: "iMessage", chat: "+15550002222", text: "Thanks" }, ] as never[]; const lower = shape(rows, "thanks", 10).map((h) => h.ts); const titled = shape(rows, "Thanks", 10).map((h) => h.ts); expect(lower).toEqual(titled); - expect(lower[0]).toBe("2026-08-25 20:45:00"); + expect(lower[0]).toBe("2026-08-25T20:45:00Z"); }); test("same match quality ties break on message time, not thread order", () => { const { shapeResults: shape } = require("./search") as typeof import("./search"); const rows = [ - { ts: "2025-07-15 17:24:00", from_me: false, handle: "+15550001111", name: "Alice", + { ts: "2025-07-15T17:24:00Z", from_me: false, handle: "+15550001111", name: "Alice", service: "iMessage", chat: "quiet-old-thread", text: "Thanks everyone." }, - { ts: "2026-05-25 22:20:00", from_me: false, handle: "+15550002222", name: "Bob", + { ts: "2026-05-25T22:20:00Z", from_me: false, handle: "+15550002222", name: "Bob", service: "iMessage", chat: "busy-new-thread", text: "Alice thanks Bob." }, ] as never[]; const out = shape(rows, "thanks", 10); @@ -363,7 +363,7 @@ describe("search shaping", () => { const runner = () => ({ status: 0, stdout: JSON.stringify([{ - ts: "2026-05-01 10:00:00", from_me: false, handle: "+15550002222", name: "Bob", + ts: "2026-05-01T10:00:00Z", from_me: false, handle: "+15550002222", name: "Bob", service: "iMessage", chat: "+15550002222", text: "the car is red", }]), stderr: "", @@ -386,7 +386,7 @@ describe("search shaping", () => { }]; const msgs = [{ chat: "+2", name: "Bob", handle: "+2", service: "iMessage", - ts: "2026-09-02 10:00:00", from_me: true, text: "ann called", group: false, + ts: "2026-09-02T10:00:00Z", from_me: true, text: "ann called", group: false, }]; const out = mergeSearchResults(people, msgs); expect(out.map((h) => h.name)).toEqual(["Ann", "Bob"]); @@ -612,8 +612,8 @@ describe("war-room hardening (2.1)", () => { expect(sanitizeName("写真.png")).toBe("写真.png"); }); test("a DM thread never admits the same person's GROUP messages", () => { - const dm = { ts: "2026-08-30 12:00:00", from_me: false, handle: "+15551234567", name: "A", service: "iMessage", chat: "+15551234567", text: "dm" } as never; - const grp = { ts: "2026-08-30 12:01:00", from_me: false, handle: "+15551234567", name: "A", service: "iMessage", chat: "abcdef0123456789abcdef0123456789", text: "in group" } as never; + const dm = { ts: "2026-08-30T12:00:00Z", from_me: false, handle: "+15551234567", name: "A", service: "iMessage", chat: "+15551234567", text: "dm" } as never; + const grp = { ts: "2026-08-30T12:01:00Z", from_me: false, handle: "+15551234567", name: "A", service: "iMessage", chat: "abcdef0123456789abcdef0123456789", text: "in group" } as never; const out = selectThread([dm, grp], "+15551234567", false, 50); expect(out.map((m: { text: string }) => m.text)).toEqual(["dm"]); }); @@ -765,8 +765,8 @@ describe("the contact graph never rides argv", () => { }); test("a bad or absent map degrades ranking, never the search", () => { - expect(parseRecency('{"+15550100011":"2026-09-03 09:00:00"}')) - .toEqual({ "+15550100011": "2026-09-03 09:00:00" }); + expect(parseRecency('{"+15550100011":"2026-09-03T09:00:00Z"}')) + .toEqual({ "+15550100011": "2026-09-03T09:00:00Z" }); expect(parseRecency("not json")).toEqual({}); expect(parseRecency("[1,2,3]")).toEqual({}); expect(parseRecency("null")).toEqual({}); diff --git a/timezone.test.ts b/timezone.test.ts new file mode 100644 index 0000000..0ed631e --- /dev/null +++ b/timezone.test.ts @@ -0,0 +1,168 @@ +/** + * The UTC wire format, and the local-time display built on it. + * + * The rest of the suite runs pinned to UTC (test-setup.ts), where local time + * and wire time are the same string and every bug this file is about is + * invisible. So these tests move the clock themselves: a Mac and a Linux box + * in different zones, and the DST fall-back hour in which a naive wall clock + * stops being ordered at all. + */ +import { afterEach, describe, expect, test } from "bun:test"; +import { isUnread, loadState, nowTs, normalizeMsgStamps, saveState, toUtcStamp, type ImsgMessage } from "./collector"; +import { dayLabel, clockLabel, decorate, localDay, minutesBetween, stampMs } from "./thread"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const ORIGINAL_TZ = process.env.TZ; +afterEach(() => { process.env.TZ = ORIGINAL_TZ; }); + +/** Run body with the process clock in `zone`. */ +function inZone(zone: string, body: () => T): T { + process.env.TZ = zone; + try { return body(); } finally { process.env.TZ = ORIGINAL_TZ; } +} + +const msg = (over: Partial): ImsgMessage => ({ + id: 1, ts: "2026-08-30T10:00:00Z", from_me: false, handle: "+15551234567", name: "A", + service: "iMessage", chat: "+15551234567", text: "hi", read: false, ...over, +} as ImsgMessage); + +describe("the wire is UTC, the labels are local", () => { + test("one instant renders in each reader's own clock", () => { + const ts = "2026-09-07T18:33:12Z"; + expect(inZone("UTC", () => clockLabel(ts))).toBe("6:33 PM"); + expect(inZone("America/New_York", () => clockLabel(ts))).toBe("2:33 PM"); + expect(inZone("Asia/Tokyo", () => clockLabel(ts))).toBe("3:33 AM"); + }); + + test("the day divider breaks at the READER's midnight, not UTC's", () => { + // 2026-08-31 02:00 UTC is still the evening of the 30th in New York. + const ts = "2026-08-31T02:00:00Z"; + expect(inZone("UTC", () => localDay(ts))).toBe("2026-08-31"); + expect(inZone("America/New_York", () => localDay(ts))).toBe("2026-08-30"); + // …so on a New York reader's 30th the same message is "Today", while a + // UTC reader is still on the 30th and it is already dated the 31st. + expect(inZone("America/New_York", () => dayLabel(ts, "2026-08-30"))).toBe("Today"); + expect(inZone("UTC", () => dayLabel(ts, "2026-08-30"))).toBe("Aug 31"); + }); + + test("a bare date is the local calendar day, never UTC midnight", () => { + // Date.parse("2026-08-30") is UTC midnight by the ISO rule — which is the + // 29th anywhere west of Greenwich. localToday() hands dayLabel() bare dates. + expect(inZone("America/New_York", () => localDay("2026-08-30"))).toBe("2026-08-30"); + }); +}); + +describe("the DST fall-back hour", () => { + // 2026-11-01 in America/New_York: 01:00-01:59 happens twice, EDT then EST. + const edt = "2026-11-01T05:00:00Z"; // 01:00 EDT — first + const est = "2026-11-01T06:00:00Z"; // 01:00 EST — an hour later + + test("both instants wear the same local clock face", () => { + // This is exactly why a naive wall clock cannot be the wire format: as + // strings these two were EQUAL, an hour apart. + inZone("America/New_York", () => { + expect(clockLabel(edt)).toBe("1:00 AM"); + expect(clockLabel(est)).toBe("1:00 AM"); + }); + }); + + test("UTC keeps them ordered and an hour apart", () => { + expect(edt < est).toBe(true); // the comparison every ledger makes + expect(minutesBetween(edt, est)).toBe(60); + expect(stampMs(est) - stampMs(edt)).toBe(3600_000); + }); + + test("a run breaks across the repeated hour instead of merging", () => { + // 60 minutes > GROUP_GAP_MINUTES, so these are two runs. On naive stamps + // the gap read as 0 and they merged into one. + const bubbles = inZone("America/New_York", () => + decorate([msg({ id: 1, ts: edt }), msg({ id: 2, ts: est })], "2026-11-01")); + expect(bubbles.map((b) => b.groupStart)).toEqual([true, true]); + }); +}); + +describe("read marks survive a Mac in another timezone", () => { + test("a message that just arrived is unread whatever either clock reads", () => { + // The mark is this machine's clock; the stamp is the Mac's. Both UTC now, + // so the comparison holds in every zone — it did not when each side wrote + // its own wall clock. + for (const zone of ["UTC", "America/New_York", "Asia/Tokyo", "Pacific/Kiritimati"]) { + inZone(zone, () => { + const mark = nowTs(new Date(Date.now() - 60_000)); // read a minute ago + const arriving = msg({ ts: nowTs(), read: false }); + expect(isUnread(arriving, mark)).toBe(true); + }); + } + }); + + test("a mark taken now covers a message from a moment ago, in every zone", () => { + for (const zone of ["UTC", "America/New_York", "Asia/Tokyo"]) { + inZone(zone, () => { + const earlier = msg({ ts: nowTs(new Date(Date.now() - 60_000)) }); + expect(isUnread(earlier, nowTs())).toBe(false); + }); + } + }); +}); + +describe("upgrading from the pre-UTC format", () => { + test("a naive stamp is re-anchored through this machine's offset", () => { + // 14:33 in New York IS 18:33 UTC. + expect(inZone("America/New_York", () => toUtcStamp("2026-09-07 14:33:12"))) + .toBe("2026-09-07T18:33:12Z"); + // Already-wire stamps pass through untouched; junk becomes "". + expect(toUtcStamp("2026-09-07T18:33:12Z")).toBe("2026-09-07T18:33:12Z"); + expect(toUtcStamp("x")).toBe(""); + expect(toUtcStamp("")).toBe(""); + }); + + test("state.json written by an older release loads as UTC", () => { + const p = join(mkdtempSync(join(tmpdir(), "blip-tz-")), "state.json"); + writeFileSync(p, JSON.stringify({ + watermark: "2026-09-07 14:33:12", + readMark: "2026-09-07 14:00:00", + readMarks: { A: "2026-09-07 13:00:00" }, + unreadCounts: { A: 1 }, + unreadOldest: { A: "2026-09-07 13:30:00" }, + unreadInitialized: true, + })); + const state = inZone("America/New_York", () => loadState(p)); + expect(state.watermark).toBe("2026-09-07T18:33:12Z"); + expect(state.readMark).toBe("2026-09-07T18:00:00Z"); + expect(state.readMarks.A).toBe("2026-09-07T17:00:00Z"); + expect(state.unreadOldest.A).toBe("2026-09-07T17:30:00Z"); + }); + + test("a Mac still on the old bridge has its stamps normalised at the door", () => { + // Marks migrate on load; if the bridge kept emitting naive stamps and + // nothing normalised them, EVERY message would sort below EVERY mark + // (" " < "T") and the badge would silently go quiet. + const legacy = msg({ ts: "2026-09-07 14:33:12", read_at: "2026-09-07 14:35:00" } as Partial); + const fixed = inZone("America/New_York", () => normalizeMsgStamps(legacy)); + expect(fixed.ts).toBe("2026-09-07T18:33:12Z"); + expect(fixed.read_at).toBe("2026-09-07T18:35:00Z"); + + const mark = inZone("America/New_York", () => toUtcStamp("2026-09-07 14:00:00")); + expect(isUnread(fixed, mark)).toBe(true); // newer than the mark, as it should be + }); + + test("normalising leaves a wire-format message untouched", () => { + const m = msg({ ts: "2026-09-07T18:33:12Z" }); + expect(normalizeMsgStamps(m)).toBe(m); // same object: no needless copy + }); +}); + +describe("state round-trips in the wire format", () => { + test("marks saved and reloaded are unchanged", () => { + const p = join(mkdtempSync(join(tmpdir(), "blip-tz-")), "state.json"); + const watermark = nowTs(); + saveState({ + watermark, readMark: watermark, unreadCounts: {}, unreadOldest: {}, + unreadInitialized: true, selfChats: [], readMarks: {}, groups: {}, + chatAliases: {}, pins: {}, toasted: [], + }, p); + expect(inZone("Asia/Tokyo", () => loadState(p).watermark)).toBe(watermark); + }); +}); From 1feab93720a1b9b7194cfc57a6250463b528cc22 Mon Sep 17 00:00:00 2001 From: Zach Wilke Date: Mon, 7 Sep 2026 19:30:07 -0500 Subject: [PATCH 2/7] gifs: an animated attachment arrives animated, and renders animated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A GIF was flattened twice on the way to the screen. fetch.ts treats any image/* under the auto-fetch cap as a preview candidate and asks the Mac to resample it with sips — which collapses an animation to one frame, so a 1.4 MB GIF landed as a 198 KB JPEG with the motion already gone. Animated formats now skip that path entirely and cross as their original bytes, into the `orig` cache slot a click already shares. The second flattening was in QML: Image paints a single frame whatever you give it. Animated mimes now render through AnimatedImage. Stills deliberately stay on Image — it is the one that applies autoTransform, the EXIF fall-back for anything cached before fetch.ts started baking orientation in. Sizing is computed once on a wrapping Item so the two renderers can never disagree about the delegate's width, and only the active one takes a source, or every photo would decode twice. AnimatedImage honours sourceSize (measured), so the decode ceiling still applies to a huge GIF. imageMetrics also learned the GIF header: it reported 0×0 before, which left the bubble with nothing to compute a height from. Verified against a real 498×372, 18-frame, infinitely-looping GIF in chat.db: the bridge now delivers all 1,357,761 bytes instead of a 198 KB still, and the panel reports animated=true playing=true frameCount=18 with the frame counter cycling 0→6→12→0 while ten successive screenshots of the bubble come back as ten distinct images. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015xxS9WCT7ifHhbDjYyGnHS --- BlipView.qml | 88 ++++++++++++++++++++++++++++++++++++++------------- CHANGELOG.md | 13 ++++++++ fetch.ts | 25 ++++++++++++++- tier2.test.ts | 63 +++++++++++++++++++++++++++++++++++- 4 files changed, 165 insertions(+), 24 deletions(-) diff --git a/BlipView.qml b/BlipView.qml index 70221f5..1c69a9d 100644 --- a/BlipView.qml +++ b/BlipView.qml @@ -657,6 +657,10 @@ FocusScope { // ---------------------------------------------------- attachment fetching function isImageMime(m) { return String(m || "").indexOf("image/") === 0 } + /** Formats that MOVE — they render through AnimatedImage, not Image, and + * fetch.ts keeps them off the resampling path that would flatten them. + * Mirrors isAnimatedMime() in fetch.ts; keep the two in step. */ + function isAnimatedMime(m) { return String(m || "").toLowerCase() === "image/gif" } function linkHost(u) { var m = /^https?:\/\/([^/?#]+)/i.exec(String(u || "")); return (m ? m[1] : String(u || "")).replace(/^www\./, "").toLowerCase() } /** Only http(s) ever reaches xdg-open from a card (thread.ts filters too). */ function openLink(u) { @@ -2739,10 +2743,21 @@ FocusScope { spacing: 0 Item { Layout.fillWidth: true; visible: bubbleRow.mine } - // fetched image renders inline, like Messages; click = full view - Image { + // fetched image renders inline, like Messages; click = full view. + // A GIF has to MOVE, and Image paints frame one and stops, + // so animated formats go through AnimatedImage instead. + // Static formats stay on Image: it is the one that applies + // autoTransform, the EXIF fall-back for anything cached + // before fetch.ts started baking orientation in. Sizing is + // computed ONCE here so the two can never disagree — a + // delegate whose implicit width outgrows the panel takes + // every right-aligned element off-screen with it. + Item { id: attImage visible: chipRow.showImage + readonly property bool animated: root.isAnimatedMime(chipRow.modelData.mime) + readonly property var view: animated ? attAnimated : attStill + readonly property int status: view.status readonly property real maxW: Math.round(content.width * 0.6) // Retina PNGs carry their density in the header (read by // fetch.ts); divide it out so a 2x screenshot draws at @@ -2753,35 +2768,64 @@ FocusScope { readonly property real naturalWidth: Number(chipRow.imageMetrics.pixelWidth || 0) > 0 ? Number(chipRow.imageMetrics.pixelWidth) / pixelRatio - : implicitWidth + : view.implicitWidth readonly property real naturalHeight: Number(chipRow.imageMetrics.pixelHeight || 0) > 0 ? Number(chipRow.imageMetrics.pixelHeight) / pixelRatio - : implicitHeight - source: chipRow.showImage ? chipRow.fileUrl : "" - asynchronous: true - fillMode: Image.PreserveAspectFit - // iPhone photos store rotation as an EXIF tag, not in - // the pixels (sips keeps the tag when it converts HEIC); - // Qt ignores it unless asked, so portraits came out on - // their side. implicitWidth/Height follow the transform. - autoTransform: true - // bound the DECODE in BOTH axes, not just the paint — a - // 12MP photo (or a 100×100000 sliver) must not cost - // 50 MB of texture (Codex review points 19 and #3) - sourceSize.width: 800 - sourceSize.height: 800 + : view.implicitHeight + + Layout.preferredWidth: status === Image.Ready ? Math.min(maxW, naturalWidth) : maxW + Layout.preferredHeight: status === Image.Ready && naturalWidth > 0 + ? Layout.preferredWidth * naturalHeight / naturalWidth + : Style.space(120) + // LRU eviction or a corrupt file: fall back to the chip // (⚠ marker); a click re-fetches through fetch.ts. - onStatusChanged: if (status === Image.Error) { + function forget() { var m = Object.assign({}, root.attFiles) m[chipRow.attId] = "" root.attFiles = m } - Layout.preferredWidth: status === Image.Ready ? Math.min(maxW, naturalWidth) : maxW - Layout.preferredHeight: status === Image.Ready && naturalWidth > 0 - ? Layout.preferredWidth * naturalHeight / naturalWidth - : Style.space(120) + + Image { + id: attStill + anchors.fill: parent + visible: !attImage.animated + // Only the ACTIVE renderer loads: two sources would + // decode every photo twice. + source: chipRow.showImage && !attImage.animated ? chipRow.fileUrl : "" + asynchronous: true + fillMode: Image.PreserveAspectFit + // iPhone photos store rotation as an EXIF tag, not in + // the pixels (sips keeps the tag when it converts HEIC); + // Qt ignores it unless asked, so portraits came out on + // their side. implicitWidth/Height follow the transform. + autoTransform: true + // bound the DECODE in BOTH axes, not just the paint — a + // 12MP photo (or a 100×100000 sliver) must not cost + // 50 MB of texture (Codex review points 19 and #3) + sourceSize.width: 800 + sourceSize.height: 800 + onStatusChanged: if (status === Image.Error) attImage.forget() + } + + AnimatedImage { + id: attAnimated + anchors.fill: parent + visible: attImage.animated + source: chipRow.showImage && attImage.animated ? chipRow.fileUrl : "" + asynchronous: true + fillMode: Image.PreserveAspectFit + // AnimatedImage honours sourceSize (measured), so the + // same decode ceiling applies to a huge GIF. + sourceSize.width: 800 + sourceSize.height: 800 + // Nothing animates off-screen: a thread full of GIFs + // would otherwise decode frames nobody is looking at. + playing: visible && root.inThread + onStatusChanged: if (status === Image.Error) attImage.forget() + } + HoverHandler { cursorShape: Qt.PointingHandCursor } TapHandler { onTapped: root.openAttachment(chipRow.modelData) } } diff --git a/CHANGELOG.md b/CHANGELOG.md index ec96e91..699bc35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## Unreleased +- **GIFs move.** An animated GIF arrived as a still, and did so twice over. The + inline-preview path asks the Mac to resample every image with sips, which + flattens an animation to a single frame — a 1.4 MB GIF reached Linux as a + 198 KB JPEG, the motion gone before the panel ever saw it. Animated formats + now skip that path and cross as their own bytes, into the same cache slot a + click uses. And a QML `Image` paints one frame whatever you hand it, so + animated attachments render through `AnimatedImage` instead; stills stay on + `Image`, which is what applies `autoTransform` (the EXIF fall-back for + anything cached before orientation was baked in at fetch time). Only the + active renderer loads, so no photo decodes twice, and the decode is still + bounded in both axes. GIF dimensions now come off the header too — they read + 0×0 before, leaving the bubble nothing to size itself from. + - **Time crosses the bridge as UTC.** Stamps used to arrive as the Mac's naive wall clock ("2026-09-07 14:33:12") and were compared against the Linux clock — the same string only while both machines sat in one timezone. A Mac diff --git a/fetch.ts b/fetch.ts index 65372a8..d8f5442 100644 --- a/fetch.ts +++ b/fetch.ts @@ -52,6 +52,18 @@ export function wantsJpeg(mime: string): boolean { return mime === "image/heic" || mime === "image/heif"; } +/** + * Formats whose whole point is that they MOVE. + * + * The inline preview path asks the Mac to resample with sips, which flattens + * an animated GIF to a single frame — a 1.4 MB animation arrived as a 198 KB + * still. So these never take that path: they cross as their original bytes, + * capped like any other auto-fetch, and land in the shared `orig` cache slot. + */ +export function isAnimatedMime(mime: string): boolean { + return String(mime || "").toLowerCase() === "image/gif"; +} + /** Anything the panel would draw inline. */ export function isImageMime(mime: string): boolean { return String(mime || "").startsWith("image/"); @@ -126,6 +138,17 @@ export function imageMetrics(bytes: Buffer, mime: string): ImageMetrics { if (!String(mime || "").startsWith("image/") || !bytes || bytes.length < 10) return { ...EMPTY_IMAGE_METRICS }; + // GIF: "GIF87a"/"GIF89a" then the logical screen size, u16 little-endian. + // Without this an animated GIF reported 0×0 and the bubble had nothing to + // size itself from. + if (bytes.length >= 10 && bytes.toString("ascii", 0, 3) === "GIF") { + return { + pixelWidth: bytes.readUInt16LE(6), + pixelHeight: bytes.readUInt16LE(8), + pixelRatio: 1, // GIF carries no density + }; + } + const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); if (bytes.length >= 24 && bytes.subarray(0, 8).equals(png)) { const pixelWidth = bytes.readUInt32BE(16); @@ -335,7 +358,7 @@ export function fetchAttachment( if (!/^[0-9]{1,18}$/.test(id)) return fail("bad attachment id"); mkdirSync(CACHE_DIR, { recursive: true, mode: 0o700 }); - const preview = maxBytes < FETCH_MAX_BYTES && isImageMime(mime); + const preview = maxBytes < FETCH_MAX_BYTES && isImageMime(mime) && !isAnimatedMime(mime); const file = join(CACHE_DIR, cacheFileName(id, name, mime, preview)); try { const st = lstatSync(file); diff --git a/tier2.test.ts b/tier2.test.ts index 9007909..d178430 100644 --- a/tier2.test.ts +++ b/tier2.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; -import { CACHE_DIR, bakeOrientation, cacheFileName, exifOrientation, fetchAttachment, imageMetrics, isImageMime, jpegtranArgs, lruEvictions, sanitizeName, wantsJpeg } from "./fetch"; +import { CACHE_DIR, bakeOrientation, cacheFileName, exifOrientation, fetchAttachment, imageMetrics, isAnimatedMime, isImageMime, jpegtranArgs, lruEvictions, sanitizeName, wantsJpeg } from "./fetch"; import { extFor, existingLocalFile, firstFileUri, pickImageType, snapshotClipboard } from "./paste"; import { resolveTarget, sendFile } from "./send-file"; import { linkHost, linkify, normalizeLink, selectThread } from "./thread"; @@ -829,6 +829,67 @@ describe("multi-photo messages and the preview transform (#Crystal/Thatchers)", }); }); +describe("animated images have to keep moving", () => { + /** "GIF89a", then the logical screen size as u16 little-endian. */ + const gifHeader = (w: number, h: number) => { + const b = Buffer.alloc(13); + b.write("GIF89a", 0, "ascii"); + b.writeUInt16LE(w, 6); + b.writeUInt16LE(h, 8); + return b; + }; + + test("only the formats that actually animate count", () => { + expect(isAnimatedMime("image/gif")).toBe(true); + expect(isAnimatedMime("IMAGE/GIF")).toBe(true); // mimes arrive in any case + expect(isAnimatedMime("image/png")).toBe(false); + expect(isAnimatedMime("image/heic")).toBe(false); + expect(isAnimatedMime("")).toBe(false); + }); + + test("a GIF auto-fetch is NEVER resampled — sips would flatten it to one frame", () => { + const seen: string[][] = []; + const runner = ((_c: string, args: string[]) => { + seen.push(args); + return { status: 0, stdout: gifHeader(498, 372), stderr: "" }; + }) as never; + // The same cap that makes every other image take the resampling path. + fetchAttachment("881", "loop.gif", "image/gif", runner, 5 * 1024 * 1024); + expect(seen[0]).not.toContain("--max-dim"); + expect(seen[0]).not.toContain("--jpeg"); + // …while a still image on that same path still is resampled. + fetchAttachment("882", "shot.png", "image/png", runner, 5 * 1024 * 1024); + expect(seen[1]).toContain("--max-dim"); + for (const f of ["881-orig-loop.gif", "882-prev-shot.jpg"]) { + try { unlinkSync(`${CACHE_DIR}/${f}`); } catch { /* fine */ } + } + }); + + test("it keeps its own extension and the shared orig slot", () => { + // Not "-prev-…jpg": that name is a promise the bytes are a JPEG, and + // xdg-open dispatches on the extension. + expect(cacheFileName("7", "loop.gif", "image/gif", false)).toBe("7-orig-loop.gif"); + }); + + test("dimensions come from the GIF header, so the bubble can size itself", () => { + // 0×0 left the delegate with nothing to compute a height from. + expect(imageMetrics(gifHeader(498, 372), "image/gif")) + .toEqual({ pixelWidth: 498, pixelHeight: 372, pixelRatio: 1 }); + }); + + test("the panel draws animated mimes with AnimatedImage and stills with Image", () => { + const panelSrc = readFileSync(new URL("./BlipView.qml", import.meta.url), "utf8"); + expect(panelSrc).toContain("AnimatedImage {"); + expect(panelSrc).toContain("root.isAnimatedMime(chipRow.modelData.mime)"); + // The still path keeps autoTransform: it is the EXIF fall-back for + // anything cached before fetch.ts began baking orientation in. + expect(panelSrc).toContain("autoTransform: true"); + // Only the active renderer loads, or every photo decodes twice. + expect(panelSrc).toContain("chipRow.showImage && !attImage.animated ? chipRow.fileUrl : \"\""); + expect(panelSrc).toContain("chipRow.showImage && attImage.animated ? chipRow.fileUrl : \"\""); + }); +}); + describe("cache file names (Astra B#1)", () => { const { cacheFileName } = require("./fetch") as typeof import("./fetch"); test("an unmapped MIME never keeps the sender's extension", () => { From 20f8bc1172255932b5d2e1b562d57cade16c9717 Mon Sep 17 00:00:00 2001 From: Zach Wilke Date: Mon, 7 Sep 2026 19:35:25 -0500 Subject: [PATCH 3/7] compose: a message can carry several files, not just the first one dropped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dropping five photos attached one and discarded four in silence — the drop handler took urls[0] and draftPath was a single string. A drop that loses most of what you dropped, with no message, is worse than one that refuses. Drafts are a list now. Drag-and-drop queues every file it was given, /attach and Ctrl+V append rather than replace, and each chip removes only itself. Sends go out one part per file in queue order through the existing send-file.ts path (unchanged: still one file per invocation, still stdin, still refusing groups whose guid is unknown). The caption rides the first part alone, or five files would post the same sentence five times. Two things a batch has to survive. The service is captured when it STARTS, never re-read from root.active per part, so switching threads mid-send cannot push a later part onto a different service (war room #2). And a failing part stops the batch, leaves the remaining files attached and says how many are left, instead of making the user work out which of five actually went. Capped at ten files: a stray drop of a whole folder should be refused, not turned into eighty sends. Chips are one per row for the reason the received ones are — a RowLayout of N sums implicit widths and drags the conversation column past the panel edge, taking every right-aligned element with it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015xxS9WCT7ifHhbDjYyGnHS --- BlipView.qml | 219 +++++++++++++++++++++++++++++++++------------- CHANGELOG.md | 14 +++ collector.test.ts | 8 +- thread.test.ts | 6 +- ui.test.ts | 80 ++++++++++++++++- 5 files changed, 257 insertions(+), 70 deletions(-) diff --git a/BlipView.qml b/BlipView.qml index 1c69a9d..24a0f58 100644 --- a/BlipView.qml +++ b/BlipView.qml @@ -161,8 +161,15 @@ FocusScope { property var fetchQueue: [] property string fetchingId: "" // Compose draft attachment (one per message in v1). - property string draftPath: "" - property string draftLabel: "" + // Queued attachments, oldest first: [{ path, label }]. A single draft could + // only ever ship the first of five dropped photos, silently discarding the + // rest — a drop that loses four files with no message is worse than a + // refusal. Sends go out one part per file, in this order. + property var attachDrafts: [] + readonly property int attachCount: attachDrafts.length + /** How many files one message may carry. A stray drop of a whole folder + * should be refused, not turned into eighty sends. */ + readonly property int attachMax: 10 // What the in-flight file send / paste belong to, so late completions // can't clobber a NEWER draft or land in a DIFFERENT conversation. property string sendDraftPath: "" @@ -345,6 +352,15 @@ FocusScope { // Text sends queue instead of refusing while one is on the wire; each gets // its bubble the instant Enter is pressed (see pendingSends). property var sendQueue: [] + // Files still to ship for the send in flight: one part per file, in order. + // The caption rides the FIRST part only — repeating it on each would post + // the same sentence five times. + property var fileQueue: [] + property string sendCaption: "" + // The service is captured when the batch STARTS, never re-read per part: + // root.active can change under a multi-part send, and a later part must not + // go out on a different service from the first (war room #2). + property string sendService: "" // Sends the Mac has not written a row for yet, {chat, text, ts}. Every // thread reload carries them to thread.ts (--pending-stdin, never argv), // which keeps their bubbles until the real row lands. Memory only. @@ -458,7 +474,7 @@ FocusScope { loading = false pendingThreadChat = "" composeField.text = "" - clearDraft() // a queued file must never survive into another thread + clearAttachments() // a queued file must never survive into another thread pinToBottom = false // Top of the list for a mouse user; the cursor row for a keyboard user. Qt.callLater(function() { @@ -486,7 +502,7 @@ FocusScope { loading = true composeField.text = drafts[String(t.chat)] || "" // this conversation's unsent text composeField.cursorPosition = composeField.length - clearDraft() // a queued file must never survive into another thread + clearAttachments() // a queued file must never survive into another thread requestThreadLoad(String(t.chat)) Qt.callLater(function() { composeField.forceActiveFocus() }) } @@ -859,18 +875,41 @@ FocusScope { // ------------------------------------------------------ compose attachment - function setDraft(path) { + /** Queue one more file on the draft. The same path twice is one attachment. */ + function addAttachment(path) { var p = String(path || "") if (p === "") return - draftPath = p + for (var i = 0; i < root.attachDrafts.length; i++) if (root.attachDrafts[i].path === p) return + if (root.attachDrafts.length >= root.attachMax) { + root.note = "at most " + root.attachMax + " files per message" + return + } var parts = p.split("/") - draftLabel = parts[parts.length - 1] || "file" + root.attachDrafts = root.attachDrafts.concat([{ path: p, label: parts[parts.length - 1] || "file" }]) Qt.callLater(function() { composeField.forceActiveFocus() }) } - function clearDraft() { - draftPath = "" - draftLabel = "" + /** Queue several at once — a multi-file drop. Reports what would not fit + * rather than dropping it on the floor. */ + function addAttachments(paths) { + var before = root.attachDrafts.length + for (var i = 0; i < (paths || []).length; i++) { + if (root.attachDrafts.length >= root.attachMax) { + root.note = "attached " + (root.attachDrafts.length - before) + " — at most " + + root.attachMax + " files per message" + return + } + root.addAttachment(paths[i]) + } + } + + function removeAttachment(path) { + root.attachDrafts = root.attachDrafts.filter(function(d) { return d.path !== String(path) }) + } + + function clearAttachments() { + root.attachDrafts = [] + root.fileQueue = [] } function startPaste() { @@ -1219,35 +1258,35 @@ FocusScope { if (trimmed.indexOf("/attach ") === 0) { var p = trimmed.slice(8).trim() if (p.indexOf("~/") === 0) p = root.home + p.slice(1) - setDraft(p) + addAttachment(p) composeField.text = "" - note = "attached — type a caption or press Enter to send" + note = root.attachCount > 1 + ? root.attachCount + " files attached — caption, or Enter to send" + : "attached — type a caption or press Enter to send" return } - if (draftPath === "" && trimmed === "") return + if (root.attachCount === 0 && trimmed === "") return if (!isSendable(root.active)) { note = "Read-only — group id unknown — send from your phone" return } - if (draftPath !== "") { + if (root.attachCount > 0) { if (sendProc.running || fileSendProc.running) { note = "a message is already sending" return } - note = "sending…" sendChat = String(root.active.chat) sendText = text - // send-file.ts owns target resolution (group guid or DM handle). - sendDraftPath = draftPath - // caption on stdin — never in this process's argv (audit #4, war room #1/#13) - fileSendProc.command = ["bun", root.sendFileScript, sendChat, draftPath, "--caption-stdin"] - .concat(/^(SMS|RCS)$/i.test(String(root.active.service || "")) ? ["--service", String(root.active.service).toUpperCase()] : []) - fileSendProc.stdinEnabled = true - fileSendProc.running = true - fileSendProc.write(trimmed !== "" ? text : "") - fileSendProc.stdinEnabled = false + // One part per file, in the order they were queued; the caption goes + // with the first only. fileSendProc is a single Process, so they ship + // strictly one at a time — pumpFileSend() starts the next as each lands. + root.fileQueue = root.attachDrafts.map(function(d) { return d.path }) + root.sendCaption = trimmed + var svc = String(root.active.service || "") + root.sendService = /^(SMS|RCS)$/i.test(svc) ? svc.toUpperCase() : "" + pumpFileSend() return } @@ -1336,6 +1375,27 @@ FocusScope { sendProc.stdinEnabled = false } + /** Ship the next queued file. The caption rides the FIRST part only, and + * goes over stdin — never this process's argv (audit #4, war room #1/#13). */ + function pumpFileSend() { + if (fileSendProc.running) return + if (root.fileQueue.length === 0) return + var path = root.fileQueue[0] + root.fileQueue = root.fileQueue.slice(1) + root.sendDraftPath = path + var left = root.fileQueue.length + root.note = left > 0 ? "sending… (" + (left + 1) + " left)" : "sending…" + // send-file.ts owns target resolution (group guid or DM handle). + fileSendProc.command = ["bun", root.sendFileScript, root.sendChat, path, "--caption-stdin"] + .concat(root.sendService !== "" ? ["--service", root.sendService] : []) + fileSendProc.stdinEnabled = true + fileSendProc.running = true + fileSendProc.write(root.sendCaption) + fileSendProc.stdinEnabled = false + // Spent: only the first part carries it. + root.sendCaption = "" + } + Process { id: copyProc } function copyText(t) { if (t === "") return @@ -1558,7 +1618,7 @@ FocusScope { if (!root.inThread || String(root.active.chat) !== root.pasteChat) return try { var d = JSON.parse(text.trim()) - if (d.kind === "image" || d.kind === "file") root.setDraft(String(d.path || "")) + if (d.kind === "image" || d.kind === "file") root.addAttachment(String(d.path || "")) else if (d.kind === "text") { composeField.insert(composeField.cursorPosition, String(d.text || "")) } @@ -1580,18 +1640,33 @@ FocusScope { if (!ok) err = d.online === false ? "not sent — Mac unreachable" : String(d.error || err) } catch (e) { /* fall through */ } if (ok) { - // Only clear the draft this send actually shipped — the user may - // have queued a NEWER file while this one was in flight. - if (root.draftPath === root.sendDraftPath) root.clearDraft() + // Retire only the part that actually shipped — the user may have + // queued another file while this one was in flight. + root.removeAttachment(root.sendDraftPath) + root.sendDraftPath = "" + root.reloadChat = root.sendChat + reloadTimer.restart() + if (root.fileQueue.length > 0) { + // More parts to go: keep sendChat/sendText, do NOT clear the + // field or hand focus back mid-batch. + root.pumpFileSend() + return + } if (belongsHere) { root.note = "" if (composeField.text === root.sendText) composeField.text = "" } - root.reloadChat = root.sendChat - reloadTimer.restart() } else if (belongsHere) { - root.note = err + // Stop the batch: the parts still queued stay attached, so the user + // can retry them rather than hunt for which of five went out. + var left = root.fileQueue.length + root.note = left > 0 ? err + " — " + (left + 1) + " still attached" : err + root.fileQueue = [] + root.sendDraftPath = "" } + root.fileQueue = [] + root.sendCaption = "" + root.sendService = "" root.sendChat = "" root.sendText = "" if (belongsHere) composeField.forceActiveFocus() @@ -1777,7 +1852,7 @@ FocusScope { || (text >= "1" && text <= "9") if (!jump) return false if (searchField.activeFocus || newField.activeFocus || bubbleFocused) return false - if (composeField.activeFocus && (composeField.text.length > 0 || root.draftPath !== "")) + if (composeField.activeFocus && (composeField.text.length > 0 || root.attachCount > 0)) return false return handleTextKey(text) === true } @@ -3164,32 +3239,47 @@ FocusScope { foreground: root.foreground } - // queued attachment — one per message; ✕ removes it - RowLayout { + // queued attachments — ✕ removes one. ONE CHIP PER ROW, never a + // RowLayout of N: summed implicit widths stretch the whole column + // past the panel and take every right-aligned element off-screen + // with it (CLAUDE.md; the same rule the received chips follow). + ColumnLayout { + id: attachList Layout.fillWidth: true - visible: root.inThread && root.draftPath !== "" - spacing: 0 - Rectangle { - Layout.preferredWidth: Math.ceil(draftText.implicitWidth) + Style.space(18) - Layout.preferredHeight: Math.ceil(draftText.implicitHeight) + Style.space(12) - radius: Style.space(14) - color: root.mineFill - opacity: 0.9 - Text { - id: draftText - anchors.centerIn: parent - text: "📎 " + (root.draftLabel.length > 40 - ? root.draftLabel.slice(0, 37) + "…" - : root.draftLabel) + " ✕" - textFormat: Text.PlainText - color: root.mineText - font.family: root.fontFamily - font.pixelSize: root.fontCaption + visible: root.inThread && root.attachCount > 0 + spacing: Style.space(4) + Repeater { + model: root.attachDrafts + delegate: RowLayout { + required property var modelData + Layout.fillWidth: true + spacing: 0 + Rectangle { + // Capped at the compose column: a long filename must not + // push the row wider than the panel. + Layout.preferredWidth: Math.min( + Math.ceil(attachChipText.implicitWidth) + Style.space(18), + attachList.width) + Layout.preferredHeight: Math.ceil(attachChipText.implicitHeight) + Style.space(12) + radius: Style.space(14) + color: root.mineFill + opacity: 0.9 + Text { + id: attachChipText + anchors.centerIn: parent + readonly property string label: String(modelData.label || "file") + text: "📎 " + (label.length > 40 ? label.slice(0, 37) + "…" : label) + " ✕" + textFormat: Text.PlainText + color: root.mineText + font.family: root.fontFamily + font.pixelSize: root.fontCaption + } + HoverHandler { cursorShape: Qt.PointingHandCursor } + TapHandler { onTapped: root.removeAttachment(modelData.path) } + } + Item { Layout.fillWidth: true } } - HoverHandler { cursorShape: Qt.PointingHandCursor } - TapHandler { onTapped: root.clearDraft() } } - Item { Layout.fillWidth: true } } RowLayout { @@ -3270,7 +3360,7 @@ FocusScope { // instead; send() is the authoritative online/sendability guard. enabled: true readOnly: !root.online || !root.isSendable(root.active) - placeholderText: root.draftPath !== "" + placeholderText: root.attachCount > 0 ? "caption (optional) — Enter sends the file" : root.isSendable(root.active) ? "iMessage" : "Read-only — group id unknown" color: root.foreground @@ -3329,7 +3419,7 @@ FocusScope { } // Actions on the selected bubble. Enter is free here: with no // text and no queued file, send() would do nothing anyway. - var b = empty && root.draftPath === "" ? root.selectedBubble() : null + var b = empty && root.attachCount === 0 ? root.selectedBubble() : null if (b) { if (event.key === Qt.Key_Return || event.key === Qt.Key_Enter) { event.accepted = true; root.openBubble(b); return } if (event.matches(StandardKey.Copy)) { event.accepted = true; root.copyBubble(b); return } @@ -3364,7 +3454,7 @@ FocusScope { // send button — the blue arrow circle (lit when text OR a file is queued) Rectangle { Layout.alignment: Qt.AlignBottom - readonly property bool armed: composeField.text.trim() !== "" || root.draftPath !== "" + readonly property bool armed: composeField.text.trim() !== "" || root.attachCount > 0 width: Style.space(28); height: width; radius: width / 2 color: armed ? root.mineFill : Qt.rgba(root.foreground.r, root.foreground.g, root.foreground.b, 0.15) Text { @@ -3404,9 +3494,16 @@ FocusScope { keys: ["text/uri-list"] onDropped: (drop) => { if (!drop.hasUrls || drop.urls.length === 0) return - var u = String(drop.urls[0]) - if (u.indexOf("file://") !== 0) return - root.setDraft(decodeURIComponent(u.replace(/^file:\/\//, ""))) + // EVERY dropped file, not just urls[0]: dropping five photos used to + // attach one and discard four without saying so. + var paths = [] + for (var i = 0; i < drop.urls.length; i++) { + var u = String(drop.urls[i]) + if (u.indexOf("file://") !== 0) continue + paths.push(decodeURIComponent(u.replace(/^file:\/\//, ""))) + } + if (paths.length === 0) return + root.addAttachments(paths) drop.accept() } } diff --git a/CHANGELOG.md b/CHANGELOG.md index 699bc35..f98d8af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ ## Unreleased +- **A message can carry several files.** Dropping five photos on a + conversation attached one and threw the other four away without a word: the + drop handler read `urls[0]` and the draft was a single path. Drafts are a + list now — drag-and-drop takes every file, `/attach` and Ctrl+V add to it, + and each chip has its own ✕. They ship one part per file in the order you + queued them, with the caption on the first only (repeating it would post the + same sentence five times), and the service is captured when the batch starts + so switching threads mid-send cannot push a later part onto a different one. + A part that fails stops the batch and leaves the rest attached, saying how + many, rather than making you work out which of five went out. Capped at ten + files, because a stray drop of a folder should be refused and not become + eighty sends. Chips are one per row, like the received ones — a row of N + sums its implicit widths and drags the whole column off the panel. + - **GIFs move.** An animated GIF arrived as a still, and did so twice over. The inline-preview path asks the Mac to resample every image with sips, which flattens an animation to a single frame — a 1.4 MB GIF reached Linux as a diff --git a/collector.test.ts b/collector.test.ts index e234349..ccdc0ca 100644 --- a/collector.test.ts +++ b/collector.test.ts @@ -803,7 +803,7 @@ describe("a link that just arrived opens the share sheet", () => { test("every link of a message rides along; url stays the first", () => { const out = selectIncomingLinks([ - msg({ ts: "2026-09-02 12:30:00", from_me: false, text: "two: https://a.test/1, and https://b.test/2." }), + msg({ ts: "2026-09-02T12:30:00Z", from_me: false, text: "two: https://a.test/1, and https://b.test/2." }), ], WM, []); expect(out[0]!.url).toBe("https://a.test/1"); expect(out[0]!.urls).toEqual(["https://a.test/1", "https://b.test/2"]); @@ -882,13 +882,13 @@ describe("a re-keyed group is ONE conversation", () => { const email = "pat@example.com"; const out = foldThreadAliases( [ - thread(phone, "2026-09-04 21:32:29", 40, 0), - thread(email, "2026-09-05 17:24:01", 12, 1), + thread(phone, "2026-09-04T21:32:29Z", 40, 0), + thread(email, "2026-09-05T17:24:01Z", 12, 1), ], { [phone]: email }, ); expect(out).toHaveLength(1); - expect(out[0]).toMatchObject({ chat: email, last_ts: "2026-09-05 17:24:01", unread: 1 }); + expect(out[0]).toMatchObject({ chat: email, last_ts: "2026-09-05T17:24:01Z", unread: 1 }); }); }); diff --git a/thread.test.ts b/thread.test.ts index ea6aa03..c15165b 100644 --- a/thread.test.ts +++ b/thread.test.ts @@ -466,9 +466,9 @@ describe("selectThread", () => { const phone = "+15550100001"; const email = "pat@example.com"; const raw = [ - msg({ chat: phone, handle: phone, ts: "2026-09-04 21:32:29", text: "sms" }), - msg({ chat: email, handle: email, ts: "2026-09-05 17:24:01", text: "imessage" }), - msg({ chat: "+15550100002", handle: "+15550100002", ts: "2026-09-05 18:00:00", text: "other" }), + msg({ chat: phone, handle: phone, ts: "2026-09-04T21:32:29Z", text: "sms" }), + msg({ chat: email, handle: email, ts: "2026-09-05T17:24:01Z", text: "imessage" }), + msg({ chat: "+15550100002", handle: "+15550100002", ts: "2026-09-05T18:00:00Z", text: "other" }), ]; const aliases = { [phone]: email }; expect(selectThread(raw, email, false, 80, [], aliases).map((m) => m.text)) diff --git a/ui.test.ts b/ui.test.ts index ee0622b..00f1939 100644 --- a/ui.test.ts +++ b/ui.test.ts @@ -242,7 +242,9 @@ describe("QML safety invariants", () => { expect(panel).toContain("openThread(threads[i])"); const catcher = panel.slice(panel.indexOf("function catchNavText"), panel.indexOf("function catchEscape")); expect(catcher).toContain('text >= "1" && text <= "9"'); - expect(catcher).toContain("root.draftPath"); + // A queued attachment still blocks the 1-9 jump: with a draft armed the + // keystroke belongs to the caption, not to thread navigation. + expect(catcher).toContain("root.attachCount > 0"); expect(catcher).toContain("return handleTextKey(text) === true"); const fn = handleTextKeySource(); expect(fn).toContain("if (searching || newMode) return false"); @@ -371,7 +373,9 @@ describe("QML safety invariants", () => { test("bubble actions reuse the click handlers and never steal a real send", () => { // Enter/Ctrl+C/Ctrl+R act on the selection only with an empty field and // no queued file — a queued file's Enter is a send, and must stay one. - expect(panel).toContain('var b = empty && root.draftPath === "" ? root.selectedBubble() : null'); + // draftPath became attachCount when a message learned to carry SEVERAL + // files; the guard is the same one — no queued attachment. + expect(panel).toContain('var b = empty && root.attachCount === 0 ? root.selectedBubble() : null'); const open = qmlFunction("openBubble"); expect(open).toContain("openAttachment(b.attachments[0])"); expect(open).toContain("openShare(urls, false)"); // a link goes to the sheet, never straight to the browser @@ -624,3 +628,75 @@ test("the share sheet steps through a message's links", () => { expect(sheet).toContain('visible: root.shareQr !== "" || qrProc.running'); expect(qmlFunction("showShareUrl")).not.toContain('shareQr = ""'); }); + +describe("a message can carry several files (multi-file drafts)", () => { + function source(fn: string, until: string) { + const start = panel.indexOf(`function ${fn}`); + return panel.slice(start, panel.indexOf(until, start)); + } + + test("a drop attaches EVERY file, not just the first", () => { + // drop.urls[0] attached one photo of five and discarded the rest with no + // message — worse than refusing the drop. + expect(panel).not.toContain("var u = String(drop.urls[0])"); + expect(panel).toContain("for (var i = 0; i < drop.urls.length; i++)"); + expect(panel).toContain("root.addAttachments(paths)"); + }); + + test("the draft is a list, and it is capped", () => { + expect(panel).toContain("property var attachDrafts: []"); + expect(panel).toContain("readonly property int attachMax: 10"); + // A stray drop of a whole folder is refused, not turned into 80 sends. + const add = source("addAttachment", "function addAttachments"); + expect(add).toContain("root.attachDrafts.length >= root.attachMax"); + // the same file twice is one attachment + expect(add).toContain("root.attachDrafts[i].path === p"); + }); + + test("files ship one part at a time, and only the first carries the caption", () => { + const pump = source("pumpFileSend", "Process { id: copyProc }"); + // fileSendProc is a single Process: a second start would clobber the first. + expect(pump).toContain("if (fileSendProc.running) return"); + expect(pump).toContain("root.fileQueue = root.fileQueue.slice(1)"); + // caption over stdin, never argv (audit #4) + expect(pump).toContain("--caption-stdin"); + expect(pump).not.toContain("root.sendCaption]"); + // spent after the first part, so five files do not post one sentence five times + expect(pump).toContain('root.sendCaption = ""'); + }); + + test("only the part that shipped is retired; a failure keeps the rest attached", () => { + expect(panel).toContain("root.removeAttachment(root.sendDraftPath)"); + // mid-batch the field must not clear and focus must not jump + expect(panel).toContain("if (root.fileQueue.length > 0)"); + expect(panel).toContain("still attached"); + }); + + test("draft chips are one per row, never a RowLayout of N", () => { + // Summed implicit widths stretch the column past the panel and take every + // right-aligned element off-screen with it (CLAUDE.md). + expect(panel).toContain("id: attachList"); + expect(panel).toContain("model: root.attachDrafts"); + // each chip removes ITSELF, not the whole draft + expect(panel).toContain("root.removeAttachment(modelData.path)"); + expect(panel).toContain("attachList.width"); + }); + + test("switching threads still drops every queued file", () => { + // a queued file must never survive into another conversation + const clear = source("clearAttachments", "/** Ship the next queued file"); + expect(clear).toContain("root.attachDrafts = []"); + expect(clear).toContain("root.fileQueue = []"); + }); +}); + +describe("a multi-part send is pinned to the thread it started in", () => { + test("the service is captured once, not re-read per part", () => { + // root.active can change under a batch; a later part must not go out on a + // different service from the first (war room #2). + expect(panel).toContain("property string sendService"); + expect(panel).toContain('root.sendService !== "" ? ["--service", root.sendService] : []'); + const pump = panel.slice(panel.indexOf("function pumpFileSend"), panel.indexOf("Process { id: copyProc }")); + expect(pump).not.toContain("root.active.service"); + }); +}); From 0727e9160bcb74177da7e660836e0c83c015318d Mon Sep 17 00:00:00 2001 From: Zach Wilke Date: Mon, 7 Sep 2026 19:46:39 -0500 Subject: [PATCH 4/7] perf: stop paying a startup toll on every bridge call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blip felt subtly laggy rather than slow. Measuring said why: nothing is slow, everything pays startup. A bridge call cost ~133 ms while the SQL underneath ran in ~1 ms — 16 ms ssh round trip, ~25 ms for blip-dispatch's Python start, ~20 ms for imsg's, 27 ms to open a 218 MB chat.db. Worse, the shim fired a SECOND full ssh round trip before every call as a connectivity probe — 41 ms, ~31% of the total — bought only to turn a transport failure into a friendlier sentence. ssh reports that itself as 255, and every caller (collector, thread, fetch, avatar, search, send-file, contact-search) has always treated 255 exactly like 69. The probe is gone; the real call carries the news and the shim translates 255 → 69, so the documented contract is unchanged and ssh's own reason now reaches stderr, which is more than the probe's one-line tail ever carried. It also retires the trap that came with it: the probe HAD to be `ssh -n` or it ate stdin and silently emptied --file-stdin payloads. Verified with a stub ssh that stdin still reaches the remote command intact, that a transport failure exits 69, and that any other exit code passes through untouched. The push debounce came down from 250 ms to 60 ms. It is paid on every received message and on every send — Messages writing the row is itself a chat.db change, so the watch ping is what resolves the "Sending…" bubble — and at 250 ms it was more than twice the cost of the 116 ms fetch it was coalescing. A burst still costs one fetch. The post-send reload fell from 600 ms to 250 ms and is now the fall-back for a downed watcher rather than the mechanism. Measured, warm, medians: bridge call 132 → ~95 ms, open a conversation 223 → ~185 ms, poll 160 → ~118 ms, deep poll 707 → ~590 ms. Two things the measurements talked us out of, both recorded in CLAUDE.md so nobody re-tries them: replacing cmd_chats' 300-query N+1 with a single window function is SLOWER (70 → 107 ms, the per-chat lookups are indexed), and bundling the deep poll's three sequential calls into one would be undone by the persistent channel coming next. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015xxS9WCT7ifHhbDjYyGnHS --- BarWidget.qml | 9 +++++++- BlipView.qml | 9 +++++++- CHANGELOG.md | 25 ++++++++++++++++++++++ CLAUDE.md | 28 ++++++++++++++++++++++--- bridge/linux/blip-shim | 47 +++++++++++++++++++++++++++++------------- 5 files changed, 99 insertions(+), 19 deletions(-) diff --git a/BarWidget.qml b/BarWidget.qml index 381933e..0efec4d 100644 --- a/BarWidget.qml +++ b/BarWidget.qml @@ -552,7 +552,14 @@ BarWidget { } Timer { id: pingDebounce - interval: 250 + // 60 ms, not 250: this delay is paid on EVERY received message and on + // every send (Messages writing the row is itself a chat.db change, so the + // ping is what resolves the "Sending…" bubble). A burst still costs one + // fetch — messages in a burst land milliseconds apart, far inside 60 ms — + // but a single message no longer waits a quarter second before anything + // starts. Measured: the fetch it triggers is ~116 ms, so the debounce was + // more than twice the cost of the work it was coalescing. + interval: 60 onTriggered: { var panel = panelLoader.item // Carrying the open thread's chat means a message landing in the diff --git a/BlipView.qml b/BlipView.qml index 24a0f58..f30ba0d 100644 --- a/BlipView.qml +++ b/BlipView.qml @@ -1793,7 +1793,14 @@ FocusScope { // returning; a reload that beats it keeps the pending bubble (thread.ts) // and tries again. No `loading` flag: the bubble is already on screen, // and a "loading…" flash after every send is the thing we are removing. - interval: 600 + // + // This is now the FALL-BACK, not the mechanism: Messages writing the row + // is a chat.db change, so `imsg watch` pings and the 60 ms push debounce + // reloads the open conversation well before this fires. It stays for the + // case where the watcher is down (sleep, network) and the 6 s poll is all + // there is — 250 ms rather than 600 because a retry that arrives early + // costs one cheap reload and keeps the pending bubble anyway. + interval: 250 onTriggered: if (root.inThread && String(root.active.chat) === root.reloadChat) { root.requestThreadLoad(root.reloadChat) } diff --git a/CHANGELOG.md b/CHANGELOG.md index f98d8af..6b35e5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,31 @@ ## Unreleased +- **The bridge stopped paying a toll on every call.** Blip felt subtly laggy + rather than slow, and measuring said why: nothing was slow, everything paid + startup. A bridge call cost ~133 ms while the SQL underneath ran in about a + millisecond — 16 ms ssh, ~25 ms for `blip-dispatch`'s Python start, ~20 ms + for `imsg`'s, 27 ms to open a 218 MB chat.db. On top of that the shim fired + a *second* full ssh round trip before every call, purely as a connectivity + probe: 41 ms, a third of the total, to turn a transport failure into a + friendlier sentence. ssh already reports that as 255 and every caller in + Blip has always treated 255 exactly like 69, so the probe is gone and the + real call carries the news; the `69 → Blip greys out` contract is unchanged. + That also retires the trap it came with — the probe had to be `ssh -n` or it + ate stdin and silently emptied `--file-stdin` payloads. + The push debounce came down from 250 ms to 60 ms. It is paid on every + received message *and* on every send (Messages writing the row is itself a + chat.db change), and at 250 ms it was more than twice the cost of the 116 ms + fetch it was coalescing; a burst still costs one fetch. The post-send reload + fell from 600 ms to 250 ms, now a fall-back for when the watcher is down + rather than the mechanism. + Measured, warm: a bridge call 132 → ~95 ms, opening a conversation 223 → + ~185 ms, a poll 160 → ~118 ms, a deep poll 707 → ~590 ms. + Two things measurement talked us OUT of: replacing `cmd_chats`' 300-query + N+1 with one window function is slower (70 → 107 ms; the per-chat lookups + are indexed), and bundling the deep poll's three calls into one would be + undone by the persistent channel coming next. + - **A message can carry several files.** Dropping five photos on a conversation attached one and threw the other four away without a word: the drop handler read `urls[0]` and the draft was a single path. Drafts are a diff --git a/CLAUDE.md b/CLAUDE.md index 2f00cba..5499bfd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -148,9 +148,18 @@ what it is handed. Keep it that way. them by design; do not persist them. `push-read.log` beside state.json records each read-push's exit code and `imsg-read`'s status line — never content. -- **The Linux shims' ssh preflight must use `ssh -n`.** A bare - `ssh true` connectivity probe EATS STDIN, which silently empties - `imsg-send --file-stdin` payloads. Fixed 2026-08-31. +- **There is NO ssh preflight in the shim, and adding one back is a bug.** + It cost a full ssh round trip plus a Python start on the Mac — 41 ms + measured, on every call, ~31% of a 133 ms query — and bought only a + friendlier error string. ssh reports transport failure as 255 itself, every + caller already treats 255 exactly like 69 (collector, thread, fetch, avatar, + search, send-file, contact-search), and the shim translates it so the + documented `69 → Blip greys out` contract is unchanged. This also retires + the trap that came with it: the probe HAD to be `ssh -n`, because a bare + `ssh true` EATS STDIN and silently empties `imsg-send --file-stdin` + payloads. No probe, no trap. The shim no longer `exec`s ssh (it needs the + exit code to translate); stdio is inherited either way, so attachment + streaming and `--file-stdin` are untouched — covered by a stub-ssh test. - **`bridge.conf` is data, never `source`d.** The shim parses four keys and validates them; keep it that way (audit #7). `automation=on` is what lets `qs ipc … goto/compose/bubbles` work — off, they return a refusal string. @@ -303,6 +312,19 @@ what it is handed. Keep it that way. Attachment chips are one per row for this reason. Debug trick: log `bubbleRow.width` per delegate; 1136 in a 560 panel = this bug. +- **Latency is startup, not work — measure before tuning.** Every bridge call + pays a fixed tax while the SQL underneath is ~1 ms: ssh round trip 16 ms, + `blip-dispatch` Python start ~25 ms, `imsg` Python start + module parse + ~20 ms, sqlite connect (218 MB chat.db) 27 ms. That is why Blip feels + subtly laggy rather than slow, and why "make the query faster" is almost + always the wrong move. Two measured examples: replacing `cmd_chats`' + 300-query N+1 with a single window function made it SLOWER (70 ms → 107 ms) + — the per-chat lookups are indexed; and in `cmd_chats` name resolution + costs ~104 ms, of which 46 ms is 60,112 `_same_number` calls, because a + handle that misses the exact last-ten key falls back to a linear scan of + every contact. Profile on the Mac with `cProfile` + `runpy` before changing + anything (see the war-room note on wheel scrolling for the same lesson). + ## Working on it ``` diff --git a/bridge/linux/blip-shim b/bridge/linux/blip-shim index 2796b2c..698be35 100755 --- a/bridge/linux/blip-shim +++ b/bridge/linux/blip-shim @@ -14,8 +14,9 @@ # holding the tools; default ~/.blip/bin), python (default python3), # automation (on|off — read by the plugin, not the shim). # -# The connectivity probe MUST be `ssh -n`: a bare probe eats stdin and -# silently empties `imsg-send --file-stdin` payloads (learned the hard way). +# There is deliberately NO connectivity probe — see the note above the ssh +# call at the bottom. The old one cost 41 ms on every call and had to be +# `ssh -n` or it ate stdin and silently emptied --file-stdin payloads. set -euo pipefail CONF="${BLIP_BRIDGE_CONF:-$HOME/.config/blip/bridge.conf}" @@ -57,23 +58,41 @@ case "$tool" in esac # With the dedicated key (blip-setup enrolls it as restrict,command=blip-dispatch) -# the Mac side can ONLY run the bridge tools, so the remote command is bare and -# the probe is dispatch's "ping". The key gets its own ControlMaster socket: -# sharing the general key's master would silently bypass the forced command. +# the Mac side can ONLY run the bridge tools, so the remote command is bare. +# The key gets its own ControlMaster socket: sharing the general key's master +# would silently bypass the forced command. if [[ -r "$key" ]]; then ssh=(ssh -i "$key" -o IdentitiesOnly=yes -o ControlMaster=auto -o "ControlPath=$HOME/.ssh/cm-blip-%r@%h:%p" -o ControlPersist=10m) - probe="ping"; remote="$tool ${*@Q}" + remote="$tool ${*@Q}" else ssh=(ssh) - probe="true"; remote="PATH=/opt/homebrew/bin:/usr/local/bin:\$PATH $python $remote_bin/$tool ${*@Q}" + remote="PATH=/opt/homebrew/bin:/usr/local/bin:\$PATH $python $remote_bin/$tool ${*@Q}" fi -if ! err="$("${ssh[@]}" -n -o ConnectTimeout=5 -o BatchMode=yes -- "$host" "$probe" 2>&1 >/dev/null)"; then - # keep the exit code (69 → Blip greys out) but say why: auth, host key, dispatch refusal… - echo "$tool: $host unreachable — the iMessage bridge is offline.${err:+ ($(printf '%s' "$err" | tail -n1))}" >&2 - exit 69 # EX_UNAVAILABLE (Blip greys out on this code) -fi - +# NO PREFLIGHT PROBE. There used to be one, and it cost a full ssh round trip +# plus a Python start on the Mac — 41 ms measured, on EVERY call, ~31% of a +# 133 ms query — purely to turn a transport failure into a friendlier message. +# ssh already reports that itself as 255, and every caller in Blip treats 255 +# exactly like 69 (collector, thread, fetch, avatar, search, send-file, +# contact-search), so the real call carries the news at no cost. We still +# translate it, so the documented 69 contract is unchanged. +# +# This also retires the old "the probe MUST be ssh -n" trap: a probe without +# it ate stdin and silently emptied `imsg-send --file-stdin` payloads. No +# probe, no trap — do not add one back. +# # ${*@Q} quotes each argument for the remote side (dispatch shlex-splits it # exactly like a shell): apostrophes, spaces, leading dashes survive intact. -exec "${ssh[@]}" -- "$host" "$remote" +# Not exec: we need the exit code to translate it. stdio is inherited either +# way, so attachment streaming and --file-stdin are untouched. +set +e +"${ssh[@]}" -o ConnectTimeout=5 -o BatchMode=yes -- "$host" "$remote" +rc=$? +set -e +if (( rc == 255 )); then + # ssh has already printed its own reason (auth, host key, dispatch refusal) + # on stderr, which is more than the probe's one-line tail ever carried. + echo "$tool: $host unreachable — the iMessage bridge is offline." >&2 + exit 69 # EX_UNAVAILABLE (Blip greys out on this code) +fi +exit "$rc" From 5b8b01eecbc308ca127decaea9569987b8911a60 Mon Sep 17 00:00:00 2001 From: Zach Wilke Date: Mon, 7 Sep 2026 19:59:08 -0500 Subject: [PATCH 5/7] perf: hold the Mac channel open, so a query costs the query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the preflight probe gone a bridge call still paid ~90 ms before it read a single row: the ssh round trip, blip-dispatch's Python start, imsg's own, and 27 ms to open a 218 MB chat.db — for SQL that runs in about a millisecond. Startup, not work, was the whole of the lag. `imsg serve` answers many requests on one process with chat.db already open. Requests are re-dispatched through build_parser(), the SAME parser the CLI uses, so there is no second command table to drift into a second security surface, and only read-only queries are reachable (SERVE_ALLOWED): attachment and avatar stream binary and would park the channel behind a 100 MB photo, watch blocks forever by design, serve would recurse. Framing is a {"status","len"} line plus exactly len bytes — counted, not escaped, or a 122 KB payload would be re-encoded into a JSON string. A request carries its stdin separately, so message text still never rides argv. blip-bridged holds two of those channels open behind a unix socket in XDG_RUNTIME_DIR. Two, not one: Blip refreshes the thread list and reloads the open conversation in parallel on every push ping, and a single channel would have queued the reload behind a 259 ms `chats` call — making the case it was meant to help slower. The leader BarWidget runs it, exactly as it already runs the push watcher, which is why Blip still needs no daemon of its own. Nothing depends on it. No socket, no socat, a daemon that died mid-request, a frame that will not parse, a short body — bridgeRun() falls through to the ordinary one-shot ssh path on any fault at all. A caller that passes its own runner never touches the socket, which is what keeps the existing suite honest: those tests inject a runner and assert on real argv. Measured end to end, medians: opening a conversation 223 → ~105 ms, a poll 160 → ~36 ms, a deep poll 707 → ~370 ms. On the channel itself, recent 150 went 100 → 18 ms and groups 145 → 26 ms; `chats` stays ~260 ms because it is compute-bound (179 ms SQL, 104 ms name resolution), not startup-bound. One trap worth the comment it now carries: `encoding: "buffer"` throws ERR_UNKNOWN_ENCODING in Bun's spawnSync, the catch swallowed it, and the fast path silently never ran while every call quietly took the slow one. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015xxS9WCT7ifHhbDjYyGnHS --- .github/workflows/ci.yml | 4 +- BarWidget.qml | 27 ++++ CHANGELOG.md | 19 +++ CLAUDE.md | 23 ++++ bridge/linux/blip-bridged | 262 ++++++++++++++++++++++++++++++++++++++ bridge/mac/imsg | 110 +++++++++++++++- bridge/mac/test_serve.py | 60 +++++++++ collector.ts | 100 +++++++++++++-- scripts/blip-setup | 13 +- thread.test.ts | 5 +- thread.ts | 6 +- tier2.test.ts | 40 ++++++ ui.test.ts | 10 ++ 13 files changed, 659 insertions(+), 20 deletions(-) create mode 100755 bridge/linux/blip-bridged create mode 100644 bridge/mac/test_serve.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9756b4..6a336f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: - name: unit tests (collector, thread, tier2, ui invariants) run: bun test - name: Mac tools compile (python3 syntax, no macOS needed) - run: python3 -m py_compile bridge/mac/imsg bridge/mac/imsg-send bridge/mac/contacts bridge/mac/tcc-check bridge/mac/blip-check bridge/mac/blip-dispatch bridge/mac/imsg-read bridge/mac/calling_codes.py + run: python3 -m py_compile bridge/linux/blip-bridged bridge/mac/imsg bridge/mac/imsg-send bridge/mac/contacts bridge/mac/tcc-check bridge/mac/blip-check bridge/mac/blip-dispatch bridge/mac/imsg-read bridge/mac/calling_codes.py - name: group-photo lookup (no macOS needed) run: python3 bridge/mac/test_group_photo.py - name: phone matching, the way Contacts does it (no macOS needed) @@ -27,6 +27,8 @@ jobs: run: python3 bridge/mac/test_cluster.py - name: stamps cross the bridge as UTC (no macOS needed) run: python3 bridge/mac/test_wire_time.py + - name: the serve channel exposes only read-only queries (no macOS needed) + run: python3 bridge/mac/test_serve.py - name: shell scripts (bash -n + shellcheck) run: | bash -n bridge/linux/blip-shim scripts/blip-setup scripts/sync-bridge.sh scripts/demo/blip-shots bridge/mac/install.sh diff --git a/BarWidget.qml b/BarWidget.qml index 0efec4d..da26343 100644 --- a/BarWidget.qml +++ b/BarWidget.qml @@ -509,6 +509,33 @@ BarWidget { root.activeReadChat(), root.activeSeenTs()) } + // ------------------------------------------------- the accelerator channel + // blip-bridged holds `imsg serve` channels open to the Mac, so a query costs + // the query instead of ~90 ms of ssh + two Python starts + opening a 218 MB + // chat.db. Measured end to end: opening a conversation 223 → 103 ms, a poll + // 160 → 38 ms. It rides here for the same reason the watcher does — the + // shell is the process that outlives every one-shot script, which is why + // Blip needs no daemon of its own — and only on the LEADER bar, or every + // monitor would hold its own pair of Mac processes. + // + // Nothing depends on it: collector.ts and thread.ts fall back to the plain + // one-shot ssh path whenever the socket is missing or a frame does not + // parse, so a failure here costs speed and nothing else. + Process { + id: bridgeProc + command: [root.home + "/bin/blip-bridged"] + running: root.leader + onExited: bridgeRestart.restart() + } + Timer { + id: bridgeRestart + // It exits on its own when idle; the poller keeps it busy in practice, so + // this is mostly the crash path. Slow enough not to spin if the binary is + // missing entirely (a setup that never installed it). + interval: 30000 + onTriggered: if (root.leader) bridgeProc.running = true + } + // ------------------------------------------------- real-time push // `imsg watch` blocks on the Mac and emits one line per chat.db change — an // INVALIDATION, no content (BlueFerry's design: session-visible push diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b35e5a..b95e2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## Unreleased +- **A persistent channel to the Mac.** Even with the probe gone, every query + still paid ~90 ms before it read a row: ssh, `blip-dispatch`'s Python start, + `imsg`'s own, and opening a 218 MB chat.db — for SQL that takes about a + millisecond. `imsg serve` now answers many requests on one long-lived + process with chat.db already open, and `blip-bridged` on the Linux side + keeps two of those channels up (two, because Blip refreshes the list and + reloads the open conversation in parallel, and a single channel would queue + the reload behind a 259 ms `chats` call). The bar widget starts it, on the + leader bar only, exactly as it already runs the push watcher — which is why + Blip still needs no daemon of its own. + It is an accelerator and never a dependency: no socket, no socat, a daemon + that died mid-request, a frame that will not parse — anything at all — and + the ordinary one-shot ssh path answers instead. The channel carries + read-only queries only; attachments and avatars stream binary and would park + it behind a 100 MB photo, and `watch` blocks by design, so those stay + one-shot. Message text still rides stdin rather than argv. + Measured end to end: opening a conversation 223 → ~105 ms, a poll 160 → + ~36 ms, a deep poll 707 → ~370 ms. + - **The bridge stopped paying a toll on every call.** Blip felt subtly laggy rather than slow, and measuring said why: nothing was slow, everything paid startup. A bridge call cost ~133 ms while the SQL underneath ran in about a diff --git a/CLAUDE.md b/CLAUDE.md index 5499bfd..e5d89fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -325,6 +325,29 @@ what it is handed. Keep it that way. every contact. Profile on the Mac with `cProfile` + `runpy` before changing anything (see the war-room note on wheel scrolling for the same lesson). +- **The persistent channel is an ACCELERATOR; the one-shot path is the + contract.** `blip-bridged` (Linux, started by the LEADER BarWidget) holds + two `imsg serve` channels open over ssh, so a query costs the query instead + of ~90 ms of startup. `bridgeRun()` in collector.ts routes through it and + falls back to plain `~/bin/imsg` on ANY fault — no socket, no socat, a dead + daemon, a frame that will not parse, a short body. Three rules: + (1) A caller that passes its own `runner` NEVER touches the socket. That is + what keeps the suite honest — every existing test injects a runner and + asserts on real argv. + (2) `serve` answers read-only queries only (`SERVE_ALLOWED`). `attachment` + and `avatar` stream binary and would park the channel behind a 100 MB photo; + `watch` blocks forever; `serve` would recurse. Those stay one-shot. + (3) A request carries `stdin` separately, so message text still never rides + argv. Framing is `{"status","len"}\n` + exactly len BYTES — length-counted, + not escaped, or a 122 KB payload gets re-encoded into a JSON string. + Channels are POOLED (2): Blip refreshes the list and reloads the open + conversation in parallel on every ping, and one channel would queue the + reload behind a 259 ms `chats` call. Do not pass `encoding: "buffer"` to + Bun's `spawnSync` — it throws `ERR_UNKNOWN_ENCODING`, the catch swallows it, + and the fast path silently never runs (cost us a whole debugging round). + **"Offline" in a test means clearing HOME *and* XDG_RUNTIME_DIR**, or the + socket answers underneath the test. + ## Working on it ``` diff --git a/bridge/linux/blip-bridged b/bridge/linux/blip-bridged new file mode 100755 index 0000000..01e481e --- /dev/null +++ b/bridge/linux/blip-bridged @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""blip-bridged — keeps `imsg serve` channels open to the Mac. + +Every one-shot bridge call pays a fixed toll before it reads a single row: +~16 ms ssh round trip, ~25 ms for blip-dispatch's Python start, ~20 ms for +imsg's own, and ~27 ms to open a 218 MB chat.db — while the SQL underneath +runs in about a millisecond. That toll, not the work, is what made Blip feel +subtly laggy. This daemon pays it once per channel and holds the channel open, +so a query costs the query. + +Measured on the gateway Mac, one-shot → on the channel: + recent 150 100 ms → 11 ms + groups 145 ms → 25 ms + thread 40 rich 197 ms → 71 ms + chats 300 341 ms → 259 ms (compute-bound, not startup-bound) + +Shape +----- +A unix socket in $XDG_RUNTIME_DIR/blip (user-only, 0700). One connection is +one request: the client writes a single JSON line and reads a framed reply, +then the connection closes. No client-side request ids, no multiplexing. + + -> {"argv": ["--json","recent","150"], "stdin": "optional"} + <- {"status":0,"len":N}\n followed by exactly N bytes + +`stdin` is passed through to the command, so message text keeps riding stdin +rather than argv (`search --stdin`) exactly as it does one-shot. + +Channels are POOLED. Blip refreshes the thread list and reloads the open +conversation in parallel on every push ping; with a single channel the reload +would queue behind a 259 ms `chats` call and the fix would make the case it +was meant to help slower. + +This daemon is an accelerator and never a dependency. If it is not running, +cannot start, or dies mid-flight, the shim's ordinary one-shot ssh path still +answers — the client falls back on any error at all. That is also why it may +exit freely when idle. +""" +from __future__ import annotations + +import errno +import json +import os +import socket +import subprocess +import sys +import threading +import time + +POOL_SIZE = 2 +IDLE_EXIT_SECONDS = 15 * 60 +CONNECT_TIMEOUT = "8" + + +def runtime_dir() -> str: + base = os.environ.get("XDG_RUNTIME_DIR") or f"/tmp/blip-{os.getuid()}" + d = os.path.join(base, "blip") + os.makedirs(d, mode=0o700, exist_ok=True) + return d + + +def socket_path() -> str: + return os.path.join(runtime_dir(), "bridge.sock") + + +def read_conf() -> dict: + """The same four keys the shim parses, with the same rules: the file is + DATA, never sourced, and remote_bin stays literal (the Mac expands it).""" + path = os.environ.get("BLIP_BRIDGE_CONF", os.path.expanduser("~/.config/blip/bridge.conf")) + conf = {"host": "", "key": os.path.expanduser("~/.ssh/blip_ed25519"), + "remote_bin": "$HOME/.blip/bin", "python": "python3"} + try: + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.split("#", 1)[0] + line = "".join(line.split()) + if "=" not in line: + continue + k, v = line.split("=", 1) + v = v.strip("\"'") + if k in conf: + conf[k] = v + except OSError: + pass + return conf + + +class Channel: + """One `imsg serve` process behind one ssh session.""" + + def __init__(self, conf: dict) -> None: + self.conf = conf + self.lock = threading.Lock() + self.proc: subprocess.Popen | None = None + + def _spawn(self) -> None: + conf = self.conf + key = conf["key"] + if os.access(key, os.R_OK): + # The dedicated key is enrolled with command="blip-dispatch", so the + # remote command is bare and dispatch execs the tool. Its own + # ControlMaster socket: sharing the general key's master would + # silently bypass the forced command. + ssh = ["ssh", "-i", key, "-o", "IdentitiesOnly=yes", + "-o", "ControlMaster=auto", + "-o", f"ControlPath={os.path.expanduser('~')}/.ssh/cm-blip-%r@%h:%p", + "-o", "ControlPersist=10m"] + remote = "imsg serve" + else: + ssh = ["ssh"] + remote = f"PATH=/opt/homebrew/bin:/usr/local/bin:$PATH {conf['python']} {conf['remote_bin']}/imsg serve" + cmd = ssh + ["-o", f"ConnectTimeout={CONNECT_TIMEOUT}", "-o", "BatchMode=yes", + "--", conf["host"], remote] + self.proc = subprocess.Popen( + cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, bufsize=0) + line = self.proc.stdout.readline() + if line.strip() != b"ready": + self.kill() + raise RuntimeError("serve channel did not come up") + + def kill(self) -> None: + if self.proc is not None: + try: + self.proc.kill() + except OSError: + pass + self.proc = None + + def request(self, payload: bytes) -> bytes: + """Send one request line, return the framed reply (header + body). + + Any fault kills the channel rather than leaving it half-consumed: a + desynchronised channel would hand the NEXT caller someone else's + bytes, which is far worse than a client falling back to one-shot. + """ + with self.lock: + if self.proc is None or self.proc.poll() is not None: + self._spawn() + try: + self.proc.stdin.write(payload) + self.proc.stdin.flush() + header = self.proc.stdout.readline() + if not header: + raise RuntimeError("serve channel closed") + meta = json.loads(header) + want = int(meta["len"]) + body = b"" + while len(body) < want: + chunk = self.proc.stdout.read(want - len(body)) + if not chunk: + raise RuntimeError("short body from serve channel") + body += chunk + return header + body + except Exception: + self.kill() + raise + + +class Pool: + def __init__(self, conf: dict, size: int = POOL_SIZE) -> None: + self.channels = [Channel(conf) for _ in range(size)] + self.turn = threading.Semaphore(size) + self.free = list(self.channels) + self.guard = threading.Lock() + + def request(self, payload: bytes) -> bytes: + self.turn.acquire() + with self.guard: + ch = self.free.pop() + try: + return ch.request(payload) + finally: + with self.guard: + self.free.append(ch) + self.turn.release() + + def shutdown(self) -> None: + for ch in self.channels: + ch.kill() + + +def serve_client(conn: socket.socket, pool: Pool) -> None: + conn.settimeout(30) + try: + buf = b"" + while b"\n" not in buf: + chunk = conn.recv(65536) + if not chunk: + return + buf += chunk + if len(buf) > 1 << 20: # a request line is argv, not a payload + return + line, _ = buf.split(b"\n", 1) + reply = pool.request(line + b"\n") + conn.sendall(reply) + except Exception: + # Say nothing and close: a client that gets no usable frame falls back + # to the one-shot path, which is the whole safety story here. + pass + finally: + try: + conn.close() + except OSError: + pass + + +def main() -> None: + conf = read_conf() + if not conf["host"]: + sys.exit("blip-bridged: no Mac configured (host= in bridge.conf)") + + path = socket_path() + srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + # A stale socket from a killed daemon must not keep the fast path down + # forever, but a LIVE one means we are the second copy and should go. + try: + probe = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + probe.settimeout(1) + probe.connect(path) + probe.close() + return # somebody is already serving + except OSError as e: + if e.errno not in (errno.ECONNREFUSED, errno.ENOENT): + return + if os.path.exists(path): + os.unlink(path) + old = os.umask(0o077) + try: + srv.bind(path) + finally: + os.umask(old) + srv.listen(16) + except OSError as e: + sys.exit(f"blip-bridged: cannot listen on {path}: {e}") + + pool = Pool(conf) + last_seen = time.time() + srv.settimeout(30) + try: + while True: + try: + conn, _ = srv.accept() + except socket.timeout: + if time.time() - last_seen > IDLE_EXIT_SECONDS: + break # nothing wants us; the shim restarts us on demand + continue + last_seen = time.time() + threading.Thread(target=serve_client, args=(conn, pool), daemon=True).start() + except KeyboardInterrupt: + pass + finally: + pool.shutdown() + try: + os.unlink(path) + except OSError: + pass + + +if __name__ == "__main__": + main() diff --git a/bridge/mac/imsg b/bridge/mac/imsg index 88271e8..ad58363 100755 --- a/bridge/mac/imsg +++ b/bridge/mac/imsg @@ -25,6 +25,7 @@ import argparse import datetime as dt import functools import html +import io import json import os import plistlib @@ -1920,7 +1921,105 @@ def cmd_analyze(con, args): print() -def main() -> None: +# Commands reachable on the serve channel: read-only queries only. +# attachment/avatar stream binary and would park the channel behind a 100 MB +# photo; watch blocks forever by design; serve would recurse. Those keep the +# one-shot path, which still works and is what everything falls back to. +SERVE_ALLOWED = {"recent", "from", "thread", "search", "contacts", "chats", "groups", "analyze"} + + +def cmd_serve(con, args) -> None: + """A long-lived request channel: one process, one open chat.db, many queries. + + A one-shot call pays ~87 ms before it reads a single row — blip-dispatch's + Python start, imsg's own, and opening a 218 MB chat.db — while the SQL + underneath runs in about a millisecond. On this channel that is paid once, + at startup, and every later request is just the query. + + Protocol, newline-delimited request, framed response: + + -> {"argv": ["--json","--rich","thread","--chat","","40"]} + <- {"status":0,"len":N}\n followed by exactly N bytes + + A request may carry "stdin", which becomes the command's stdin — so + message text keeps riding stdin instead of argv (`search --stdin`) exactly + as it does one-shot. Length framing rather than escaping keeps a 122 KB + JSON payload from being re-encoded into a JSON string. + + Requests are re-dispatched through build_parser(), the SAME parser the CLI + uses: a second hand-rolled command table would drift and become a second + security surface. A bad request answers with a status and the loop lives — + one malformed line must not take the channel down. + """ + parser = build_parser() + out = sys.stdout.buffer + stdin = sys.stdin # bound once: commands swap sys.stdin below + + def reply(status: int, payload: bytes) -> None: + out.write(json.dumps({"status": status, "len": len(payload)}).encode() + b"\n") + out.write(payload) + out.flush() + + out.write(b"ready\n") + out.flush() + + for line in stdin: + line = line.strip() + if not line: + continue + try: + req = json.loads(line) + argv = req["argv"] + if not isinstance(argv, list) or not all(isinstance(a, str) for a in argv): + raise ValueError("argv must be a list of strings") + except Exception as e: + reply(64, f"serve: bad request: {e}\n".encode()) + continue + + # argparse writes usage/errors to stdout or stderr and raises + # SystemExit; both must be kept off the wire or they corrupt framing. + buf = io.StringIO() + real_out, real_err, real_in = sys.stdout, sys.stderr, sys.stdin + sys.stdout = sys.stderr = buf + try: + ns = parser.parse_args(argv) + except SystemExit: + sys.stdout, sys.stderr, sys.stdin = real_out, real_err, real_in + reply(64, ("serve: unusable command: " + buf.getvalue()).encode("utf-8")) + continue + except Exception as e: + sys.stdout, sys.stderr, sys.stdin = real_out, real_err, real_in + reply(64, f"serve: unusable command: {e}\n".encode()) + continue + + if getattr(ns, "cmd", "") not in SERVE_ALLOWED: + sys.stdout, sys.stderr, sys.stdin = real_out, real_err, real_in + reply(64, f"serve: '{getattr(ns, 'cmd', '?')}' is not available on this channel\n".encode()) + continue + + buf = io.StringIO() + sys.stdout = buf + sys.stderr = real_err # a warning belongs in the ssh log, not the payload + sys.stdin = io.StringIO(req.get("stdin") or "") + status = 0 + try: + ns.func(con, ns) + except BrokenPipeError: + raise + except Exception as e: + # One bad row, one bad request — never the whole channel. + buf = io.StringIO() + buf.write(f"imsg: {e}\n") + status = 1 + finally: + sys.stdout, sys.stderr, sys.stdin = real_out, real_err, real_in + reply(status, buf.getvalue().encode("utf-8")) + + +def build_parser() -> argparse.ArgumentParser: + """The full CLI. Split out of main() so `serve` can re-dispatch requests + through the EXACT same parser — a second, hand-rolled command table would + drift from this one and become a second security surface.""" p = argparse.ArgumentParser(prog="imsg", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) # Top-level convenience flags (also added to each subparser via parents= @@ -1979,8 +2078,15 @@ def main() -> None: s = sub.add_parser("watch", parents=[common], help="emit a line whenever chat.db changes (push channel)") s.set_defaults(func=cmd_watch) + s = sub.add_parser("serve", parents=[common], + help="long-lived request channel: one open chat.db, many queries") + s.set_defaults(func=cmd_serve) - args = p.parse_args() + return p + + +def main() -> None: + args = build_parser().parse_args() with open_db() as con: args.func(con, args) diff --git a/bridge/mac/test_serve.py b/bridge/mac/test_serve.py new file mode 100644 index 0000000..70a6687 --- /dev/null +++ b/bridge/mac/test_serve.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""The serve channel's reachable surface. + +`imsg serve` re-dispatches requests through the SAME parser the CLI uses, so +there is no second command table to drift. What it must NOT do is widen what +the confined ssh key can reach: the channel answers read-only queries and +nothing else.""" +from __future__ import annotations + +import sys +import unittest +from importlib.machinery import SourceFileLoader +from importlib.util import module_from_spec, spec_from_loader +from pathlib import Path + +HERE = Path(__file__).parent + + +def load(tool: str): + loader = SourceFileLoader(f"blip_{tool}", str(HERE / tool)) + spec = spec_from_loader(loader.name, loader) + assert spec is not None + mod = module_from_spec(spec) + sys.modules[loader.name] = mod + loader.exec_module(mod) + return mod + + +imsg = load("imsg") + + +class ServeSurface(unittest.TestCase): + def test_only_read_only_queries_are_reachable(self): + self.assertEqual( + imsg.SERVE_ALLOWED, + {"recent", "from", "thread", "search", "contacts", "chats", "groups", "analyze"}, + ) + + def test_streaming_and_blocking_commands_stay_one_shot(self): + # attachment/avatar write binary and would park the channel behind a + # 100 MB photo; watch blocks forever; serve would recurse. + for cmd in ("attachment", "avatar", "watch", "serve"): + self.assertNotIn(cmd, imsg.SERVE_ALLOWED, f"{cmd} must not be reachable on the channel") + + def test_serve_reuses_the_real_parser(self): + # A second, hand-rolled command table would drift from the CLI and + # become a second security surface. + p = imsg.build_parser() + ns = p.parse_args(["--json", "--rich", "thread", "--chat", "chat123", "40"]) + self.assertEqual(ns.cmd, "thread") + self.assertEqual(ns.chat, "chat123") + self.assertIn(ns.cmd, imsg.SERVE_ALLOWED) + + def test_serve_is_a_real_subcommand(self): + ns = imsg.build_parser().parse_args(["serve"]) + self.assertEqual(ns.func, imsg.cmd_serve) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/collector.ts b/collector.ts index 06b13d8..79e7abd 100644 --- a/collector.ts +++ b/collector.ts @@ -21,8 +21,8 @@ import { openSync, writeSync, fsyncSync, closeSync } from "node:fs"; import { homedir } from "node:os"; import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { chmodSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; -import { dirname } from "node:path"; +import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; const HOME = process.env.HOME ?? homedir(); @@ -1032,11 +1032,92 @@ export function explainBridgeError(status: number | null, stderr: string): strin return last || `imsg exit ${status}`; } -export function fetchMessages(limit: number, runner = spawnSync): FetchResult { - const res = runner(`${HOME}/bin/imsg`, ["--json", "recent", String(limit)], { - encoding: "utf8", - timeout: 15000, maxBuffer: 64 * 1024 * 1024, +// ------------------------------------------------- the accelerator channel + +/** Where blip-bridged listens, when it is running. */ +export const BRIDGE_SOCK = join( + process.env.XDG_RUNTIME_DIR || `/tmp/blip-${typeof process.getuid === "function" ? process.getuid() : ""}`, + "blip", "bridge.sock", +); + +/** + * Run one bridge command, over the persistent channel when there is one. + * + * A one-shot `~/bin/imsg` call pays ~90 ms of pure startup before it reads a + * row — ssh, blip-dispatch's Python, imsg's own, and opening a 218 MB + * chat.db — while the query underneath takes about a millisecond. blip-bridged + * holds `imsg serve` channels open so that toll is paid once; `recent 150` + * measured 100 ms one-shot against 13 ms here. + * + * It is an ACCELERATOR, never a dependency. No socket, no socat, a daemon + * that died mid-request, a frame that does not parse — anything at all — and + * this falls through to the ordinary one-shot path, which is still the only + * thing that has to work. That is also what keeps the tests honest: they + * inject their own `runner`, and a caller with a custom runner never touches + * the socket, so every existing test still exercises the real argv. + */ +export function bridgeRun( + argv: string[], + runner: typeof spawnSync = spawnSync, + opts: { input?: string; timeout?: number; maxBuffer?: number } = {}, +): { status: number | null; stdout: string; stderr: string; error?: Error } { + const timeout = opts.timeout ?? 15000; + const maxBuffer = opts.maxBuffer ?? 64 * 1024 * 1024; + if (runner === spawnSync) { + const fast = viaBridgeSocket(argv, opts.input, timeout, maxBuffer); + if (fast) return fast; + } + const res = runner(`${HOME}/bin/imsg`, argv, { + encoding: "utf8", timeout, maxBuffer, + ...(opts.input === undefined ? {} : { input: opts.input }), }); + return { + status: res.status, + stdout: (res.stdout ?? "") as string, + stderr: (res.stderr ?? "") as string, + ...(res.error ? { error: res.error as Error } : {}), + }; +} + +/** One request over the unix socket, or null to mean "use the slow path". */ +function viaBridgeSocket( + argv: string[], + input: string | undefined, + timeout: number, + maxBuffer: number, +): { status: number | null; stdout: string; stderr: string } | null { + try { + if (!existsSync(BRIDGE_SOCK)) return null; + const req = JSON.stringify(input === undefined ? { argv } : { argv, stdin: input }); + // socat, not a bun/python client: spawning either costs 7-11 ms, which is + // most of what the channel just saved. A plain spawn is ~1 ms. + // No `encoding`: the default hands back Buffers, and the frame counts + // BYTES. Passing "buffer" throws ERR_UNKNOWN_ENCODING in Bun, the catch + // below swallowed it, and the fast path silently never ran — every call + // quietly took the slow one and the whole channel looked like a no-op. + const res = spawnSync("socat", ["-t", String(Math.ceil(timeout / 1000)), "-", `UNIX-CONNECT:${BRIDGE_SOCK}`], { + input: req + "\n", timeout, maxBuffer, + }); + if (res.error || res.status !== 0 || !res.stdout) return null; + const buf = res.stdout as Buffer; + const nl = buf.indexOf(0x0a); + if (nl < 0) return null; + const meta = JSON.parse(buf.subarray(0, nl).toString("utf8")) as { status: number; len: number }; + const body = buf.subarray(nl + 1); + // A short frame means the channel desynchronised: take the slow path + // rather than hand a caller half a JSON document. + if (typeof meta.len !== "number" || body.length < meta.len) return null; + const out = body.subarray(0, meta.len).toString("utf8"); + return meta.status === 0 + ? { status: 0, stdout: out, stderr: "" } + : { status: meta.status, stdout: "", stderr: out }; + } catch { + return null; + } +} + +export function fetchMessages(limit: number, runner = spawnSync): FetchResult { + const res = bridgeRun(["--json", "recent", String(limit)], runner); if (res.error) { // spawn itself failed: ~/bin/imsg missing (run blip-setup) or not executable @@ -1215,10 +1296,7 @@ export const CHAT_LIST_LIMIT = 300; * widget's memory. Previews are never persisted (no content on disk). */ export function fetchChats(runner = spawnSync): ChatInfo[] | null { - const res = runner(`${HOME}/bin/imsg`, ["--json", "chats", String(CHAT_LIST_LIMIT)], { - encoding: "utf8", - timeout: 20000, maxBuffer: 64 * 1024 * 1024, - }); + const res = bridgeRun(["--json", "chats", String(CHAT_LIST_LIMIT)], runner, { timeout: 20000 }); if (res.status !== 0) return null; try { const rows = JSON.parse(res.stdout as string); @@ -1381,7 +1459,7 @@ export function mergeChats( } export function fetchGroups(runner = spawnSync): Record | null { - const res = runner(`${HOME}/bin/imsg`, ["--json", "groups"], { encoding: "utf8", timeout: 15000, maxBuffer: 64 * 1024 * 1024 }); + const res = bridgeRun(["--json", "groups"], runner); if (res.status !== 0) return null; try { const rows = JSON.parse(res.stdout as string); diff --git a/scripts/blip-setup b/scripts/blip-setup index d81ad4e..dca453f 100755 --- a/scripts/blip-setup +++ b/scripts/blip-setup @@ -8,6 +8,7 @@ # 2. ~/.ssh/config ControlMaster block for the host (47 ms warm # round trips — Blip polls; this is required) # 3. ~/bin/{imsg,imsg-send,imsg-read,contacts} the bridge shim (existing files backed up; +# plus ~/bin/blip-bridged, the persistent-channel daemon the bar widget runs # claude-on-mac remote shims are left alone) # 4. Mac install copies bridge/mac to the Mac and runs # install.sh there (needs ssh access already) @@ -100,7 +101,17 @@ for t in imsg imsg-send imsg-read contacts; do fi install -m 0755 "$here/bridge/linux/blip-shim" "$target" done -echo "✓ shims installed: ~/bin/imsg ~/bin/imsg-send ~/bin/imsg-read ~/bin/contacts" +install -m 0755 "$here/bridge/linux/blip-bridged" "$HOME/bin/blip-bridged" +echo "✓ shims installed: ~/bin/imsg ~/bin/imsg-send ~/bin/imsg-read ~/bin/contacts ~/bin/blip-bridged" +# blip-bridged holds `imsg serve` channels open so a query costs the query +# instead of ~90 ms of ssh + two Python starts + opening a 218 MB chat.db. +# The bar widget starts it. It needs socat to be reachable from the plugin; +# without socat Blip simply keeps using the one-shot path, so this is a note, +# never a failure. +if ! command -v socat >/dev/null 2>&1; then + echo "• socat not found — Blip will work, but every query pays ~90 ms of startup." + echo " Install it for the fast path: pacman -S socat" +fi # 4. Mac install (needs ssh to already work — key-based) if ssh -n -o ConnectTimeout=6 -o BatchMode=yes -- "$host" true 2>/dev/null; then diff --git a/thread.test.ts b/thread.test.ts index c15165b..6fd5361 100644 --- a/thread.test.ts +++ b/thread.test.ts @@ -711,7 +711,10 @@ describe("pending sends (the bubble drawn before the Mac writes the row)", () => test("loadThread --pending-stdin runs end to end through the CLI", async () => { const proc = Bun.spawn(["bun", "thread.ts", "+15550100001", "5", "--pending-stdin"], { stdin: new TextEncoder().encode(JSON.stringify([{ chat: "+15550100001", text: "hi", ts: localToday() + " 00:00:00" }])), - env: { ...process.env, HOME: "/nonexistent-blip-home" }, // no ~/bin/imsg: offline, pending echoed back + // Offline means offline: no ~/bin/imsg (HOME) AND no accelerator socket + // (XDG_RUNTIME_DIR). blip-bridged's socket is found under the latter, so + // clearing only HOME left a real, working bridge underneath the test. + env: { ...process.env, HOME: "/nonexistent-blip-home", XDG_RUNTIME_DIR: "/nonexistent-blip-runtime" }, stdout: "pipe", }); const out = JSON.parse(await new Response(proc.stdout).text()); diff --git a/thread.ts b/thread.ts index 4d1ed8a..8436e95 100644 --- a/thread.ts +++ b/thread.ts @@ -16,6 +16,7 @@ import { readFileSync } from "node:fs"; import { chatKey, dedupeSelfEcho, + bridgeRun, isGroupChat, loadState, normalizeMsgStamps, @@ -538,10 +539,7 @@ export function loadThread( // window (war room #14). A DM's chat_identifier IS the handle. const group = isGroupChat(chat); const args = ["--json", "--rich", "thread", "--chat", chat, String(limit)]; - const res = runner(`${HOME}/bin/imsg`, args, { - encoding: "utf8", - timeout: 15000, maxBuffer: 64 * 1024 * 1024, - }); + const res = bridgeRun(args, runner); if (res.status === 69 || res.status === 255) { return { ok: false, online: false, error: "Mac unreachable", bubbles: [] }; diff --git a/tier2.test.ts b/tier2.test.ts index d178430..431dc3c 100644 --- a/tier2.test.ts +++ b/tier2.test.ts @@ -901,3 +901,43 @@ describe("cache file names (Astra B#1)", () => { expect(cacheFileName("9", "doc.pdf", "application/pdf")).toBe("9-orig-doc.pdf"); }); }); + +describe("bridgeRun: the fast path is optional and invisible", () => { + const { bridgeRun } = require("./collector") as typeof import("./collector"); + + test("a caller with its own runner never touches the socket", () => { + // This is what keeps every other test in this suite honest: they inject a + // runner and assert on real argv, so the accelerator must stay out of the + // way entirely when one is supplied. + const seen: { cmd: string; args: string[] }[] = []; + const runner = ((cmd: string, args: string[]) => { + seen.push({ cmd, args }); + return { status: 0, stdout: "[]", stderr: "" }; + }) as never; + const res = bridgeRun(["--json", "recent", "5"], runner); + expect(seen).toHaveLength(1); + expect(seen[0]!.cmd).toContain("/bin/imsg"); + expect(seen[0]!.args).toEqual(["--json", "recent", "5"]); + expect(res.status).toBe(0); + expect(res.stdout).toBe("[]"); + }); + + test("a stdin payload is passed through, never folded into argv", () => { + // Message text must not ride argv on either machine (audit #4). + const seen: { args: string[]; opts: { input?: string } }[] = []; + const runner = ((_c: string, args: string[], opts: { input?: string }) => { + seen.push({ args, opts }); + return { status: 0, stdout: "[]", stderr: "" }; + }) as never; + bridgeRun(["--json", "search", "--stdin"], runner, { input: "secret words" }); + expect(seen[0]!.opts.input).toBe("secret words"); + expect(seen[0]!.args.join(" ")).not.toContain("secret"); + }); + + test("a non-zero exit is reported, not swallowed", () => { + const runner = (() => ({ status: 3, stdout: "", stderr: "imsg: boom" })) as never; + const res = bridgeRun(["--json", "recent", "5"], runner); + expect(res.status).toBe(3); + expect(res.stderr).toContain("boom"); + }); +}); diff --git a/ui.test.ts b/ui.test.ts index 00f1939..832658f 100644 --- a/ui.test.ts +++ b/ui.test.ts @@ -700,3 +700,13 @@ describe("a multi-part send is pinned to the thread it started in", () => { expect(pump).not.toContain("root.active.service"); }); }); + +describe("the accelerator channel is an optimisation, never a dependency", () => { + test("the leader bar supervises blip-bridged, and only the leader", () => { + // Two bars would hold two pairs of Mac processes. + expect(widget).toContain('command: [root.home + "/bin/blip-bridged"]'); + const proc = widget.slice(widget.indexOf("id: bridgeProc"), widget.indexOf("id: bridgeRestart")); + expect(proc).toContain("running: root.leader"); + expect(proc).toContain("onExited: bridgeRestart.restart()"); + }); +}); From 934ee1fb6aa66b6c888acd5d5f3a929260e57327 Mon Sep 17 00:00:00 2001 From: Zach Wilke Date: Mon, 7 Sep 2026 20:05:46 -0500 Subject: [PATCH 6/7] perf: a name lookup that misses no longer scans the whole address book MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With startup amortised by the serve channel, what was left in `chats` was compute: 179 ms of SQL and 104 ms resolving names. The name half was algorithmic. A handle with no exact last-ten key fell back to comparing it against every saved number, so 298 conversations cost 60,112 _same_number calls — 46 ms a request, and it grows with the address book. Cards are bucketed by their last seven digits now. That is sound rather than convenient: _same_number only matches when one national number is a SUFFIX of the other with a floor of seven digits, so the last seven always agree, and a card that does not share them cannot match at any calling code. The stripped trunk-zero variant is bucketed too, since the rule tries the card both ways. Verified against the old full scan over all 750 distinct handles in a real chat.db: zero mismatches, largest bucket 2 cards instead of 442. That comparison is the check to repeat if this is ever touched — CLAUDE.md now says so, because these matching rules have been got wrong twice before. name_for also memoises per handle, and group clusters and pins are memoised across requests now that a process outlives one. They key on what they actually derive from — the chat table's (count, max ROWID) and the pinning plist's mtime — never on chat.db's mtime, which changes on every message and would have cached nothing during exactly the busy minute that matters. The Contacts index gains an invalidation for the same reason in reverse: a one-shot run rebuilt it every time, while the serve channel would otherwise answer with the names it started with until it died. cmd_chats 292 → 215 ms; a rich thread load 72 → 52 ms (the cluster cache). End to end against where this started: opening a conversation 223 → 83 ms, a poll 160 → 39 ms, a deep poll 707 → 342 ms. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015xxS9WCT7ifHhbDjYyGnHS --- CHANGELOG.md | 15 +++++ CLAUDE.md | 16 +++++ bridge/mac/imsg | 127 ++++++++++++++++++++++++++++++++++++++- bridge/mac/test_serve.py | 32 ++++++++++ 4 files changed, 187 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b95e2c8..d068606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## Unreleased +- **The conversation list stopped scanning the address book.** With startup + amortised, what was left was compute — and `chats` was spending 104 ms of + every request resolving names, because a handle with no exact last-ten key + fell back to comparing it against all 442 saved numbers. 298 conversations + meant 60,112 comparisons. They are bucketed by their last seven digits now, + which is sound rather than lucky: the matching rule only ever matches when + one number is a suffix of the other with a seven-digit floor, so the last + seven always agree. Checked against the old full scan over all 750 distinct + handles in a real chat.db: identical answers, zero mismatches. + Group clusters and pins are memoised too, against the chat table's shape and + the pinning plist's mtime rather than chat.db's — which changes on every + message and would have cached nothing during the busy minute that matters. + Opening a conversation is 83 ms now (223 ms before any of this), a poll + 39 ms (160 ms), a deep poll 342 ms (707 ms). + - **A persistent channel to the Mac.** Even with the probe gone, every query still paid ~90 ms before it read a row: ssh, `blip-dispatch`'s Python start, `imsg`'s own, and opening a 218 MB chat.db — for SQL that takes about a diff --git a/CLAUDE.md b/CLAUDE.md index e5d89fa..e88c1e1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -348,6 +348,22 @@ what it is handed. Keep it that way. **"Offline" in a test means clearing HOME *and* XDG_RUNTIME_DIR**, or the socket answers underneath the test. +- **A name lookup that misses must not scan the address book.** `_same_number` + only matches when one national number is a SUFFIX of the other with a floor + of seven digits, so the last seven digits ALWAYS agree — which is what makes + `_PHONE_SUFFIX` (last-seven → cards) sound. Before it, every handle without + an exact last-ten key scanned all 442 cards: 298 conversations cost 60,112 + `_same_number` calls, 46 ms of a 292 ms `chats`. Verified identical against + the full scan over all 750 distinct handles in a real chat.db — zero + mismatches — and that check is the one to repeat if this is ever touched, + because the matching rules have been got wrong twice before. + **Caches invalidate on what they DERIVE from, never on chat.db's mtime**, + which changes on every message and would cache nothing during exactly the + busy minute that matters: group clusters key on the chat table's + (count, max ROWID), pins on the pinning plist's mtime, and the Contacts + index on the address books' — the last one exists because the serve channel + outlives an edit in Contacts.app, where a one-shot run always rebuilt. + ## Working on it ``` diff --git a/bridge/mac/imsg b/bridge/mac/imsg index ad58363..265dab6 100755 --- a/bridge/mac/imsg +++ b/bridge/mac/imsg @@ -268,6 +268,14 @@ def _vote(local: dict, key, name: str) -> None: _EXACT_SEEN: set = set() # every exact key ANY source carried, ambiguous or not _NAME_INDEX: dict[str, str] | None = None # exact keys: last-ten digits, lowercased email _PHONE_INDEX: dict[Phone, str] | None = None # every saved number, parsed, for the region-aware fallback +# The same cards bucketed by their last seven digits. _same_number only ever +# matches when one national number is a SUFFIX of the other with a floor of +# seven digits — so the last seven always agree, and a candidate that does not +# share them cannot match. Without this, every handle that missed the exact +# key scanned the whole address book: 298 conversations cost 60,112 +# _same_number calls (46 ms of a 292 ms `chats`). +_PHONE_SUFFIX: dict[str, list[tuple[Phone, str]]] | None = None +_NAME_CACHE: dict[str, str | None] = {} # handle -> resolved name, per process def _is_dm_id(s: str | None) -> bool: @@ -305,9 +313,14 @@ def name_for(handle: str | None) -> str | None: """Resolve a Messages handle (phone or email) to a Contacts display name. Lazily builds an in-memory index from all AddressBook source DBs on first call. Returns None if no contact match.""" - global _NAME_INDEX, _PHONE_INDEX + global _NAME_INDEX, _PHONE_INDEX, _PHONE_SUFFIX if not handle: return None + # Same handle, same answer: `recent 150` asks for a couple of dozen people + # over and over, and on the serve channel the index outlives the request + # anyway. Reset with _forget_contacts() when the address book changes. + if handle in _NAME_CACHE: + return _NAME_CACHE[handle] if _NAME_INDEX is None: # One person usually exists in SEVERAL sources under slightly different # spellings ("Rob" vs "Robert"). That is not ambiguity. Ambiguity is two @@ -366,6 +379,20 @@ def name_for(handle: str | None) -> str | None: resolved = {k: min(v.items(), key=lambda kv: (kv[1][0], -kv[1][1]))[0] for k, v in votes.items()} _NAME_INDEX = {k: n for k, n in resolved.items() if isinstance(k, str)} _PHONE_INDEX = {k: n for k, n in resolved.items() if isinstance(k, tuple)} + _PHONE_SUFFIX = {} + for card, n in _PHONE_INDEX.items(): + # Bucket on the stripped variant too: _same_number tries the card + # both with and without a saved trunk zero. + for nsn in {card[1], card[1].removeprefix("0")}: + if len(nsn) >= _LOCAL_NUMBER_DIGITS: + _PHONE_SUFFIX.setdefault(nsn[-_LOCAL_NUMBER_DIGITS:], []).append((card, n)) + answer = _resolve_name(handle) + _NAME_CACHE[handle] = answer + return answer + + +def _resolve_name(handle: str) -> str | None: + """name_for() minus the memo — the index is built by the time we get here.""" if "@" in handle: return _NAME_INDEX.get(handle.strip().lower()) or None hit = _NAME_INDEX.get(_normalize_phone(handle)) @@ -375,12 +402,31 @@ def name_for(handle: str | None) -> str | None: # suffix match name a stranger over it (Astra C#2). if _normalize_phone(handle) in _EXACT_SEEN: return None - # No exact key: a unique region-aware match names it; two different names do not. + # No exact key: a unique region-aware match names it; two different names do + # not. Only cards sharing the last seven digits can match at all, so ask the + # bucket rather than the whole book — same answers, one lookup instead of a + # scan per unknown number. parsed = _parse_phone(handle) - near = {n for card, n in _PHONE_INDEX.items() if parsed and _same_number(parsed, card)} + if not parsed: + return None + nsn = parsed[1] + if len(nsn) < _LOCAL_NUMBER_DIGITS: + return None + bucket = (_PHONE_SUFFIX or {}).get(nsn[-_LOCAL_NUMBER_DIGITS:], ()) + near = {n for card, n in bucket if _same_number(parsed, card)} return near.pop() if len(near) == 1 else None +def _forget_contacts() -> None: + """Drop the Contacts indexes so the next lookup rebuilds them. One-shot + runs never needed this — the process ended — but the serve channel + outlives an edit in Contacts.app.""" + global _NAME_INDEX, _PHONE_INDEX, _PHONE_SUFFIX + _NAME_INDEX = _PHONE_INDEX = _PHONE_SUFFIX = None + _EXACT_SEEN.clear() + _NAME_CACHE.clear() + + def label(handle: str | None, is_from_me: int, with_names: bool) -> str: """Pretty label for the 'who' column.""" if is_from_me: @@ -559,6 +605,18 @@ def _app_card_text(row) -> str | None: def pinned_chat_identifiers() -> list[set[str]]: + """Memoised _pinned_chat_identifiers_uncached: the plist only changes when + you pin or unpin, and parsing it drives pin_order_for (~22 ms a request).""" + global _PINS_CACHE + gen = _pins_generation() + if _PINS_CACHE is not None and _PINS_CACHE[0] == gen: + return _PINS_CACHE[1] + out = _pinned_chat_identifiers_uncached() + _PINS_CACHE = (gen, out) + return out + + +def _pinned_chat_identifiers_uncached() -> list[set[str]]: """Ordered pins from Messages' pinning preferences, one alias set each. Observed layout (macOS 15/26, undocumented): ``pD.pP`` is the ORDERED list @@ -1278,7 +1336,61 @@ def _sniff_image(head: bytes) -> str: return "" +# Recomputed per request one-shot, which cost nothing because the process +# ended anyway. On the serve channel a request is not the end of the world, so +# both memoise against the thing they actually derive from — never against +# chat.db's mtime, which changes on every message and would cache nothing +# during exactly the busy minute you care about. +_CLUSTER_CACHE: tuple | None = None +_PINS_CACHE: tuple | None = None + + +def _chat_table_generation(con) -> tuple: + """Cheap fingerprint of the chat table. Group clusters change only when a + chat row is added or re-keyed, never when a message lands.""" + row = con.execute("SELECT COUNT(*), COALESCE(MAX(ROWID), 0) FROM chat").fetchone() + return (row[0], row[1]) + + +def _pins_generation() -> tuple: + """Pins live in a plist Messages rewrites when you pin or unpin.""" + out = [] + for path in PINNING_PLIST_PATHS: + try: + st = os.stat(path) + out.append((st.st_mtime_ns, st.st_size)) + except OSError: + out.append((0, 0)) + return tuple(out) + + +def _contacts_generation() -> tuple: + """The address books, so an edit in Contacts.app is seen by a channel that + outlives it. One-shot runs rebuilt the index every time and never needed + this; serve would have gone on answering with the names it started with.""" + out = [] + for path in _ab_sources(): + try: + st = os.stat(path) + out.append((st.st_mtime_ns, st.st_size)) + except OSError: + out.append((0, 0)) + return tuple(out) + + def _group_cluster_map(con) -> dict: + """Memoised _group_cluster_map_uncached: ~21 ms a request, and the answer + only moves when a chat row appears or is re-keyed.""" + global _CLUSTER_CACHE + gen = _chat_table_generation(con) + if _CLUSTER_CACHE is not None and _CLUSTER_CACHE[0] == gen: + return _CLUSTER_CACHE[1] + out = _group_cluster_map_uncached(con) + _CLUSTER_CACHE = (gen, out) + return out + + +def _group_cluster_map_uncached(con) -> dict: """chat_identifier -> canonical chat_identifier, for group rows that are the SAME Messages conversation split across several chat rows. @@ -1963,10 +2075,19 @@ def cmd_serve(con, args) -> None: out.write(b"ready\n") out.flush() + contacts_gen = _contacts_generation() + for line in stdin: line = line.strip() if not line: continue + # A one-shot run rebuilt the Contacts index every time; this channel + # would have answered with the names it started with until it died. + # Four stats, so it costs nothing to check per request. + gen = _contacts_generation() + if gen != contacts_gen: + contacts_gen = gen + _forget_contacts() try: req = json.loads(line) argv = req["argv"] diff --git a/bridge/mac/test_serve.py b/bridge/mac/test_serve.py index 70a6687..e11ec0d 100644 --- a/bridge/mac/test_serve.py +++ b/bridge/mac/test_serve.py @@ -56,5 +56,37 @@ def test_serve_is_a_real_subcommand(self): self.assertEqual(ns.func, imsg.cmd_serve) +class ContactIndex(unittest.TestCase): + """The suffix bucket must answer exactly as the old full scan did.""" + + def test_a_match_always_shares_the_last_seven_digits(self): + # This is what makes bucketing sound: _same_number only matches when + # one national number is a SUFFIX of the other, with a floor of seven + # digits — so the last seven always agree. + n = imsg._LOCAL_NUMBER_DIGITS + self.assertEqual(n, 7) + pairs = [ + (("1", "2145550123"), ("1", "2145550123")), + (("47", "12345678"), ("47", "12345678")), + (("44", "7700900123"), ("44", "07700900123")), # saved trunk zero + ] + for handle, card in pairs: + if imsg._same_number(handle, card): + stripped = card[1].removeprefix("0") + self.assertTrue( + handle[1][-n:] in (card[1][-n:], stripped[-n:]), + f"{handle} vs {card} matched without sharing the last {n} digits", + ) + + def test_forgetting_contacts_clears_every_index(self): + # The serve channel outlives an edit in Contacts.app, so it re-reads. + imsg._NAME_CACHE["x"] = "stale" + imsg._forget_contacts() + self.assertEqual(imsg._NAME_CACHE, {}) + self.assertIsNone(imsg._NAME_INDEX) + self.assertIsNone(imsg._PHONE_SUFFIX) + self.assertEqual(imsg._EXACT_SEEN, set()) + + if __name__ == "__main__": unittest.main(verbosity=2) From 586a4def28280148f5c195f83cc62ede2d4b63ca Mon Sep 17 00:00:00 2001 From: Zach Wilke Date: Mon, 7 Sep 2026 20:08:14 -0500 Subject: [PATCH 7/7] docs: "tapbacks out is impossible" is half wrong on macOS 26 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verdict predates the OS. Enumerating Messages' menu bar over System Events on the gateway Mac turns up real menu items for three of the four things that note rules out: Edit ▸ Tapback Message…, Edit ▸ Reply to Message…, Edit ▸ Edit Last Message…, plus Send Later…. A menu item is scriptable, which is precisely how imsg-read overturned the old "marking read on the Mac is impossible" note. Typing indicators have no menu item and stay out. Nothing is claimed to work yet, and the note says so. All four read enabled=false from the background, which by our own read-push finding is evidence of nothing — AppKit validates menus against the ACTIVE app's responder chain — so feasibility has to be retested with Messages frontmost. The harder obstacles are recorded with it: the item acts on the SELECTED message and selecting an arbitrary bubble from Linux is the unsolved part, the "…" implies a picker to navigate for the emoji, Messages must come forward so it steals focus, and a group still cannot be addressed at all. Recording this because a stale "not possible" is the kind of note that stops someone looking for years. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015xxS9WCT7ifHhbDjYyGnHS --- CLAUDE.md | 17 ++++++++++++++++- ROADMAP.md | 11 +++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index e88c1e1..753b02b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -392,7 +392,22 @@ to whatever has focus otherwise. ## Things that are not possible -- Tapbacks, edits, typing indicators out. Needs SIP-off code injection; rejected. +- ~~Tapbacks, edits, typing indicators out — needs SIP-off code injection.~~ + **Half wrong; do not quote this as settled** (2026-09-07). macOS 26 Messages + has real MENU ITEMS for three of them — `Edit ▸ Tapback Message…`, + `Edit ▸ Reply to Message…`, `Edit ▸ Edit Last Message…`, plus `Send Later…` + — enumerated over System Events on the gateway Mac. A menu item is + scriptable, which is exactly how `imsg-read` overturned the old "marking + read is impossible" note. Typing indicators have no menu item and stay out. + What is NOT yet proved, and what anyone picking this up must establish + first: all four read `enabled=false` from the background, which per the + read-push note is evidence of NOTHING (AppKit validates menus against the + ACTIVE app's responder chain) — so feasibility has to be tested with + Messages frontmost. Then the real obstacles: the item acts on the SELECTED + message, and selecting an arbitrary bubble from Linux is the unsolved part; + the "…" suggests a picker that needs further navigation for the emoji; + Messages must come forward, so it steals focus the way `--chat` read-push + does; and a GROUP still cannot be addressed at all (next bullet). - Selecting a GROUP on the Mac from Linux. `imessage://` addresses a handle; a group's `chat` id has no URL form. So per-conversation read-push is DMs only; groups clear through `--all`. diff --git a/ROADMAP.md b/ROADMAP.md index 609ab18..41b060a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -176,6 +176,17 @@ panel (3 lenses × finding, against current main) confirmed 11 closed and - [x] One BarWidget per monitor → leader election (2.2.0; follower bars show the badge from state.json and forward clicks over IPC). - [x] SMS/RCS threads send on their own service (2.2.0). +- [ ] **Sending tapbacks may be possible after all** (2026-09-07). The + "needs SIP-off injection" verdict predates macOS 26: Messages now has + `Edit ▸ Tapback Message…` as a real menu item (also `Reply to Message…` + and `Edit Last Message…`), enumerated over System Events. That is the same + shape `imsg-read` used to overturn "marking read is impossible". Unproven: + the items read `enabled=false` from the background, which proves nothing + (responder-chain validation — test with Messages frontmost); the item acts + on the SELECTED message and selecting an arbitrary bubble from Linux is the + open problem; the picker needs navigating for the emoji; and it steals + focus. Groups remain unaddressable. + - [ ] **3–4 digit short codes classify as groups** — `isGroupChat()` treats a digits-only id shorter than 5 as "not a phone", so a carrier-style sender opens read-only with "group id unknown". Widen to `{3,15}` (E.164 max is