From 216d255eccb96b2d5c5959247babd457fd880b20 Mon Sep 17 00:00:00 2001 From: Lukas Kawerau <2502467+lukaskawerau@users.noreply.github.com> Date: Thu, 25 Jun 2026 10:20:58 +0200 Subject: [PATCH 1/4] fix: persist bird quoted tweets --- CHANGELOG.md | 1 + src/lib/bird.test.ts | 40 +++++++++++++++++++++ src/lib/bird.ts | 34 +++++++++++++++--- src/lib/profile-analysis.ts | 26 +++++++++----- src/lib/timeline-live.test.ts | 65 +++++++++++++++++++++++++++++++++++ src/lib/timeline-live.ts | 5 +++ src/lib/tweet-repository.ts | 39 +++++++++++++++------ src/lib/types.ts | 2 ++ 8 files changed, 188 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e49a32..cbbe9a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Fixed +- Persist quoted tweet payloads returned by Bird-backed live syncs so quote cards render without a separate hydrate. - Show full tweet text in Today citation popovers instead of truncating long posts after six lines. - Show inline tweet images in Today citation popovers instead of leaving media-only `t.co` links in the preview text. diff --git a/src/lib/bird.test.ts b/src/lib/bird.test.ts index dfb765a..eb17bb4 100644 --- a/src/lib/bird.test.ts +++ b/src/lib/bird.test.ts @@ -947,6 +947,46 @@ describe("bird transport wrapper", () => { quotedTweet: { id: "quoted" }, }), ).toEqual([{ type: "quoted", id: "quoted" }]); + expect( + __test__.normalizeBirdTweets([ + { + id: "1", + text: "quoting", + createdAt: "2026-05-02T00:00:00.000Z", + authorId: "42", + author: { username: "sam", name: "Sam" }, + quotedTweet: { + id: "quoted", + text: "quoted body", + createdAt: "2026-05-01T00:00:00.000Z", + authorId: "43", + author: { username: "alex", name: "Alex" }, + }, + }, + ]), + ).toEqual( + expect.objectContaining({ + data: [ + expect.objectContaining({ + id: "1", + referenced_tweets: [{ type: "quoted", id: "quoted" }], + }), + ], + includes: { + users: [ + { id: "42", username: "sam", name: "Sam" }, + { id: "43", username: "alex", name: "Alex" }, + ], + tweets: [ + expect.objectContaining({ + id: "quoted", + author_id: "43", + text: "quoted body", + }), + ], + }, + }), + ); expect( __test__.normalizeBirdTweets([ { diff --git a/src/lib/bird.ts b/src/lib/bird.ts index 87eb4de..5803887 100644 --- a/src/lib/bird.ts +++ b/src/lib/bird.ts @@ -46,7 +46,7 @@ interface BirdTweetItem { inReplyToStatusId?: string | null; quotedStatusId?: string | null; retweetedStatusId?: string | null; - quotedTweet?: { id?: string | null } | null; + quotedTweet?: Partial | null; retweetedTweet?: { id?: string | null } | null; author?: BirdTweetAuthor; authorId?: string; @@ -444,9 +444,19 @@ function toReferencedTweets(item: BirdTweetItem) { return references.length > 0 ? references : undefined; } +function isHydratedBirdTweetItem(value: unknown): value is BirdTweetItem { + const item = value as BirdTweetItem | undefined; + return ( + typeof item?.id === "string" && + typeof item.text === "string" && + typeof item.createdAt === "string" + ); +} + function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse { const users = new Map(); - const data = items.map((item): XurlMentionData => { + const includedTweets = new Map(); + const normalizeItem = (item: BirdTweetItem): XurlMentionData => { const authorId = String( item.authorId ?? item.author?.username ?? "unknown", ); @@ -458,6 +468,13 @@ function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse { }); } + if (isHydratedBirdTweetItem(item.quotedTweet)) { + const quotedTweet = normalizeItem(item.quotedTweet); + if (quotedTweet.id !== item.id) { + includedTweets.set(quotedTweet.id, quotedTweet); + } + } + return { id: item.id, author_id: authorId, @@ -473,12 +490,19 @@ function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse { }, edit_history_tweet_ids: [item.id], }; - }); + }; + const data = items.map((item): XurlMentionData => normalizeItem(item)); + const includes: XurlMentionsResponse["includes"] = {}; + if (users.size > 0) { + includes.users = Array.from(users.values()); + } + if (includedTweets.size > 0) { + includes.tweets = Array.from(includedTweets.values()); + } return { data, - includes: - users.size > 0 ? { users: Array.from(users.values()) } : undefined, + includes: users.size > 0 || includedTweets.size > 0 ? includes : undefined, meta: { result_count: data.length, page_count: 1, diff --git a/src/lib/profile-analysis.ts b/src/lib/profile-analysis.ts index 8c62e2d..e2912dc 100644 --- a/src/lib/profile-analysis.ts +++ b/src/lib/profile-analysis.ts @@ -365,7 +365,15 @@ function mergeXurlTweetsIntoLocalStore( const existingTweet = db.prepare("select text from tweets where id = ?"); const seenAt = new Date().toISOString(); db.transaction(() => { + const primaryTweetIds = new Set(payload.data.map((tweet) => tweet.id)); + const tweetsById = new Map(); + for (const tweet of payload.includes?.tweets ?? []) { + tweetsById.set(tweet.id, tweet); + } for (const tweet of payload.data) { + tweetsById.set(tweet.id, tweet); + } + for (const tweet of tweetsById.values()) { const authorId = tweet.author_id; if (!authorId) continue; const author = usersById.get(authorId); @@ -396,14 +404,16 @@ function mergeXurlTweetsIntoLocalStore( buildMediaJsonFromIncludes(tweet, payload.includes?.media), quotedTweetId, ); - upsertTweetAccountEdge(db, { - accountId, - tweetId: tweet.id, - kind: edgeKind, - source, - seenAt, - rawJson: JSON.stringify(tweet), - }); + if (primaryTweetIds.has(tweet.id)) { + upsertTweetAccountEdge(db, { + accountId, + tweetId: tweet.id, + kind: edgeKind, + source, + seenAt, + rawJson: JSON.stringify(tweet), + }); + } refreshTweetFts(db, tweet.id, tweet.text, previousText); } })(); diff --git a/src/lib/timeline-live.test.ts b/src/lib/timeline-live.test.ts index 089c6ac..f8e4111 100644 --- a/src/lib/timeline-live.test.ts +++ b/src/lib/timeline-live.test.ts @@ -153,6 +153,71 @@ describe("live home timeline sync", () => { ]); }); + it("persists included quoted tweets without home timeline edges", async () => { + makeTempHome(); + const db = getNativeDb(); + listHomeTimelineViaBirdMock.mockResolvedValueOnce({ + data: [ + { + id: "tweet_quote_ref", + author_id: "42", + text: "read this quote", + created_at: "2026-04-26T13:43:34.000Z", + referenced_tweets: [{ type: "quoted", id: "tweet_quoted" }], + public_metrics: { like_count: 12 }, + }, + ], + includes: { + users: [ + { id: "42", username: "sam", name: "Sam" }, + { id: "43", username: "alex", name: "Alex" }, + ], + tweets: [ + { + id: "tweet_quoted", + author_id: "43", + text: "the quoted body", + created_at: "2026-04-25T13:43:34.000Z", + public_metrics: { like_count: 5 }, + }, + ], + }, + meta: { result_count: 1 }, + }); + const { syncHomeTimeline } = await import("./timeline-live"); + + await syncHomeTimeline({ + account: "acct_primary", + limit: 5, + refresh: true, + }); + + expect( + db.prepare("select text from tweets where id = ?").get("tweet_quoted"), + ).toEqual({ text: "the quoted body" }); + expect( + db + .prepare("select tweet_id from tweet_account_edges where tweet_id = ?") + .get("tweet_quoted"), + ).toBeUndefined(); + expect( + listTimelineItems({ + resource: "home", + account: "acct_primary", + search: "quote", + limit: 5, + }), + ).toEqual([ + expect.objectContaining({ + id: "tweet_quote_ref", + quotedTweet: expect.objectContaining({ + id: "tweet_quoted", + text: "the quoted body", + }), + }), + ]); + }); + it("preserves existing media_json when home payload omits media details", async () => { makeTempHome(); const db = getNativeDb(); diff --git a/src/lib/timeline-live.ts b/src/lib/timeline-live.ts index f4be4bd..4778c89 100644 --- a/src/lib/timeline-live.ts +++ b/src/lib/timeline-live.ts @@ -91,6 +91,7 @@ function mergeTimelinePayloads( ) { const data: XurlMentionsResponse["data"] = []; const usersById = new Map(); + const tweetsById = new Map(); const mediaByKey = new Map(); let meta: XurlMentionsResponse["meta"] | undefined; @@ -104,6 +105,9 @@ function mergeTimelinePayloads( for (const user of payload.includes?.users ?? []) { usersById.set(user.id, user); } + for (const tweet of payload.includes?.tweets ?? []) { + tweetsById.set(tweet.id, tweet); + } for (const media of payload.includes?.media ?? []) { mediaByKey.set(media.media_key, media); } @@ -114,6 +118,7 @@ function mergeTimelinePayloads( data, includes: { users: [...usersById.values()], + tweets: [...tweetsById.values()], media: [...mediaByKey.values()], }, meta, diff --git a/src/lib/tweet-repository.ts b/src/lib/tweet-repository.ts index 3e929cc..e743fdd 100644 --- a/src/lib/tweet-repository.ts +++ b/src/lib/tweet-repository.ts @@ -23,6 +23,17 @@ function getReferencedTweetId(tweet: XurlMentionData, type: string) { ); } +function toCanonicalTweets(payload: XurlMentionsResponse) { + const tweetsById = new Map(); + for (const tweet of payload.includes?.tweets ?? []) { + tweetsById.set(tweet.id, tweet); + } + for (const tweet of payload.data) { + tweetsById.set(tweet.id, tweet); + } + return tweetsById.values(); +} + export function replaceTweetFts(db: Database, tweetId: string, text: string) { db.prepare("delete from tweets_fts where tweet_id = ?").run(tweetId); db.prepare("insert into tweets_fts (tweet_id, text) values (?, ?)").run( @@ -80,7 +91,8 @@ export function ingestTweetPayload( db.transaction(() => { const observedAt = new Date().toISOString(); - for (const tweet of payload.data) { + const primaryTweetIds = new Set(payload.data.map((tweet) => tweet.id)); + for (const tweet of toCanonicalTweets(payload)) { const author = usersById.get(tweet.author_id); const profile = author ? upsertProfileFromXUser(db, author) @@ -100,7 +112,8 @@ export function ingestTweetPayload( buildMediaJsonFromIncludes(tweet, payload.includes?.media), quotedTweetId, ); - if (edgeKind) { + const isPrimaryTweet = primaryTweetIds.has(tweet.id); + if (edgeKind && isPrimaryTweet) { upsertTweetAccountEdge(db, { accountId, tweetId: tweet.id, @@ -110,16 +123,20 @@ export function ingestTweetPayload( rawJson: JSON.stringify(tweet), }); } - upsertCollection?.run( - accountId, - tweet.id, - collectionKind, - source, - JSON.stringify(tweet), - observedAt, - ); + if (isPrimaryTweet) { + upsertCollection?.run( + accountId, + tweet.id, + collectionKind, + source, + JSON.stringify(tweet), + observedAt, + ); + } replaceTweetFts(db, tweet.id, tweet.text); - tweetIds.push(tweet.id); + if (isPrimaryTweet) { + tweetIds.push(tweet.id); + } } })(); diff --git a/src/lib/types.ts b/src/lib/types.ts index 90ecbbe..867c6a1 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -593,6 +593,7 @@ export interface XurlMentionsResponse { data: XurlMentionData[]; includes?: { users?: XurlMentionUser[]; + tweets?: XurlTweetData[]; media?: XurlMediaItem[]; }; meta?: Record; @@ -623,6 +624,7 @@ export interface XurlTweetsResponse { data: XurlTweetData[]; includes?: { users?: XurlMentionUser[]; + tweets?: XurlTweetData[]; media?: XurlMediaItem[]; }; meta?: Record; From f11c77e4f8dd0e09a325b6872741b378932b5cdd Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 1 Jul 2026 09:25:33 +0100 Subject: [PATCH 2/4] refactor: narrow quoted tweet persistence scope --- CHANGELOG.md | 2 +- src/lib/profile-analysis.ts | 26 ++++++++------------------ 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbbe9a7..02446c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Fixed -- Persist quoted tweet payloads returned by Bird-backed live syncs so quote cards render without a separate hydrate. +- Persist quoted tweet payloads returned by Bird-backed live syncs so quote cards render without a separate hydrate. (#76 - thanks @lukaskawerau) - Show full tweet text in Today citation popovers instead of truncating long posts after six lines. - Show inline tweet images in Today citation popovers instead of leaving media-only `t.co` links in the preview text. diff --git a/src/lib/profile-analysis.ts b/src/lib/profile-analysis.ts index e2912dc..8c62e2d 100644 --- a/src/lib/profile-analysis.ts +++ b/src/lib/profile-analysis.ts @@ -365,15 +365,7 @@ function mergeXurlTweetsIntoLocalStore( const existingTweet = db.prepare("select text from tweets where id = ?"); const seenAt = new Date().toISOString(); db.transaction(() => { - const primaryTweetIds = new Set(payload.data.map((tweet) => tweet.id)); - const tweetsById = new Map(); - for (const tweet of payload.includes?.tweets ?? []) { - tweetsById.set(tweet.id, tweet); - } for (const tweet of payload.data) { - tweetsById.set(tweet.id, tweet); - } - for (const tweet of tweetsById.values()) { const authorId = tweet.author_id; if (!authorId) continue; const author = usersById.get(authorId); @@ -404,16 +396,14 @@ function mergeXurlTweetsIntoLocalStore( buildMediaJsonFromIncludes(tweet, payload.includes?.media), quotedTweetId, ); - if (primaryTweetIds.has(tweet.id)) { - upsertTweetAccountEdge(db, { - accountId, - tweetId: tweet.id, - kind: edgeKind, - source, - seenAt, - rawJson: JSON.stringify(tweet), - }); - } + upsertTweetAccountEdge(db, { + accountId, + tweetId: tweet.id, + kind: edgeKind, + source, + seenAt, + rawJson: JSON.stringify(tweet), + }); refreshTweetFts(db, tweet.id, tweet.text, previousText); } })(); From d1b8f20ceaf505d86d2cf7e40e4b21bd477e4a58 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 1 Jul 2026 09:28:52 +0100 Subject: [PATCH 3/4] fix: preserve quote metrics during live ingest --- src/lib/timeline-live.test.ts | 47 +++++++++++++++++++++++++++++++++++ src/lib/tweet-repository.ts | 7 ++++-- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/lib/timeline-live.test.ts b/src/lib/timeline-live.test.ts index f8e4111..845b2b9 100644 --- a/src/lib/timeline-live.test.ts +++ b/src/lib/timeline-live.test.ts @@ -218,6 +218,53 @@ describe("live home timeline sync", () => { ]); }); + it("preserves stored metrics when an included quote omits them", async () => { + makeTempHome(); + const db = getNativeDb(); + db.prepare( + `insert into tweets ( + id, author_profile_id, text, created_at, is_replied, + like_count, media_count, entities_json, media_json + ) values (?, 'profile_user_42', ?, ?, 0, 42, 0, '{}', '[]')`, + ).run("tweet_quoted_existing", "stored quote", "2026-04-25T13:43:34.000Z"); + listHomeTimelineViaBirdMock.mockResolvedValueOnce({ + data: [ + { + id: "tweet_quote_ref_existing", + author_id: "42", + text: "read this existing quote", + created_at: "2026-04-26T13:43:34.000Z", + referenced_tweets: [{ type: "quoted", id: "tweet_quoted_existing" }], + }, + ], + includes: { + users: [{ id: "42", username: "sam", name: "Sam" }], + tweets: [ + { + id: "tweet_quoted_existing", + author_id: "42", + text: "updated quote body", + created_at: "2026-04-25T13:43:34.000Z", + }, + ], + }, + meta: { result_count: 1 }, + }); + const { syncHomeTimeline } = await import("./timeline-live"); + + await syncHomeTimeline({ + account: "acct_primary", + limit: 5, + refresh: true, + }); + + expect( + db + .prepare("select text, like_count from tweets where id = ?") + .get("tweet_quoted_existing"), + ).toEqual({ text: "updated quote body", like_count: 42 }); + }); + it("preserves existing media_json when home payload omits media details", async () => { makeTempHome(); const db = getNativeDb(); diff --git a/src/lib/tweet-repository.ts b/src/lib/tweet-repository.ts index e743fdd..d3a1957 100644 --- a/src/lib/tweet-repository.ts +++ b/src/lib/tweet-repository.ts @@ -67,7 +67,7 @@ export function ingestTweetPayload( created_at = excluded.created_at, is_replied = max(tweets.is_replied, excluded.is_replied), reply_to_id = coalesce(tweets.reply_to_id, excluded.reply_to_id), - like_count = excluded.like_count, + like_count = case when ? then tweets.like_count else excluded.like_count end, media_count = max(tweets.media_count, excluded.media_count), entities_json = excluded.entities_json, media_json = case @@ -93,6 +93,7 @@ export function ingestTweetPayload( const observedAt = new Date().toISOString(); const primaryTweetIds = new Set(payload.data.map((tweet) => tweet.id)); for (const tweet of toCanonicalTweets(payload)) { + const isPrimaryTweet = primaryTweetIds.has(tweet.id); const author = usersById.get(tweet.author_id); const profile = author ? upsertProfileFromXUser(db, author) @@ -111,8 +112,10 @@ export function ingestTweetPayload( JSON.stringify(tweetEntitiesFromXurl(tweet.entities)), buildMediaJsonFromIncludes(tweet, payload.includes?.media), quotedTweetId, + !isPrimaryTweet && tweet.public_metrics?.like_count === undefined + ? 1 + : 0, ); - const isPrimaryTweet = primaryTweetIds.has(tweet.id); if (edgeKind && isPrimaryTweet) { upsertTweetAccountEdge(db, { accountId, From bf88a1073090a6fa007d07a96b23b1aa6e12a98c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 1 Jul 2026 09:31:05 +0100 Subject: [PATCH 4/4] fix: retain omitted quote metrics --- src/lib/bird.test.ts | 1 + src/lib/bird.ts | 29 ++++++++++++++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/lib/bird.test.ts b/src/lib/bird.test.ts index eb17bb4..85c9a49 100644 --- a/src/lib/bird.test.ts +++ b/src/lib/bird.test.ts @@ -982,6 +982,7 @@ describe("bird transport wrapper", () => { id: "quoted", author_id: "43", text: "quoted body", + public_metrics: {}, }), ], }, diff --git a/src/lib/bird.ts b/src/lib/bird.ts index 5803887..d34043a 100644 --- a/src/lib/bird.ts +++ b/src/lib/bird.ts @@ -456,7 +456,10 @@ function isHydratedBirdTweetItem(value: unknown): value is BirdTweetItem { function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse { const users = new Map(); const includedTweets = new Map(); - const normalizeItem = (item: BirdTweetItem): XurlMentionData => { + const normalizeItem = ( + item: BirdTweetItem, + preserveMissingMetrics = false, + ): XurlMentionData => { const authorId = String( item.authorId ?? item.author?.username ?? "unknown", ); @@ -469,7 +472,7 @@ function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse { } if (isHydratedBirdTweetItem(item.quotedTweet)) { - const quotedTweet = normalizeItem(item.quotedTweet); + const quotedTweet = normalizeItem(item.quotedTweet, true); if (quotedTweet.id !== item.id) { includedTweets.set(quotedTweet.id, quotedTweet); } @@ -483,11 +486,23 @@ function normalizeBirdTweets(items: BirdTweetItem[]): XurlMentionsResponse { conversation_id: item.conversationId ?? item.id, entities: toTweetEntities(item), referenced_tweets: toReferencedTweets(item), - public_metrics: { - reply_count: Number(item.replyCount ?? 0), - retweet_count: Number(item.retweetCount ?? 0), - like_count: Number(item.likeCount ?? 0), - }, + public_metrics: preserveMissingMetrics + ? { + ...(item.replyCount === undefined + ? {} + : { reply_count: Number(item.replyCount) }), + ...(item.retweetCount === undefined + ? {} + : { retweet_count: Number(item.retweetCount) }), + ...(item.likeCount === undefined + ? {} + : { like_count: Number(item.likeCount) }), + } + : { + reply_count: Number(item.replyCount ?? 0), + retweet_count: Number(item.retweetCount ?? 0), + like_count: Number(item.likeCount ?? 0), + }, edit_history_tweet_ids: [item.id], }; };