diff --git a/bun.lock b/bun.lock index 6b22d90c..2c428e78 100644 --- a/bun.lock +++ b/bun.lock @@ -132,7 +132,7 @@ "version": "12.7.0", "dependencies": { "@grpc/grpc-js": "^1.14.4", - "@photon-ai/advanced-imessage": "^2.0.2", + "@photon-ai/advanced-imessage": "^2.1.0", "@photon-ai/otel": "^3.3.0", "lru-cache": "^11.0.0", "marked": "^18.0.5", @@ -459,7 +459,7 @@ "@parseaple/typedstream": ["@parseaple/typedstream@2.0.2", "", { "dependencies": { "@parseaple/bplist": "1.0.1" } }, "sha512-bzKNZjiBtZN2T2U7Nqq/hhv+EezkPnaWoB5WvPuBwNpQvOr3EI0XrDTkFnAzsA9DKdu57es/gcb1d2ctd1u8xw=="], - "@photon-ai/advanced-imessage": ["@photon-ai/advanced-imessage@2.0.2", "", { "dependencies": { "@bufbuild/protobuf": "^2.11.0" }, "peerDependencies": { "@grpc/grpc-js": "^1.12.6", "nice-grpc": "^2.1.11", "nice-grpc-common": "^2.0.2" }, "optionalPeers": ["@grpc/grpc-js", "nice-grpc", "nice-grpc-common"] }, "sha512-2BbK2mUXPivgaeiHpN8hwF6tKW0AIeEyNiIOOYXc7h42SiPsBmlMEuKHdmCGbI+8mVb090a16HZJLOySDcg7Ug=="], + "@photon-ai/advanced-imessage": ["@photon-ai/advanced-imessage@2.1.0", "", { "dependencies": { "@bufbuild/protobuf": "^2.11.0" }, "peerDependencies": { "@grpc/grpc-js": "^1.12.6", "nice-grpc": "^2.1.11", "nice-grpc-common": "^2.0.2" }, "optionalPeers": ["@grpc/grpc-js", "nice-grpc", "nice-grpc-common"] }, "sha512-KT/aCBWcq667alyKe8DclqDUa/0fUuW2ZYDCxfIKuQBm08U4JcNG56fCdNUd69MFkKkzZgW35j59EYsx5DnLcQ=="], "@photon-ai/imessage-kit": ["@photon-ai/imessage-kit@3.0.0", "", { "dependencies": { "@parseaple/typedstream": "^2.0.0" }, "optionalDependencies": { "better-sqlite3": "^12.5.0" } }, "sha512-GGDETlRbrPXP5eaRvswaead7MCBGQkm8f7d3aVbx8+rXhB4GEyPiw7Q1/yz5ix53rmxqKRevWywOfG7n+kZDBw=="], diff --git a/packages/core/src/authoring.ts b/packages/core/src/authoring.ts index 3803fd63..b22104f0 100644 --- a/packages/core/src/authoring.ts +++ b/packages/core/src/authoring.ts @@ -36,6 +36,7 @@ export { setLogLevel, } from "@photon-ai/otel"; // Content factories, schemas, and the inbound-record type (from `content/`). +export { asResolvedApp } from "./content/app"; export { asAttachment } from "./content/attachment"; export { avatarSchema } from "./content/avatar"; export { asContact } from "./content/contact"; diff --git a/packages/core/src/content/app.ts b/packages/core/src/content/app.ts index 999a29a6..55c63ad9 100644 --- a/packages/core/src/content/app.ts +++ b/packages/core/src/content/app.ts @@ -16,17 +16,18 @@ import type { ContentBuilder } from "./types"; * `image` and `imageTitle` must be set together; `imageSubtitle` requires * `image`. */ -export const appLayoutSchema = z - .object({ - caption: z.string().nonempty().optional(), - subcaption: z.string().nonempty().optional(), - trailingCaption: z.string().nonempty().optional(), - trailingSubcaption: z.string().nonempty().optional(), - image: z.instanceof(Uint8Array).optional(), - imageTitle: z.string().nonempty().optional(), - imageSubtitle: z.string().nonempty().optional(), - summary: z.string().nonempty().optional(), - }) +const resolvedAppLayoutSchema = z.object({ + caption: z.string().nonempty().optional(), + subcaption: z.string().nonempty().optional(), + trailingCaption: z.string().nonempty().optional(), + trailingSubcaption: z.string().nonempty().optional(), + image: z.instanceof(Uint8Array).optional(), + imageTitle: z.string().nonempty().optional(), + imageSubtitle: z.string().nonempty().optional(), + summary: z.string().nonempty().optional(), +}); + +export const appLayoutSchema = resolvedAppLayoutSchema .refine( (layout) => layout.caption !== undefined || @@ -63,7 +64,7 @@ export type AppLayout = z.infer; const urlAccessor = z.function({ input: [], output: z.promise(z.url()) }); const layoutAccessor = z.function({ input: [], - output: z.promise(appLayoutSchema), + output: z.promise(resolvedAppLayoutSchema), }); export const appSchema = z.object({ @@ -192,6 +193,36 @@ export const asApp = (url: AppUrl, options: AppOptions = {}): App => { }); }; +/** + * Construct app content from an already-decoded URL and visible layout. + * Provider adapters use this for inbound native app cards so reading the + * content never refetches or invents metadata that the sender did not supply. + * + * @param url - The decoded URL delivered by the native app card. + * @param layout - The decoded layout or a lazy resolver for native media. + * @param options - Optional app rendering behavior. + * @returns Validated app content backed only by the decoded native fields. + */ +export const asResolvedApp = ( + url: string, + layout: AppLayout | (() => Promise), + options: AppOptions = {} +): App => { + const resolvedUrl = z.url().parse(url); + const getLayout = memoize(async () => + resolvedAppLayoutSchema.parse( + typeof layout === "function" ? await layout() : layout + ) + ); + + return appSchema.parse({ + type: "app", + url: () => Promise.resolve(resolvedUrl), + layout: getLayout, + ...options, + }); +}; + /** * Construct an app card from a URL. * diff --git a/packages/imessage/package.json b/packages/imessage/package.json index feb86143..d469665c 100644 --- a/packages/imessage/package.json +++ b/packages/imessage/package.json @@ -46,7 +46,7 @@ }, "dependencies": { "@grpc/grpc-js": "^1.14.4", - "@photon-ai/advanced-imessage": "^2.0.2", + "@photon-ai/advanced-imessage": "^2.1.0", "@photon-ai/otel": "^3.3.0", "lru-cache": "^11.0.0", "marked": "^18.0.5", diff --git a/packages/imessage/src/remote/inbound.ts b/packages/imessage/src/remote/inbound.ts index a94761d7..8fee791e 100644 --- a/packages/imessage/src/remote/inbound.ts +++ b/packages/imessage/src/remote/inbound.ts @@ -11,6 +11,7 @@ import { asContact, asCustom, asReply, + asResolvedApp, asText, asVoice, createLogger, @@ -252,12 +253,47 @@ const buildOrderedPartMessage = async ( const unsupportedMessageContent = (): Content => asCustom({ imessage_type: "unsupported-message" }); +const toMiniAppContent = ( + client: AdvancedIMessage, + message: AppleMessage +): Content | undefined => { + const miniApp = message.content.miniApp; + if (!(miniApp?.url && miniApp.layout)) { + return; + } + + const imageAttachment = messageAttachments(message).find((attachment) => + normalizeAppleAttachmentMimeType(attachment).startsWith("image/") + ); + const layout = imageAttachment + ? async () => ({ + ...miniApp.layout, + image: Uint8Array.from( + await downloadPrimaryAttachment(client, imageAttachment.guid) + ), + }) + : miniApp.layout; + + return asResolvedApp(miniApp.url, layout, { + live: miniApp.live, + }); +}; + const buildUnwrappedContentMessage = async ( client: AdvancedIMessage, base: RemoteMessageBase, message: AppleMessage, messageGuidStr: string ): Promise => { + const miniAppContent = toMiniAppContent(client, message); + if (miniAppContent) { + return { + ...base, + id: messageGuidStr, + content: miniAppContent, + }; + } + const attachments = messageAttachments(message); const voiceAttachmentGuid = message.isAudioMessage ? attachments.find((attachment) => appleAudioMimeType(attachment))?.guid diff --git a/packages/imessage/src/remote/message-metadata.ts b/packages/imessage/src/remote/message-metadata.ts index 6b6c9482..7b011a22 100644 --- a/packages/imessage/src/remote/message-metadata.ts +++ b/packages/imessage/src/remote/message-metadata.ts @@ -5,6 +5,8 @@ import type { MessageMention, MessagePlacedSticker, MessageReaction, + MiniAppContent, + MiniAppLayoutInfo, SingleServiceAddressInfo, StickerPlacement, TextFormat, @@ -13,6 +15,8 @@ import type { IMessageAppliedReaction, IMessageAttachmentMetadata, IMessageMention, + IMessageMiniAppContent, + IMessageMiniAppLayoutInfo, IMessageNativeMessageMetadata, IMessagePlacedSticker, IMessageReaction, @@ -34,6 +38,29 @@ const toMention = (mention: MessageMention): IMessageMention => ({ start: mention.start, }); +const toMiniAppLayout = ( + layout: MiniAppLayoutInfo +): IMessageMiniAppLayoutInfo => ({ + caption: layout.caption, + imageSubtitle: layout.imageSubtitle, + imageTitle: layout.imageTitle, + subcaption: layout.subcaption, + summary: layout.summary, + trailingCaption: layout.trailingCaption, + trailingSubcaption: layout.trailingSubcaption, +}); + +const toMiniAppContent = (miniApp: MiniAppContent): IMessageMiniAppContent => ({ + appName: miniApp.appName, + appStoreId: miniApp.appStoreId, + extensionBundleId: miniApp.extensionBundleId, + layout: miniApp.layout ? toMiniAppLayout(miniApp.layout) : undefined, + live: miniApp.live, + sessionId: miniApp.sessionId, + teamId: miniApp.teamId, + url: miniApp.url, +}); + const toAttachmentMetadata = ( attachment: AttachmentInfo ): IMessageAttachmentMetadata => ({ @@ -134,6 +161,9 @@ export const toMessageMetadata = ( sendErrorCode: native.sendErrorCode, nativeText: native.content?.text, + miniApp: native.content?.miniApp + ? toMiniAppContent(native.content.miniApp) + : undefined, formatting: native.content?.formatting?.map(toTextFormat) ?? [], mentions: native.content?.mentions?.map(toMention) ?? [], subject: native.subject, diff --git a/packages/imessage/src/types.ts b/packages/imessage/src/types.ts index e0d33ca8..f672e637 100644 --- a/packages/imessage/src/types.ts +++ b/packages/imessage/src/types.ts @@ -57,6 +57,31 @@ const miniAppCardSessionSchema = z.object({ targetMessageGuid: z.string(), }); +const miniAppLayoutInfoSchema = z + .object({ + caption: z.string().optional(), + imageSubtitle: z.string().optional(), + imageTitle: z.string().optional(), + subcaption: z.string().optional(), + summary: z.string().optional(), + trailingCaption: z.string().optional(), + trailingSubcaption: z.string().optional(), + }) + .readonly(); + +const miniAppContentSchema = z + .object({ + appName: z.string().optional(), + appStoreId: z.number().int().positive().optional(), + extensionBundleId: z.string(), + layout: miniAppLayoutInfoSchema.optional(), + live: z.boolean(), + sessionId: z.string().optional(), + teamId: z.string(), + url: z.url().optional(), + }) + .readonly(); + const textFormatSchema = z .object({ effectName: z.string().optional(), @@ -178,6 +203,7 @@ export const nativeMessageMetadataSchema = z.object({ sendErrorCode: z.number().int(), nativeText: z.string().optional(), + miniApp: miniAppContentSchema.optional(), formatting: z.array(textFormatSchema).readonly(), mentions: z.array(mentionSchema).readonly(), subject: z.string().optional(), @@ -231,6 +257,8 @@ export type IMessageAttachmentMetadata = z.infer< typeof attachmentMetadataSchema >; export type IMessageMention = z.infer; +export type IMessageMiniAppContent = z.infer; +export type IMessageMiniAppLayoutInfo = z.infer; export type IMessageNativeMessageMetadata = z.infer< typeof nativeMessageMetadataSchema >; diff --git a/packages/imessage/test/remote/inbound.test.ts b/packages/imessage/test/remote/inbound.test.ts index b833365d..38171127 100644 --- a/packages/imessage/test/remote/inbound.test.ts +++ b/packages/imessage/test/remote/inbound.test.ts @@ -1,5 +1,6 @@ import { type AdvancedIMessage, + decodeCatchUpEvent, type MessageEvent, NotFoundError, } from "@photon-ai/advanced-imessage/grpc"; @@ -19,6 +20,14 @@ type GroupItem = Extract< const RECEIVED_AT = new Date(1_700_000_000_000); const ATTACHMENT_PLACEHOLDER = "\uFFFC"; +// Fixed v11.3.0 CatchUpEventsResponse frame containing Message.content.mini_app. +// It catches field-number drift across server, generated client, and Spectrum. +const MINI_APP_CATCH_UP_FRAME = Uint8Array.from( + Buffer.from( + "CMsRUt8CChdpTWVzc2FnZTstOysxNTU1MTIzNDU2NxIGCIfH8dMGUrsCCrgCCg1taW5pLWFwcC1ndWlkEvMBOvABCgpQOFhUNjIzMlNMEiZjb2Rlcy5waG90b24uRXhhbXBsZS5NZXNzYWdlc0V4dGVuc2lvbhoLRXhhbXBsZSBBcHAiHmh0dHBzOi8vZXhhbXBsZS5jb20vY2FyZD9pZD00MiokOEQ4OTgwMzQtNDA3Qi00RkY1LTkxRTgtOURDMTg5MTFEQ0E5MNKF2MwEOAFCXwoHQ2FwdGlvbhIKU3ViY2FwdGlvbhoIVHJhaWxpbmciD1RyYWlsaW5nIGRldGFpbCoLSW1hZ2UgdGl0bGUyDkltYWdlIHN1YnRpdGxlOhBGYWxsYmFjayBzdW1tYXJ5UgYIh8fx0waiAQ4KDCsxNTU1MTIzNDU2N+IFF2lNZXNzYWdlOy07KzE1NTUxMjM0NTY3", + "base64" + ) +); const client = { messages: { @@ -170,6 +179,185 @@ describe("iMessage remote toInboundMessages content", () => { ); }); + it("maps decoded mini-app cards to app content without refetching metadata", async () => { + const [message] = await toInboundMessages( + client, + new MessageCache(), + receivedEvent( + { address: "+15551234567" }, + { + text: undefined, + miniApp: { + appName: "Example App", + appStoreId: 1_234_567_890, + extensionBundleId: "codes.photon.Example.MessagesExtension", + layout: { + caption: "Caption", + imageSubtitle: "Image subtitle", + imageTitle: "Image title", + subcaption: "Subcaption", + summary: "Fallback summary", + trailingCaption: "Trailing", + trailingSubcaption: "Trailing detail", + }, + live: true, + sessionId: "8D898034-407B-4FF5-91E8-9DC18911DCA9", + teamId: "P8XT6232SL", + url: "https://example.com/card?id=42", + }, + } + ), + "+15550000000" + ); + + expect(message?.content.type).toBe("app"); + if (message?.content.type !== "app") { + throw new Error("expected app content"); + } + expect(await message.content.url()).toBe("https://example.com/card?id=42"); + expect(await message.content.layout()).toEqual({ + caption: "Caption", + imageSubtitle: "Image subtitle", + imageTitle: "Image title", + subcaption: "Subcaption", + summary: "Fallback summary", + trailingCaption: "Trailing", + trailingSubcaption: "Trailing detail", + }); + expect(message.content.live).toBe(true); + expect(message.miniApp).toMatchObject({ + appName: "Example App", + extensionBundleId: "codes.photon.Example.MessagesExtension", + sessionId: "8D898034-407B-4FF5-91E8-9DC18911DCA9", + teamId: "P8XT6232SL", + }); + }); + + it("hydrates the mini-app image lazily from its native attachment", async () => { + const imageBytes = Uint8Array.from([0xff, 0xd8, 0xff, 0xd9]); + let downloadCount = 0; + const imageClient = { + attachments: { + downloadStream: () => { + const frames = { + async *[Symbol.asyncIterator]() { + downloadCount += 1; + yield { data: imageBytes, type: "primaryChunk" }; + }, + close: async () => undefined, + }; + return frames; + }, + }, + messages: client.messages, + } as unknown as AdvancedIMessage; + + const [message] = await toInboundMessages( + imageClient, + new MessageCache(), + receivedEvent( + { address: "+15551234567" }, + { + attachments: [ + attachment( + "card-image", + "card.jpg", + "image/jpeg", + imageBytes.length + ), + ], + miniApp: { + extensionBundleId: "codes.photon.Example.MessagesExtension", + layout: { caption: "Caption", imageTitle: "Image title" }, + live: false, + teamId: "P8XT6232SL", + url: "https://example.com/card", + }, + text: undefined, + } + ), + "+15550000000" + ); + + expect(message?.content.type).toBe("app"); + if (message?.content.type !== "app") { + throw new Error("expected app content"); + } + expect(downloadCount).toBe(0); + expect(await message.content.layout()).toEqual({ + caption: "Caption", + image: imageBytes, + imageTitle: "Image title", + }); + expect(await message.content.layout()).toEqual({ + caption: "Caption", + image: imageBytes, + imageTitle: "Image title", + }); + expect(downloadCount).toBe(1); + }); + + it("preserves mini-app content from the released protobuf frame end to end", async () => { + const event = decodeCatchUpEvent(MINI_APP_CATCH_UP_FRAME); + if (event?.type !== "message.received") { + throw new Error("expected released message.received fixture"); + } + + const [message] = await toInboundMessages( + client, + new MessageCache(), + event, + "+15550000000" + ); + + expect(message?.content.type).toBe("app"); + if (message?.content.type !== "app") { + throw new Error("expected app content"); + } + expect(await message.content.url()).toBe("https://example.com/card?id=42"); + expect(await message.content.layout()).toMatchObject({ + caption: "Caption", + imageSubtitle: "Image subtitle", + imageTitle: "Image title", + summary: "Fallback summary", + }); + expect(message.miniApp?.appStoreId).toBe(1_234_567_890); + }); + + it("keeps identity-only mini-app metadata when no renderable card is available", async () => { + const [message] = await toInboundMessages( + client, + new MessageCache(), + receivedEvent( + { address: "+15551234567" }, + { + text: undefined, + miniApp: { + extensionBundleId: "codes.photon.Example.MessagesExtension", + live: false, + teamId: "P8XT6232SL", + }, + } + ), + "+15550000000" + ); + + expect(message?.content).toEqual({ + raw: { imessage_type: "unsupported-message" }, + type: "custom", + }); + expect(message?.miniApp).toEqual({ + appName: undefined, + appStoreId: undefined, + extensionBundleId: "codes.photon.Example.MessagesExtension", + layout: undefined, + live: false, + sessionId: undefined, + teamId: "P8XT6232SL", + url: undefined, + }); + }); + it("surfaces a URL balloon as plain text rather than a rich link", async () => { const [message] = await toInboundMessages( client, diff --git a/packages/imessage/test/remote/message-metadata.test.ts b/packages/imessage/test/remote/message-metadata.test.ts index 4bfc2482..2bd669c1 100644 --- a/packages/imessage/test/remote/message-metadata.test.ts +++ b/packages/imessage/test/remote/message-metadata.test.ts @@ -62,6 +62,24 @@ const completeAdvancedMessage = (): AdvancedIMessageMessage => ({ }, ], mentions: [{ address: "+15551230002", length: 12, start: 6 }], + miniApp: { + appName: "Example App", + appStoreId: 1_234_567_890, + extensionBundleId: "codes.photon.Example.MessagesExtension", + layout: { + caption: "Caption", + imageSubtitle: "Image subtitle", + imageTitle: "Image title", + subcaption: "Subcaption", + summary: "Fallback summary", + trailingCaption: "Trailing", + trailingSubcaption: "Trailing detail", + }, + live: true, + sessionId: "8D898034-407B-4FF5-91E8-9DC18911DCA9", + teamId: "P8XT6232SL", + url: "https://example.com/card?id=42", + }, text: "Hello @Taylor", }, dataDetectorResultsPresent: true, @@ -227,6 +245,24 @@ describe("curated Advanced iMessage metadata", () => { }, ], mentions: [{ address: "+15551230002", length: 12, start: 6 }], + miniApp: { + appName: "Example App", + appStoreId: 1_234_567_890, + extensionBundleId: "codes.photon.Example.MessagesExtension", + layout: { + caption: "Caption", + imageSubtitle: "Image subtitle", + imageTitle: "Image title", + subcaption: "Subcaption", + summary: "Fallback summary", + trailingCaption: "Trailing", + trailingSubcaption: "Trailing detail", + }, + live: true, + sessionId: "8D898034-407B-4FF5-91E8-9DC18911DCA9", + teamId: "P8XT6232SL", + url: "https://example.com/card?id=42", + }, nativeText: "Hello @Taylor", subject: "Native subject", }); diff --git a/packages/spectrum-ts/test/imessage-package-boundary.test.ts b/packages/spectrum-ts/test/imessage-package-boundary.test.ts index 5239c85b..b5726060 100644 --- a/packages/spectrum-ts/test/imessage-package-boundary.test.ts +++ b/packages/spectrum-ts/test/imessage-package-boundary.test.ts @@ -39,7 +39,7 @@ describe("iMessage package boundary", () => { expect(manifest.dependencies).toMatchObject({ "@grpc/grpc-js": "^1.14.4", - "@photon-ai/advanced-imessage": "^2.0.2", + "@photon-ai/advanced-imessage": "^2.1.0", "nice-grpc": "^2.1.16", "nice-grpc-common": "^2.0.3", });