diff --git a/apps/mobile/src/MathLabel.tsx b/apps/mobile/src/MathLabel.tsx index cb5b43d28a..5da69d6695 100644 --- a/apps/mobile/src/MathLabel.tsx +++ b/apps/mobile/src/MathLabel.tsx @@ -8,10 +8,19 @@ import { useTheme } from "./theme"; /** The web draws math with KaTeX at 1.21em; MathJax's TeX fonts share its metrics. */ export const MATH_SCALE = 1.21; -/** Source Sans 3's ascent and descent (hhea 1024 and 400 per 1000), which the phone lays - * prose lines out with. */ +/** Source Sans 3's ascent and descent (hhea 1024 and 400 per 1000, every weight and slant), + * which the phone lays prose lines out with. */ const ASCENT = 1.024; const DESCENT = 0.4; +/** JetBrains Mono's (hhea 1020 and 300 per 1000), for inline code. */ +const MONO = { ascent: 1.02, descent: 0.3 }; +/** The descender of the font iOS gives an inline view's character, in points: React Native + * gives that character no font (RCTNSAttributedStringFragmentFromFragment), so TextKit uses + * its default, Helvetica at 12 points (descender 0.23 em), at any text size. */ +const VIEW_FONT_DESCENT = 2.76; + +/** A font in a line of text: its size, and whether it is the code face. */ +export type TextRun = { fontSize: number; mono?: boolean }; /** Row and screen padding beside a label, so inline math wider than this scales down. */ const BESIDE_LABEL = 120; /** No equation is drawn taller than this many lines, so huge math never allocates a huge picture. */ @@ -94,6 +103,37 @@ export function lineHeightFor(sizes: Size[], fontSize: number, line: number, fon return paragraphLine(scaled, fontSize * fontScale, line * fontScale); } +/** The fonts around an inline picture on iOS: those in the hard line it sits in (up to and + * including that line's newline), and every one in its Text. */ +export type InlineFonts = { line: TextRun[]; text: TextRun[] }; + +/** The largest descent and the tallest line height among some fonts, in points. */ +function iosFonts(runs: TextRun[]) { + const fonts = runs.map(({ fontSize, mono }) => mono ? { ascent: MONO.ascent * fontSize, descent: MONO.descent * fontSize } + : { ascent: ASCENT * fontSize, descent: DESCENT * fontSize }); + return { descent: Math.max(...fonts.map((font) => font.descent)), tallest: Math.max(...fonts.map((font) => font.ascent + font.descent)) }; +} + +/** How far an inline picture must rise on iOS to sit on its line's baseline, in points at + * the font scale. React Native's iOS text puts an inline view's bottom at its line's bottom + * plus its character's font descender (RCTTextLayoutManager's attachment frames), and that + * character's font is TextKit's default, not the text's. The text's baseline is its line's + * largest descent up from the bottom (TextKit), raised by half the lineHeight over the Text's + * tallest font (RCTApplyBaselineOffsetForRange, over the whole Text). A line with no text has + * only TextKit's font, so its pictures already sit on its baseline. The lift never takes a + * picture above its line's top: a picture taller than the line leaves above the baseline sits + * lower instead, as the line doesn't grow for it. A line known here is a hard line; when text + * wraps, a wrapped line holding only math is lifted as if it held the hard line's text, still + * inside its line. Android puts an inline view on the baseline itself. */ +export function inlineLift(lineHeight: number, size: Size, fonts: InlineFonts, fontScale: number, platform: string = Platform.OS): number { + if (platform !== "ios" || !fonts.line.length) return 0; + const { descent } = iosFonts(fonts.line); + const { tallest } = iosFonts(fonts.text); + const baseline = (descent + Math.max(0, (lineHeight - tallest) / 2)) * fontScale; + const room = (lineHeight - (size.height - size.depth)) * fontScale - VIEW_FONT_DESCENT; + return Math.max(0, Math.min(baseline - VIEW_FONT_DESCENT, room)); +} + /** A label with its math drawn as pictures, sized before it is drawn so the row never * grows later. It is never reachable by a screen reader on its own: whatever holds it * carries the spoken label. `style` must set fontSize and lineHeight. */ @@ -127,10 +167,26 @@ export function MathLabel({ pieces, style, color }: { pieces: LabelPiece[]; styl } const sizes = block.map((piece) => piece.kind === "math" ? fit(piece.drawn, em, maxWidth, maxHeight) : undefined); const line = lineHeightFor(sizes.filter((size) => size !== undefined), fontSize, lineHeight, fontScale); + // The block's text is all in the label's font: each hard line's, and the line each picture is on. + const lines: TextRun[][] = [[]]; + const lineOf = block.map((piece) => { + if (piece.kind === "math") return lines.length - 1; + piece.text.split("\n").forEach((part, at) => { + if (at) { + // A newline belongs to the line it ends. + lines.at(-1)?.push({ fontSize }); + lines.push([]); + } + if (part) lines.at(-1)?.push({ fontSize }); + }); + return -1; + }); + const text = lines.flat(); return ( {block.map((piece, at) => piece.kind === "text" ? piece.text - : )} + : )} ); })} @@ -140,8 +196,10 @@ export function MathLabel({ pieces, style, color }: { pieces: LabelPiece[]; styl /** A drawn equation in a box of its exact size. Without `live` the box stays empty, so a * picture coming or going never moves anything. */ -export function MathPicture({ svg, drawn, size, scale, color, inline, live = true }: { +export function MathPicture({ svg, drawn, size, scale, color, inline, live = true, lift = 0 }: { svg: string; drawn: DrawnMath; size: Size; scale: number; color: string; inline?: boolean; live?: boolean; + /** Inline only: inlineLift for the line it sits in, in points at the font scale. */ + lift?: number; }) { const width = size.width * scale; const height = size.height * scale; @@ -154,9 +212,10 @@ export function MathPicture({ svg, drawn, size, scale, color, inline, live = tru ); } // The inline box is the math above the baseline, which it sits on; the picture hangs - // below it by the depth, into the space paragraphLine leaves. + // below it by the depth, into the space paragraphLine leaves. Lifting moves only what + // is drawn, so the line's layout is unchanged. return ( - + {live && } diff --git a/apps/mobile/src/NativeMarkdown.tsx b/apps/mobile/src/NativeMarkdown.tsx index acd292dc12..360a7b9ea0 100644 --- a/apps/mobile/src/NativeMarkdown.tsx +++ b/apps/mobile/src/NativeMarkdown.tsx @@ -3,7 +3,7 @@ import { Text, View, useWindowDimensions, type HostInstance, type LayoutChangeEv import { drawingsOf, equationOf } from "./message-math"; import { useLivePictures } from "./math-window"; import type { LabelPiece } from "./choice-math"; -import { fit, lineHeightFor, MathPicture, MATH_SCALE, MAX_LINES, useScreenReader, useSpeech, type Size } from "./MathLabel"; +import { fit, inlineLift, lineHeightFor, MathPicture, MATH_SCALE, MAX_LINES, useScreenReader, useSpeech, type InlineFonts, type Size, type TextRun } from "./MathLabel"; import { spokenLabel, spokenMath } from "./math-speech"; import { httpsLink, type MarkdownToken } from "./render-markdown"; import { fonts, radius, space, useTheme } from "./theme"; @@ -17,6 +17,8 @@ const CLINGS = /^[.,;:!?)\]}%'"\u2019\u201d\u2026]/; const MIN_CELL_MATH = 24; /** U+2060 WORD JOINER: no line break on either side. */ const JOIN = "\u2060"; +/** Inline code's size, in JetBrains Mono. */ +const CODE_SIZE = 14; interface Node { token: MarkdownToken; @@ -35,6 +37,9 @@ function Live({ pictures, children }: { interface InlineMath { sizes: Map; live: boolean; + /** The fonts around each drawn equation, for inlineLift. */ + fonts: Map; + lineHeight: number; } function hasMath(token: MarkdownToken): boolean { @@ -96,6 +101,27 @@ export const NativeMarkdown = memo(function NativeMarkdown({ tokens, openLink }: return found; } + /** The fonts around each drawn equation in a line of text: those of the hard line it is + * on (a break belongs to the line it ends), and every one in the text. */ + function inlineFonts(nodes: Node[], sizes: Map, fontSize: number) { + const lines: TextRun[][] = [[]]; + const lineOf = new Map(); + const walk = (list: Node[]) => { + for (const { token, children } of list) { + if (token.type === "code_inline") lines.at(-1)?.push({ fontSize: CODE_SIZE, mono: true }); + else if (token.type === "softbreak" || token.type === "hardbreak") { + lines.at(-1)?.push({ fontSize }); + lines.push([]); + } else if (sizes.has(token)) lineOf.set(token, lines.length - 1); + else if (children.length) walk(children); + else if (token.type === "math_inline" || token.content) lines.at(-1)?.push({ fontSize }); + } + }; + walk(nodes); + const text = lines.flat(); + return new Map([...lineOf].map(([token, at]) => [token, { line: lines[at] ?? [], text }])); + } + /** A line of text whose line height leaves room for the inline math in it. Inline math * wider than `width` scales down. */ function line(key: number, nodes: Node[], style: { fontSize: number; lineHeight: number }, extra?: object, role?: "header", width = maxWidth) { @@ -106,7 +132,7 @@ export const NativeMarkdown = memo(function NativeMarkdown({ tokens, openLink }: {(live, ref, onLayout) => ( - {inline(nodes, { sizes, live })} + {inline(nodes, { sizes, live, fonts: inlineFonts(nodes, sizes, style.fontSize), lineHeight })} )} @@ -125,7 +151,7 @@ export const NativeMarkdown = memo(function NativeMarkdown({ tokens, openLink }: case "s_open": return {inline(children, math, bold, italic)}; case "code_inline": - return {token.content}; + return {token.content}; case "softbreak": case "hardbreak": return "\n"; @@ -140,7 +166,8 @@ export const NativeMarkdown = memo(function NativeMarkdown({ tokens, openLink }: const drawn = size && pictures.get(token); // Math that can't be drawn shows its source. if (!size || !drawn) return token.markup === "$" ? `$${token.content}$` : `\\(${token.content}\\)`; - const picture = ; + const picture = ; // A period or comma right after the math never wraps away from it. const after = nodes[index + 1]?.token; return after?.type === "text" && CLINGS.test(after.content) ? [picture, JOIN] : picture; diff --git a/apps/mobile/tests/math-baseline.test.ts b/apps/mobile/tests/math-baseline.test.ts new file mode 100644 index 0000000000..801524fcfe --- /dev/null +++ b/apps/mobile/tests/math-baseline.test.ts @@ -0,0 +1,151 @@ +// Inline math sits on the text's baseline on iOS: the answered card's math and the period +// after message math (#313). React Native's iOS text places an inline view from its line's +// bottom and its character's font, TextKit's default, so inlineLift moves each picture up +// onto the baseline. Android places inline views on the baseline and gets no lift. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { createElement, type ReactElement } from "react"; +import { Platform } from "react-native"; +import { act, create, type ReactTestInstance, type ReactTestRenderer } from "react-test-renderer"; +import { AnswerLine } from "../src/AnswerLine.tsx"; +import { ChoiceRow } from "../src/ChoiceRow.tsx"; +import { Markdown } from "../src/Markdown.tsx"; +import { inlineLift } from "../src/MathLabel.tsx"; + +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const IOS_TEXT = new URL("../node_modules/react-native/ReactCommon/react/renderer/textlayoutmanager/platform/ios/react/renderer/textlayoutmanager/", import.meta.url); +/** TextKit's Helvetica 12 descender, where React Native puts an inline view's bottom above its line's bottom. */ +const VIEW_FONT_DESCENT = 2.76; + +test("React Native still places iOS inline views the way inlineLift undoes", () => { + // When React Native places them from the line's real baseline (upstream #59921), the lift + // must go: it would raise every picture above the text. + const layout = readFileSync(new URL("RCTTextLayoutManager.mm", IOS_TEXT), "utf8"); + assert.match(layout, /glyphRect\.origin\.y \+ glyphRect\.size\.height - attachmentSize\.height \+ font\.descender/); + const attributed = readFileSync(new URL("RCTAttributedTextUtils.mm", IOS_TEXT), "utf8"); + // An inline view's character gets no font, so TextKit gives it Helvetica 12. + assert.match(attributed, /attachment\.bounds = bounds;\s+return \[\[NSMutableAttributedString attributedStringWithAttachment:attachment\] mutableCopy\];/); + // The baseline offset is worked out once over the whole Text, not per line. + assert.match(attributed, /CGFloat baseLineOffset = \(maximumLineHeight - maximumFontLineHeight\) \/ 2\.0;/); + assert.match(attributed, /RCTApplyBaselineOffsetForRange\(attributedText, NSMakeRange\(0, attributedText\.length\)\);/); +}); + +async function render(element: ReactElement) { + let tree: ReactTestRenderer | undefined; + await act(async () => { tree = create(element); }); + return tree as ReactTestRenderer; +} + +const flat = (style: unknown) => [style].flat(Infinity).reduce>((all, part) => ({ ...all, ...(part as object) }), {}); + +const lifted = (box: ReactTestInstance) => + 0 - ((flat(box.props.style).transform as { translateY?: number }[] | undefined)?.find((step) => "translateY" in step)?.translateY ?? 0); + +/** Each inline picture: how far it is lifted, its size in points, and its Text line's height. */ +function inlineBoxes(root: ReactTestInstance) { + return root.findAll((node) => String(node.type) === "Text").flatMap((line) => + line.findAll((node) => typeof node.type === "function" && node.type.name === "MathPicture" && node.parent === line) + .map((picture) => ({ + lift: lifted(picture.children[0] as ReactTestInstance), + size: picture.props.size as { width: number; height: number; depth: number }, + lineHeight: flat(line.props.style).lineHeight as number, + }))); +} + +async function onPlatform(os: string, run: () => Promise) { + const was = Platform.OS; + Object.assign(Platform, { OS: os }); + try { return await run(); } finally { Object.assign(Platform, { OS: was }); } +} + +/** Renders on a platform and hands back each inline picture, then unmounts. */ +async function boxesOn(os: string, element: ReactElement) { + return onPlatform(os, async () => { + const tree = await render(element); + try { return inlineBoxes(tree.root); } finally { act(() => tree.unmount()); } + }); +} + +const card = (labels: string[]) => createElement(AnswerLine, { answer: { kind: "quiz", answers: labels.map((label, index) => ({ index, label, value: String(index) })) } }); +const message = (text: string) => createElement(Markdown, { text }); +const choice = (label: string) => createElement(ChoiceRow, { label, multi: false, selected: false, disabled: false, onPress: () => {} }); + +test("ios: the lift matches how far math sat below the text in the iPhone check's shots", async () => { + // Measured on the iPhone check before any lift (run 44871251269, 3x pictures): the bottom of + // a digit's ink against the bottom of the letters on the same line. A reading is good to a + // pixel (1/3 pt) at the ink threshold, and React Native rounds the view's origin to the pixel + // grid, so the lift must match within half a point. After the lift the same spots measured + // 0 or -1 px (run 44884978315). + const measured = [ + // The answered card and message prose: 12, 13, 12, 13, 12 and 13 px low. + { name: "answered card", element: card(["Train: $4$; test: $8$"]), px: [12, 13, 12, 13, 12, 13] }, + // A choice row: "4" and "8" 9 px low. + { name: "choice row", element: choice("Train: $4$; test: $8$"), px: [9, 9] }, + ]; + for (const { name, element, px } of measured) { + const low = px.reduce((sum, value) => sum + value, 0) / px.length / 3; + for (const { lift } of await boxesOn("ios", element)) { + assert.ok(Math.abs(lift - low) <= 0.5, `${name}: lift ${lift.toFixed(2)} pt, measured ${low.toFixed(2)} pt`); + } + } +}); + +for (const os of ["ios", "android"] as const) { + test(`${os}: the answered card's math and a message's inline math are ${os === "ios" ? "lifted" : "not moved"}, and math alone is not moved`, async () => { + const cases = [ + ["answered card", card(["Train: $4$; test: $8$", "$(+0.20, -0.10)$"])], + ["message", message("So the step is $0.1(-1)(-2) = 0.2$.")], + ["math alone", card(["$(+0.20, -0.10)$"])], + ["message math alone", message("$\\rule{1em}{4em}$")], + // A line of its own after a newline holds no text either. + ["math alone on a line after text", message("Earlier text\n$\\rule{1em}{4em}$")], + ] as const; + for (const [name, element] of cases) { + const boxes = await boxesOn(os, element); + assert.ok(boxes.length > 0, `${name}: inline math found`); + for (const { lift } of boxes) { + if (os === "ios" && !name.includes("alone")) assert.ok(lift > 3, `${name}: lifted ${lift}`); + else assert.equal(lift, 0, `${name}: lifted ${lift}, expected none`); + } + } + }); +} + +test("ios: the lift uses the fonts of the line the math is on", async () => { + // The second line holds only 14 pt code right beside the math: its descent is the code's 4.2 pt, + // not the prose's 6.4. The offset is the whole Text's: (24 - 1.424 * 16) / 2 = 0.608. + const [math] = await boxesOn("ios", message("Some words first\n`k`$x$")); + assert.ok(math); + assert.equal(math.lineHeight, 24); + assert.ok(Math.abs(math.lift - (4.2 + 0.608 - VIEW_FONT_DESCENT)) < 1e-6, `lifted ${math.lift}`); +}); + +test("ios: lines keep the line height they have on Android, and tall or deep math stays inside its line", async () => { + for (const text of ["Earlier text\nRule: $\\rule{1em}{4em}$.", "Deep: $\\rule[-2em]{1em}{1em}$ here.", "So the step is $0.1(-1)(-2) = 0.2$."]) { + const ios = await boxesOn("ios", message(text)); + const android = await boxesOn("android", message(text)); + assert.deepEqual(ios.map(({ lineHeight }) => lineHeight), android.map(({ lineHeight }) => lineHeight), `${text}: line heights`); + for (const { lift, size, lineHeight } of ios) { + // The box's bottom, up from its line's bottom, once lifted. + const bottom = VIEW_FONT_DESCENT + lift; + assert.ok(bottom + size.height - size.depth <= lineHeight + 1e-6, `${text}: top ${bottom + size.height - size.depth} inside ${lineHeight}`); + assert.ok(bottom - size.depth >= 0, `${text}: bottom ${bottom - size.depth} inside the line`); + } + } + const [prose] = await boxesOn("ios", message("So the step is $0.1(-1)(-2) = 0.2$.")); + assert.equal(prose?.lineHeight, 24, "a message line with ordinary math stays at 24"); + const [row] = await boxesOn("ios", choice("Train: $4$; test: $8$")); + assert.equal(row?.lineHeight, 21, "a choice row stays at the 21 it had before the lift"); +}); + +test("the lift is iOS only, and none for a line with no text", () => { + const size = { width: 10, height: 13, depth: 0 }; + assert.equal(inlineLift(24, size, { line: [{ fontSize: 16 }], text: [{ fontSize: 16 }] }, 1, "android"), 0); + assert.equal(inlineLift(24, size, { line: [], text: [{ fontSize: 16 }] }, 1, "ios"), 0); + // Helvetica's descender doesn't grow with Dynamic Type, so the lift isn't the 1x lift scaled. + const once = inlineLift(24, size, { line: [{ fontSize: 16 }], text: [{ fontSize: 16 }] }, 1, "ios"); + const twice = inlineLift(24, size, { line: [{ fontSize: 16 }], text: [{ fontSize: 16 }] }, 2, "ios"); + assert.ok(Math.abs(twice - (2 * (once + VIEW_FONT_DESCENT) - VIEW_FONT_DESCENT)) < 1e-6); +}); diff --git a/tests/fixtures/mobile-quiz/scenario.json b/tests/fixtures/mobile-quiz/scenario.json index 6eabca6f25..bfa47f9f0b 100644 --- a/tests/fixtures/mobile-quiz/scenario.json +++ b/tests/fixtures/mobile-quiz/scenario.json @@ -8,7 +8,7 @@ }, "times": 1, "reply": { - "text": "The first pair fits, since $4 < 8$. The second does not, since $0.20 > -0.10$. The third fits too: $-0.10 < 0.20$." + "text": "The first pair fits, since $4 < 8$. The second does not, since $0.20 > -0.10$. The third fits too: $-0.10 < 0.20$.\n\nA plain line comes first\n$\\rule{1em}{4em}$\n\nAnother plain line\nThen a rule: $\\rule{1em}{4em}$ on this line." } } ],