From 029ae36f32e02a3b422cfbcd48d52621a4266afd Mon Sep 17 00:00:00 2001 From: provos Date: Sat, 26 Sep 2026 12:28:03 -0700 Subject: [PATCH 1/2] fix(inpaint): preview the picked candidate on the canvas Picking a candidate sets the server's display image, which since the phase 3 view modes only shows in the Input and Parallax views; the Inpaint step works in the Slice view, so the canvas kept showing the slice and its painted mask. The Slice and Composite views now draw the picked candidate in place of its slice, and the mask steps aside (and takes no strokes) while a candidate is previewed. The e2e inpainting scenario now reads the visible canvas after picking each candidate (new UiDriver.visibleCanvasPixel); it fails without this fix. Co-Authored-By: Claude Opus 5.5 --- e2e/drivers/svelte.ts | 26 +++++++++ e2e/drivers/types.ts | 5 ++ e2e/parallax-maker.spec.ts | 11 ++++ frontend/src/lib/candidatePreview.ts | 19 +++++++ .../lib/components/canvas/MaskCanvas.svelte | 8 ++- .../components/inpainting/InpaintPanel.svelte | 5 ++ .../components/upload/InputImagePanel.svelte | 16 +++++- .../components/upload/InputImagePanel.test.ts | 57 ++++++++++++++++++- 8 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 frontend/src/lib/candidatePreview.ts diff --git a/e2e/drivers/svelte.ts b/e2e/drivers/svelte.ts index d184356..4258cdd 100644 --- a/e2e/drivers/svelte.ts +++ b/e2e/drivers/svelte.ts @@ -113,6 +113,32 @@ export class SvelteDriver implements UiDriver { return this.page.getByTestId('canvas-image'); } + async visibleCanvasPixel(x: number, y: number): Promise { + const [naturalWidth, naturalHeight] = await this.mainImage().evaluate((img: HTMLImageElement) => [ + img.naturalWidth, + img.naturalHeight, + ]); + const png = await this.canvasImage().screenshot(); + return this.page.evaluate( + async ({ data, x, y, naturalWidth, naturalHeight }) => { + const shot = new Image(); + shot.src = `data:image/png;base64,${data}`; + await shot.decode(); + const canvas = document.createElement('canvas'); + canvas.width = shot.width; + canvas.height = shot.height; + const context = canvas.getContext('2d'); + if (!context) throw new Error('No 2D context'); + context.drawImage(shot, 0, 0); + // Sample the center of the source pixel in screenshot pixels. + const px = Math.floor(((x + 0.5) * shot.width) / naturalWidth); + const py = Math.floor(((y + 0.5) * shot.height) / naturalHeight); + return Array.from(context.getImageData(px, py, 1, 1).data); + }, + { data: png.toString('base64'), x, y, naturalWidth, naturalHeight }, + ); + } + depthImage(): Locator { return this.page.getByTestId('depth-image'); } diff --git a/e2e/drivers/types.ts b/e2e/drivers/types.ts index c721afc..c9e7d74 100644 --- a/e2e/drivers/types.ts +++ b/e2e/drivers/types.ts @@ -57,6 +57,11 @@ export interface UiDriver { * (its box follows zoom/pan); use it for geometry, `mainImage()` for pixels. */ canvasImage(): Locator; + /** + * The RGBA the user sees on the canvas at source pixel (x, y), whatever + * the current view draws there (a screenshot, so overlays count too). + */ + visibleCanvasPixel(x: number, y: number): Promise; depthImage(): Locator; /** Every slice thumbnail as displayed (checkerboard-composited), in no particular order. */ sliceImages(): Locator; diff --git a/e2e/parallax-maker.spec.ts b/e2e/parallax-maker.spec.ts index 5cd0a2a..4cd1fd7 100644 --- a/e2e/parallax-maker.spec.ts +++ b/e2e/parallax-maker.spec.ts @@ -266,7 +266,18 @@ test('painted mask drives three checkerboard candidates, apply, and undo', async expect(candidateOutside).toEqual(originalOutside); expect(await ui.maskCanvasPainted()).toBe(true); + // Picking a candidate previews it on the canvas, in place of the slice and + // its painted mask, so each one can be inspected before applying it. + const nearest = (rgba: number[], palette: number[][]) => + Math.min(...palette.map((color) => Math.max(...color.map((value, i) => Math.abs(rgba[i] - value))))); + await ui.selectCandidate(0); + await expect + .poll(async () => nearest(await ui.visibleCanvasPixel(...inside), [[0, 255, 255], [255, 0, 255]])) + .toBeLessThanOrEqual(24); await ui.selectCandidate(1); + await expect + .poll(async () => nearest(await ui.visibleCanvasPixel(...inside), [[255, 128, 0], [0, 64, 255]])) + .toBeLessThanOrEqual(24); await ui.applyCandidate(); await expect(ui.log()).toContainText(/Inpainting applied to slice 1/); // The applied version has no mask; the canvas must not keep showing the old stroke. diff --git a/frontend/src/lib/candidatePreview.ts b/frontend/src/lib/candidatePreview.ts new file mode 100644 index 0000000..894be9d --- /dev/null +++ b/frontend/src/lib/candidatePreview.ts @@ -0,0 +1,19 @@ +/** + * The inpainting candidate being previewed: the one picked in the Inpaint + * panel, while it belongs to the selected slice. The canvas shows it in + * place of that slice (Slice and Composite views; the Input view's server + * display image already is the candidate) and hides the painted mask, so + * the filled-in area can be inspected. + */ +import type { ProjectView } from './api/types'; + +export type CandidatePreview = { sliceIndex: number; candidate: number; url: string }; + +export function candidatePreview(view: ProjectView | null | undefined): CandidatePreview | null { + const candidates = view?.inpainting.candidates; + const picked = view?.inpainting.selectedCandidate; + if (!view || !candidates || picked == null) return null; + if (candidates.sliceIndex !== view.selectedSlice) return null; + const image = candidates.images[picked]; + return image ? { sliceIndex: candidates.sliceIndex, candidate: picked, url: image.url } : null; +} diff --git a/frontend/src/lib/components/canvas/MaskCanvas.svelte b/frontend/src/lib/components/canvas/MaskCanvas.svelte index 5775586..d61bff5 100644 --- a/frontend/src/lib/components/canvas/MaskCanvas.svelte +++ b/frontend/src/lib/components/canvas/MaskCanvas.svelte @@ -21,6 +21,7 @@ */ import { projectStore } from '../../state/project.svelte'; import { uiStore } from '../../state/ui.svelte'; + import { candidatePreview } from '../../candidatePreview'; import { isBusy } from '../../state/busy.svelte'; import { logStore } from '../../state/logs.svelte'; import { canvasSaveStore } from '../../state/canvas.svelte'; @@ -59,8 +60,11 @@ // Painting is the Brush tool's job; the painted mask shows only in the // views it is drawn over (the selected slice, or the working image). - const interactiveNow = $derived(uiStore.tool === 'brush'); - const visibleNow = $derived(uiStore.view === 'slice' || uiStore.view === 'input'); + // While a picked candidate is previewed, the mask steps aside so the + // filled-in area can be inspected (and can't be painted over unseen). + const previewing = $derived(candidatePreview(projectStore.view) !== null); + const interactiveNow = $derived(uiStore.tool === 'brush' && !previewing); + const visibleNow = $derived((uiStore.view === 'slice' || uiStore.view === 'input') && !previewing); // The brush preview is only meaningful while painting is possible; drop // it as soon as another tool is picked rather than leaving a stale circle diff --git a/frontend/src/lib/components/inpainting/InpaintPanel.svelte b/frontend/src/lib/components/inpainting/InpaintPanel.svelte index 281ad0a..40d503c 100644 --- a/frontend/src/lib/components/inpainting/InpaintPanel.svelte +++ b/frontend/src/lib/components/inpainting/InpaintPanel.svelte @@ -297,6 +297,11 @@

Candidates appear here after you generate.

{/each} + {#if selectedCandidate !== null && hasCandidates} +

+ Candidate {selectedCandidate + 1} is shown on the canvas. Click it again to go back to the mask. +

+ {/if}