fix(inpaint): preview the picked candidate on the canvas

e2e/drivers/svelte.ts

@@ -113,6 +113,34 @@ export class SvelteDriver implements UiDriver {
     return this.page.getByTestId('canvas-image');
   }
 
+  async visibleCanvasPixel(x: number, y: number): Promise<number[]> {
+    // A picked candidate swaps image sources; sample only once they are decoded.
+    await waitForImage(this.mainImage());
+    const [naturalWidth, naturalHeight] = await this.mainImage().evaluate((img: HTMLImageElement) => [
+      img.naturalWidth,
+      img.naturalHeight,
+    ]);
+    const png = await this.canvasImage().screenshot();
+    return this.page.evaluate(
+      async ({ data, x, y, naturalWidth, naturalHeight }) => {
+        const shot = new Image();
+        shot.src = `data:image/png;base64,${data}`;
+        await shot.decode();
+        const canvas = document.createElement('canvas');
+        canvas.width = shot.width;
+        canvas.height = shot.height;
+        const context = canvas.getContext('2d');
+        if (!context) throw new Error('No 2D context');
+        context.drawImage(shot, 0, 0);
+        // Sample the center of the source pixel in screenshot pixels.
+        const px = Math.floor(((x + 0.5) * shot.width) / naturalWidth);
+        const py = Math.floor(((y + 0.5) * shot.height) / naturalHeight);
+        return Array.from(context.getImageData(px, py, 1, 1).data);
+      },
+      { data: png.toString('base64'), x, y, naturalWidth, naturalHeight },
+    );
+  }
+
   depthImage(): Locator {
     return this.page.getByTestId('depth-image');
   }

e2e/drivers/types.ts

@@ -57,6 +57,11 @@ export interface UiDriver {
    * (its box follows zoom/pan); use it for geometry, `mainImage()` for pixels.
    */
   canvasImage(): Locator;
+  /**
+   * The RGBA the user sees on the canvas at source pixel (x, y), whatever
+   * the current view draws there (a screenshot, so overlays count too).
+   */
+  visibleCanvasPixel(x: number, y: number): Promise<number[]>;
   depthImage(): Locator;
   /** Every slice thumbnail as displayed (checkerboard-composited), in no particular order. */
   sliceImages(): Locator;

e2e/parallax-maker.spec.ts

@@ -266,7 +266,18 @@ test('painted mask drives three checkerboard candidates, apply, and undo', async
   expect(candidateOutside).toEqual(originalOutside);
 
   expect(await ui.maskCanvasPainted()).toBe(true);
+  // Picking a candidate previews it on the canvas, in place of the slice and
+  // its painted mask, so each one can be inspected before applying it.
+  const nearest = (rgba: number[], palette: number[][]) =>
+    Math.min(...palette.map((color) => Math.max(...color.map((value, i) => Math.abs(rgba[i] - value)))));
+  await ui.selectCandidate(0);
+  await expect
+    .poll(async () => nearest(await ui.visibleCanvasPixel(...inside), [[0, 255, 255], [255, 0, 255]]))
+    .toBeLessThanOrEqual(24);
   await ui.selectCandidate(1);
+  await expect
+    .poll(async () => nearest(await ui.visibleCanvasPixel(...inside), [[255, 128, 0], [0, 64, 255]]))
+    .toBeLessThanOrEqual(24);
   await ui.applyCandidate();
   await expect(ui.log()).toContainText(/Inpainting applied to slice 1/);
   // The applied version has no mask; the canvas must not keep showing the old stroke.

frontend/src/lib/candidatePreview.ts

@@ -0,0 +1,19 @@
+/**
+ * The inpainting candidate being previewed: the one picked in the Inpaint
+ * panel, while it belongs to the selected slice. The canvas shows it in
+ * place of that slice (Slice and Composite views; the Input view's server
+ * display image already is the candidate) and hides the painted mask, so
+ * the filled-in area can be inspected.
+ */
+import type { ProjectView } from './api/types';
+
+export type CandidatePreview = { sliceIndex: number; candidate: number; url: string };
+
+export function candidatePreview(view: ProjectView | null | undefined): CandidatePreview | null {
+  const candidates = view?.inpainting.candidates;
+  const picked = view?.inpainting.selectedCandidate;
+  if (!view || !candidates || picked == null) return null;
+  if (candidates.sliceIndex !== view.selectedSlice) return null;
+  const image = candidates.images[picked];
+  return image ? { sliceIndex: candidates.sliceIndex, candidate: picked, url: image.url } : null;
+}

frontend/src/lib/components/canvas/MaskCanvas.svelte

@@ -21,6 +21,7 @@
    */
   import { projectStore } from '../../state/project.svelte';
   import { uiStore } from '../../state/ui.svelte';
+  import { candidatePreview } from '../../candidatePreview';
   import { isBusy } from '../../state/busy.svelte';
   import { logStore } from '../../state/logs.svelte';
   import { canvasSaveStore } from '../../state/canvas.svelte';
@@ -59,8 +60,11 @@
 
   // Painting is the Brush tool's job; the painted mask shows only in the
   // views it is drawn over (the selected slice, or the working image).
-  const interactiveNow = $derived(uiStore.tool === 'brush');
-  const visibleNow = $derived(uiStore.view === 'slice' || uiStore.view === 'input');
+  // While a picked candidate is previewed, the mask steps aside so the
+  // filled-in area can be inspected (and can't be painted over unseen).
+  const previewing = $derived(candidatePreview(projectStore.view) !== null);
+  const interactiveNow = $derived(uiStore.tool === 'brush' && !previewing);
+  const visibleNow = $derived((uiStore.view === 'slice' || uiStore.view === 'input') && !previewing);
 
   // The brush preview is only meaningful while painting is possible; drop
   // it as soon as another tool is picked rather than leaving a stale circle

frontend/src/lib/components/inpainting/InpaintPanel.svelte

@@ -12,6 +12,7 @@
   import { uiStore } from '../../state/ui.svelte';
   import * as workflow from '../../workflow';
   import { registerInpaintActions } from '../../shortcuts';
+  import { candidatePreview } from '../../candidatePreview';
   import JobCard from '../feedback/JobCard.svelte';
 
   const view = $derived(projectStore.view);
@@ -105,6 +106,8 @@
   }
 
   const hasCandidates = $derived((candidates?.images.length ?? 0) > 0);
+  // The picked candidate the canvas previews (only while it belongs to the selected slice).
+  const preview = $derived(candidatePreview(view));
   const hasMask = $derived(!!view?.slices.find((s) => s.index === selectedSlice)?.mask);
 
   // Sub-step progress: 1 paint the holes, 2 describe, 3 generate and pick.
@@ -297,6 +300,11 @@
         <p class="faint">Candidates appear here after you generate.</p>
       {/each}
     </div>
+    {#if preview}
+      <p class="faint" data-testid="candidate-preview-hint">
+        Candidate {preview.candidate + 1} is shown on the canvas. Click it again to go back to the mask.
+      </p>
+    {/if}
     <button
       type="button"
       class="btn"

frontend/src/lib/components/inpainting/InpaintPanel.test.ts

@@ -295,4 +295,29 @@ describe('InpaintPanel', () => {
       expect(fetchMock.mock.calls.some(([url]) => String(url).endsWith('/selection'))).toBe(false);
     });
   });
+
+  describe('candidate preview hint', () => {
+    const withPick = (sliceIndex: number) =>
+      makeView({
+        selectedSlice: 1,
+        inpainting: {
+          ...makeView().inpainting,
+          candidates: { generationId: 'gen', sliceIndex, images: [{ url: '/c0' }, { url: '/c1' }, { url: '/c2' }] },
+          selectedCandidate: 1,
+        },
+      });
+
+    it('says which candidate the canvas shows', () => {
+      projectStore.applyView(withPick(1));
+      render(InpaintPanel);
+      expect(screen.getByTestId('candidate-preview-hint')).toHaveTextContent('Candidate 2 is shown on the canvas');
+    });
+
+    it('stays hidden when the candidates belong to another slice', () => {
+      projectStore.applyView(withPick(0));
+      render(InpaintPanel);
+      expect(screen.queryByTestId('candidate-preview-hint')).toBeNull();
+    });
+  });
 });
+

frontend/src/lib/components/upload/InputImagePanel.svelte

@@ -10,6 +10,7 @@
    * selected slice's highlight, or the latest camera render). Depth, Slice
    * and Composite are drawn here from the project's assets.
    */
+  import { candidatePreview } from '../../candidatePreview';
   import ImageUp from '@lucide/svelte/icons/image-up';
   import { projectStore } from '../../state/project.svelte';
   import { uiStore } from '../../state/ui.svelte';
@@ -82,6 +83,10 @@
     ),
   );
   const showsMain = $derived(uiStore.view === 'input' || uiStore.view === 'parallax');
+  // A picked inpainting candidate stands in for its slice on the canvas.
+  const preview = $derived(candidatePreview(projectStore.view));
+  const sliceUrl = (slice: { index: number; image: { url: string } }): string =>
+    preview && preview.sliceIndex === slice.index ? preview.url : slice.image.url;
 
   /**
    * A click on the display image with the Segment tool selects by object
@@ -299,14 +304,21 @@
         <img class="layer" data-testid="view-depth-image" alt="Depth map" src={projectStore.view.assets.depth.url} draggable="false" />
       {:else if uiStore.view === 'slice'}
         {#if selectedSlice}
-          <img class="layer" data-testid="view-slice-image" alt={`image_slice_${selectedSlice.index}`} src={selectedSlice.image.url} draggable="false" />
+          <img
+            class="layer"
+            data-testid="view-slice-image"
+            data-candidate={preview?.sliceIndex === selectedSlice.index ? preview.candidate : undefined}
+            alt={`image_slice_${selectedSlice.index}`}
+            src={sliceUrl(selectedSlice)}
+            draggable="false"
+          />
         {:else}
           <div class="layer placeholder" data-testid="view-slice-empty">Select a layer to see it on its own.</div>
         {/if}
       {:else if uiStore.view === 'composite'}
         <div class="layer" data-testid="view-composite-layers">
           {#each compositeLayers as slice (slice.index)}
-            <img class="layer" alt="" src={slice.image.url} draggable="false" />
+            <img class="layer" alt="" src={sliceUrl(slice)} draggable="false" />
           {/each}
         </div>
       {/if}

frontend/src/lib/components/upload/InputImagePanel.test.ts

@@ -7,7 +7,7 @@ import { jobStore } from '../../state/jobs.svelte';
 import { logStore } from '../../state/logs.svelte';
 import { uiStore } from '../../state/ui.svelte';
 import { viewportStore } from '../../state/viewport.svelte';
-import type { ProjectView } from '../../api/types';
+import type { ProjectView, SliceView } from '../../api/types';
 
 function jsonResponse(status: number, body: unknown): Response {
   return new Response(JSON.stringify(body), {
@@ -400,4 +400,59 @@ describe('InputImagePanel', () => {
       expect(JSON.parse(fetchMock.mock.calls[0][1]!.body as string).camera.horizonRow).toBe(110);
     });
   });
+
+  describe('inpainting candidate preview', () => {
+    const slice = (index: number) =>
+      ({ index, depth: 10 * index, image: { url: `/slice-${index}` }, thumbnail: { url: `/t-${index}` } }) as unknown as SliceView;
+    const withCandidates = (selectedCandidate: number | null, selectedSlice = 1): ProjectView =>
+      makeView({
+        slices: [slice(0), slice(1)],
+        selectedSlice,
+        inpainting: {
+          ...makeView().inpainting,
+          candidates: {
+            generationId: 'gen',
+            sliceIndex: 1,
+            images: [{ url: '/cand-0' }, { url: '/cand-1' }, { url: '/cand-2' }],
+          },
+          selectedCandidate,
+        },
+      });
+
+    beforeEach(() => uiStore.setView('slice'));
+
+    it('shows the picked candidate in place of the slice and hides the mask', () => {
+      projectStore.applyView(withCandidates(2));
+      render(InputImagePanel);
+      expect(screen.getByTestId('view-slice-image')).toHaveAttribute('src', '/cand-2');
+      expect(screen.getByTestId('view-slice-image')).toHaveAttribute('data-candidate', '2');
+      expect(screen.getByTestId('mask-canvas')).toHaveClass('hidden');
+    });
+
+    it('shows the slice and its mask again when no candidate is picked', () => {
+      projectStore.applyView(withCandidates(2));
+      render(InputImagePanel);
+      projectStore.applyView(withCandidates(null));
+      flushSync();
+      expect(screen.getByTestId('view-slice-image')).toHaveAttribute('src', '/slice-1');
+      expect(screen.getByTestId('mask-canvas')).not.toHaveClass('hidden');
+    });
+
+    it("ignores candidates of a slice that isn't selected", () => {
+      projectStore.applyView(withCandidates(0, 0));
+      render(InputImagePanel);
+      expect(screen.getByTestId('view-slice-image')).toHaveAttribute('src', '/slice-0');
+      expect(screen.getByTestId('mask-canvas')).not.toHaveClass('hidden');
+    });
+
+    it('swaps the candidate into the composite view', () => {
+      uiStore.setView('composite');
+      projectStore.applyView(withCandidates(1));
+      render(InputImagePanel);
+      const sources = [...screen.getByTestId('view-composite-layers').querySelectorAll('img')].map((img) =>
+        img.getAttribute('src'),
+      );
+      expect(sources).toEqual(['/slice-0', '/cand-1']);
+    });
+  });
 });