From 40e45098244c1cc7e61d6512cea12be03b78e996 Mon Sep 17 00:00:00 2001 From: Rob Hogan <2590098+robhogan@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:10:56 +0000 Subject: [PATCH] Make the visual platform checks neutral on differences The `Visual regression (android)` and `(ios)` checks failed on any difference, and only the `Visual regression approval` gate passes once a maintainer approves them, so an approved PR still showed two failing checks. The platform checks now report differences as neutral, and the gate is the only visual check that fails until they're approved. Changelog: [Internal] --- .../__tests__/visual-compare-test.js | 11 ++++++++ .../workflow-scripts/visual/compare-run.js | 26 +++++++++++++------ packages/rn-tester/visual-tests/README.md | 2 +- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/.github/workflow-scripts/__tests__/visual-compare-test.js b/.github/workflow-scripts/__tests__/visual-compare-test.js index 8c74326245fc..17f5bc51e68c 100644 --- a/.github/workflow-scripts/__tests__/visual-compare-test.js +++ b/.github/workflow-scripts/__tests__/visual-compare-test.js @@ -11,6 +11,7 @@ const {compareCapture, compareImages, policyFor} = require('../visual/compare'); const { comparePlatform, + platformConclusion, resolveContext, stageBaseline, } = require('../visual/compare-run'); @@ -389,6 +390,16 @@ describe('visual comparison', () => { }); }); + describe('platformConclusion', () => { + test.each([ + ['passes without differences', [{status: 'unchanged'}], null, 'success'], + ['is neutral with differences', [{status: 'changed'}], null, 'neutral'], + ['is neutral without a comparison', [], {title: 'no capture'}, 'neutral'], + ])('%s', (_, results, error, conclusion) => { + expect(platformConclusion({results, error})).toBe(conclusion); + }); + }); + describe('resolveContext', () => { const ENV = { CAPTURE_HEAD_SHA: SHA_A, diff --git a/.github/workflow-scripts/visual/compare-run.js b/.github/workflow-scripts/visual/compare-run.js index 0e5b5b3955da..57cd90f1f097 100644 --- a/.github/workflow-scripts/visual/compare-run.js +++ b/.github/workflow-scripts/visual/compare-run.js @@ -367,6 +367,17 @@ async function prepare() { } } +/** + * A platform check reports what it found without failing: differences are + * neutral, and the approval gate is the only check that fails until they're + * approved, so an approved PR has no failing visual checks. + */ +function platformConclusion(platform) { + return platform.error == null && !summarize(platform.results).hasDifferences + ? 'success' + : 'neutral'; +} + async function report() { const github = createGitHubClient(); const outDir = requireEnv('OUT_DIR'); @@ -402,16 +413,10 @@ async function report() { `${prefix}/${platform.platform}/${kind}/${id}.png`, ); const output = renderCheckOutput({...platform, imageUrl}); - const {hasDifferences} = summarize(platform.results); const checkRun = await github.createCheckRun({ name: `${CHECK_NAME_PREFIX} (${platform.platform})`, headSha: context.headSha, - conclusion: - platform.error != null - ? 'neutral' - : hasDifferences - ? 'failure' - : 'success', + conclusion: platformConclusion(platform), output, detailsUrl, }); @@ -511,4 +516,9 @@ if (require.main === module) { }); } -module.exports = {comparePlatform, resolveContext, stageBaseline}; +module.exports = { + comparePlatform, + platformConclusion, + resolveContext, + stageBaseline, +}; diff --git a/packages/rn-tester/visual-tests/README.md b/packages/rn-tester/visual-tests/README.md index 85dc4e9d170d..81c626864ab0 100644 --- a/packages/rn-tester/visual-tests/README.md +++ b/packages/rn-tester/visual-tests/README.md @@ -7,7 +7,7 @@ Each test is a [Maestro](https://maestro.dev) flow in `flows/`. Maestro only ope ## How it runs in CI 1. Test All builds RNTester Release for Android and iOS. Once a build is ready, the `test_visual_android_rntester` and `test_visual_ios_rntester` jobs (`.github/workflows/visual-capture-rntester.yml`) run every flow in this directory on one canonical device per platform: an API 35 Pixel 7 emulator and an iPhone 17 Pro simulator on iOS 26.2. Any screenshot that differs from the baseline is retaken once, and the retake is kept if it matches, so a notification or other system UI that briefly covers the app doesn't cause a failure. A new test is taken twice too, and if the two captures differ, the report marks it unstable and it isn't published as the baseline. They upload the screenshots and a manifest of the device environment as `visual-capture-android` and `visual-capture-ios` artifacts. -2. When Test All completes, Visual Compare RNTester (`.github/workflows/visual-compare-rntester.yml`) compares the screenshots with the baseline and reports a `Visual regression (android)` and `Visual regression (ios)` check and a `Visual regression approval` gate, plus a single PR comment with the expected, captured and diff images for anything that changed. +2. When Test All completes, Visual Compare RNTester (`.github/workflows/visual-compare-rntester.yml`) compares the screenshots with the baseline and reports a `Visual regression (android)` and `Visual regression (ios)` check, which are neutral when there are differences, and a `Visual regression approval` gate, which fails until a maintainer approves them, plus a single PR comment with the expected, captured and diff images for anything that changed. 3. On pushes to `main`, Visual Compare RNTester also publishes the captures as the baseline for that commit. There's no separate workflow for this: the same run publishes them once it has compared them. Only `main` and pull requests into it are compared. Stable branches have no baselines, so Visual Compare RNTester skips them.