diff --git a/.github/workflow-scripts/__tests__/visual-compare-test.js b/.github/workflow-scripts/__tests__/visual-compare-test.js index 8c74326245fc..17f5bc51e68c 100644 --- a/.github/workflow-scripts/__tests__/visual-compare-test.js +++ b/.github/workflow-scripts/__tests__/visual-compare-test.js @@ -11,6 +11,7 @@ const {compareCapture, compareImages, policyFor} = require('../visual/compare'); const { comparePlatform, + platformConclusion, resolveContext, stageBaseline, } = require('../visual/compare-run'); @@ -389,6 +390,16 @@ describe('visual comparison', () => { }); }); + describe('platformConclusion', () => { + test.each([ + ['passes without differences', [{status: 'unchanged'}], null, 'success'], + ['is neutral with differences', [{status: 'changed'}], null, 'neutral'], + ['is neutral without a comparison', [], {title: 'no capture'}, 'neutral'], + ])('%s', (_, results, error, conclusion) => { + expect(platformConclusion({results, error})).toBe(conclusion); + }); + }); + describe('resolveContext', () => { const ENV = { CAPTURE_HEAD_SHA: SHA_A, diff --git a/.github/workflow-scripts/visual/compare-run.js b/.github/workflow-scripts/visual/compare-run.js index 0e5b5b3955da..57cd90f1f097 100644 --- a/.github/workflow-scripts/visual/compare-run.js +++ b/.github/workflow-scripts/visual/compare-run.js @@ -367,6 +367,17 @@ async function prepare() { } } +/** + * A platform check reports what it found without failing: differences are + * neutral, and the approval gate is the only check that fails until they're + * approved, so an approved PR has no failing visual checks. + */ +function platformConclusion(platform) { + return platform.error == null && !summarize(platform.results).hasDifferences + ? 'success' + : 'neutral'; +} + async function report() { const github = createGitHubClient(); const outDir = requireEnv('OUT_DIR'); @@ -402,16 +413,10 @@ async function report() { `${prefix}/${platform.platform}/${kind}/${id}.png`, ); const output = renderCheckOutput({...platform, imageUrl}); - const {hasDifferences} = summarize(platform.results); const checkRun = await github.createCheckRun({ name: `${CHECK_NAME_PREFIX} (${platform.platform})`, headSha: context.headSha, - conclusion: - platform.error != null - ? 'neutral' - : hasDifferences - ? 'failure' - : 'success', + conclusion: platformConclusion(platform), output, detailsUrl, }); @@ -511,4 +516,9 @@ if (require.main === module) { }); } -module.exports = {comparePlatform, resolveContext, stageBaseline}; +module.exports = { + comparePlatform, + platformConclusion, + resolveContext, + stageBaseline, +}; diff --git a/packages/rn-tester/visual-tests/README.md b/packages/rn-tester/visual-tests/README.md index 85dc4e9d170d..81c626864ab0 100644 --- a/packages/rn-tester/visual-tests/README.md +++ b/packages/rn-tester/visual-tests/README.md @@ -7,7 +7,7 @@ Each test is a [Maestro](https://maestro.dev) flow in `flows/`. Maestro only ope ## How it runs in CI 1. Test All builds RNTester Release for Android and iOS. Once a build is ready, the `test_visual_android_rntester` and `test_visual_ios_rntester` jobs (`.github/workflows/visual-capture-rntester.yml`) run every flow in this directory on one canonical device per platform: an API 35 Pixel 7 emulator and an iPhone 17 Pro simulator on iOS 26.2. Any screenshot that differs from the baseline is retaken once, and the retake is kept if it matches, so a notification or other system UI that briefly covers the app doesn't cause a failure. A new test is taken twice too, and if the two captures differ, the report marks it unstable and it isn't published as the baseline. They upload the screenshots and a manifest of the device environment as `visual-capture-android` and `visual-capture-ios` artifacts. -2. When Test All completes, Visual Compare RNTester (`.github/workflows/visual-compare-rntester.yml`) compares the screenshots with the baseline and reports a `Visual regression (android)` and `Visual regression (ios)` check and a `Visual regression approval` gate, plus a single PR comment with the expected, captured and diff images for anything that changed. +2. When Test All completes, Visual Compare RNTester (`.github/workflows/visual-compare-rntester.yml`) compares the screenshots with the baseline and reports a `Visual regression (android)` and `Visual regression (ios)` check, which are neutral when there are differences, and a `Visual regression approval` gate, which fails until a maintainer approves them, plus a single PR comment with the expected, captured and diff images for anything that changed. 3. On pushes to `main`, Visual Compare RNTester also publishes the captures as the baseline for that commit. There's no separate workflow for this: the same run publishes them once it has compared them. Only `main` and pull requests into it are compared. Stable branches have no baselines, so Visual Compare RNTester skips them.