chore: add playwright

.github/workflows/e2e.yml

@@ -26,17 +26,16 @@ jobs:
           composer install --no-dev --prefer-dist --no-progress --no-suggest
           yarn run build
       - name: Setup Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
-          node-version: 16
-      - name: Install testing env
-        run: bash ./bin/e2e-env.sh
-      - name: Run Cypress tests
-        env:
-          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-          PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: true
-        uses: cypress-io/github-action@v6
+          node-version: 18
+          cache: "npm"
+      - name: Run E2E
+        run: npm run ci:e2e
+      - name: Upload artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v3
         with:
-          working-directory: ./e2e-tests
-          env: host=localhost,port=8080
-          browser: chrome
+          name: e2e-artifacts
+          path: ./artifacts
+          retention-days: 1

.gitignore

@@ -8,4 +8,5 @@ languages/*.pot
 **/build/*
 **/.DS_Store
 .phpunit.result.cache
-.vscode
+.vscode
+/e2e-tests/artifacts

e2e-tests/.prettierrc.json

@@ -1,6 +1,6 @@
 {
 	"trailingComma": "all",
-	"tabWidth": 2,
+	"tabWidth": 4,
 	"semi": true,
 	"singleQuote": true,
 	"printWidth": 100,

e2e-tests/.wp-env.json

@@ -0,0 +1,4 @@
+{
+    "plugins": [ ".." ],
+    "phpVersion": "8.1"
+}

e2e-tests/README.md

@@ -1,6 +1,26 @@
-### How to run cypress tests locally 
+### How to run Playwright E2E tests locally
 
-* First you will need to setup the environment on your local machine and have wp-cli installed. Assuming that your local url is http://neve.test you can `CYPRESS_BASE_URL=http://neve.test bash bin/envs/local.sh` or if you want to install a specific environment you can use `CYPRESS_BASE_URL=http://neve.test bash bin/envs/local.sh amp` where amp is the environment inside `bin/envs/amp`
-
-* Than you can move to e2e-tests folder and run `yarn install --frozen-lockfile` to install the dependencies.
-* After this you will have to open cypress with the proper base url, as `CYPRESS_BASE_URL=http://neve.test  npm run cypress:open` and run the spec that you want.
+-   First, ensure you have Node.js installed on your system.
+
+-   Install dependencies by running in this folder:
+
+    ```bash
+    npm install
+    ```
+
+-   Start the WordPress test environment:
+
+    ```bash
+    npm run wp-env start
+    ```
+
+-   To run the tests, you have several options:
+
+    -   Run all tests: `npm run test:playwright`
+    -   Run tests in debug mode: `npm run test:playwright:debug`
+    -   Run tests with UI mode: `npm run test:playwright:ui`
+
+-   For additional test options, you can run:
+    ```bash
+    npm run test:playwright:help
+    ```

e2e-tests/config/flaky-tests-reporter.ts

@@ -0,0 +1,85 @@
+/**
+ * A **flaky** test is defined as a test which passed after auto-retrying.
+ * - By default, all tests run once if they pass.
+ * - If a test fails, it will automatically re-run at most 2 times.
+ * - If it pass after retrying (below 2 times), then it's marked as **flaky**
+ *   but displayed as **passed** in the original test suite.
+ * - If it fail all 3 times, then it's a **failed** test.
+ */
+/**
+ * External dependencies
+ */
+import fs from 'fs';
+import type { Reporter, TestCase, TestResult } from '@playwright/test/reporter';
+import filenamify from 'filenamify';
+
+type FormattedTestResult = Omit< TestResult, 'steps' >;
+
+// Remove "steps" to prevent stringify circular structure.
+function formatTestResult( testResult: TestResult ): FormattedTestResult {
+	const result = { ...testResult, steps: undefined };
+	delete result.steps;
+	return result;
+}
+
+class FlakyTestsReporter implements Reporter {
+	failingTestCaseResults = new Map< string, FormattedTestResult[] >();
+
+	onBegin() {
+		try {
+			fs.mkdirSync( 'flaky-tests' );
+		} catch ( err ) {
+			if (
+				err instanceof Error &&
+				( err as NodeJS.ErrnoException ).code === 'EEXIST'
+			) {
+				// Ignore the error if the directory already exists.
+			} else {
+				throw err;
+			}
+		}
+	}
+
+	onTestEnd( test: TestCase, testCaseResult: TestResult ) {
+		const testPath = test.location.file;
+		const testTitle = test.title;
+
+		switch ( test.outcome() ) {
+			case 'unexpected': {
+				if ( ! this.failingTestCaseResults.has( testTitle ) ) {
+					this.failingTestCaseResults.set( testTitle, [] );
+				}
+				this.failingTestCaseResults
+					.get( testTitle )!
+					.push( formatTestResult( testCaseResult ) );
+				break;
+			}
+			case 'flaky': {
+				fs.writeFileSync(
+					`flaky-tests/${ filenamify( testTitle ) }.json`,
+					JSON.stringify( {
+						version: 1,
+						runner: '@playwright/test',
+						title: testTitle,
+						path: testPath,
+						results: this.failingTestCaseResults.get( testTitle ),
+					} ),
+					'utf-8'
+				);
+				break;
+			}
+			default:
+				break;
+		}
+	}
+
+	onEnd() {
+		this.failingTestCaseResults.clear();
+	}
+
+	printsToStdio() {
+		return false;
+	}
+}
+
+module.exports = FlakyTestsReporter;

e2e-tests/config/global-setup.ts

@@ -0,0 +1,44 @@
+/**
+ * External dependencies
+ */
+import { request } from '@playwright/test';
+import type { FullConfig } from '@playwright/test';
+
+/**
+ * WordPress dependencies
+ */
+import { RequestUtils } from '@wordpress/e2e-test-utils-playwright';
+
+async function globalSetup( config: FullConfig ) {
+	const { storageState, baseURL } = config.projects[ 0 ].use;
+	const storageStatePath =
+		typeof storageState === 'string' ? storageState : undefined;
+
+	const requestContext = await request.newContext( {
+		baseURL,
+	} );
+
+	const requestUtils = new RequestUtils( requestContext, {
+		storageStatePath,
+	} );
+
+	// Authenticate and save the storageState to disk.
+	await requestUtils.setupRest();
+
+	// Reset the test environment before running the tests.
+	await Promise.all( [
+		// requestUtils.activateTheme( 'twentytwentyone' ),
+		// // Disable this test plugin as it's conflicting with some of the tests.
+		// // We already have reduced motion enabled and Playwright will wait for most of the animations anyway.
+		// requestUtils.deactivatePlugin(
+		// 	'gutenberg-test-plugin-disables-the-css-animations'
+		// ),
+		requestUtils.deleteAllPosts(),
+		requestUtils.deleteAllBlocks(),
+		requestUtils.resetPreferences(),
+	] );
+
+	await requestContext.dispose();
+}
+
+export default globalSetup;