test(wtr): run in multiple browsers and with API_VERSION=66

.github/workflows/web-test-runner.yml

@@ -19,7 +19,6 @@ env:
 
 jobs:
     # TODO: upload result artifacts
-    # TODO: make it saucy 🥫
     integration-tests:
         name: Integration tests (${{ matrix.shadow_mode }} shadow)
         strategy:
@@ -47,19 +46,21 @@ jobs:
               run: yarn install --frozen-lockfile
               working-directory: ./
 
-            # - uses: saucelabs/[email protected]
-            #   with:
-            #       username: ${{ secrets.SAUCE_USERNAME }}
-            #       accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-            #       tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-            #       region: us
+            - uses: saucelabs/[email protected]
+              with:
+                  username: ${{ secrets.SAUCE_USERNAME }}
+                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+                  region: us
 
+            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
             - run: yarn test
             - run: API_VERSION=58 yarn test
             - run: API_VERSION=59 yarn test
             - run: API_VERSION=60 yarn test
             - run: API_VERSION=61 yarn test
             - run: API_VERSION=62 yarn test
+            - run: API_VERSION=66 yarn test
             - run: DISABLE_NATIVE_CUSTOM_ELEMENT_LIFECYCLE=1 yarn test || true
             - run: DISABLE_STATIC_CONTENT_OPTIMIZATION=1 yarn test
             - run: ENABLE_ARIA_REFLECTION_GLOBAL_POLYFILL=1 yarn test
@@ -89,13 +90,14 @@ jobs:
               run: yarn install --frozen-lockfile
               working-directory: ./
 
-            # - uses: saucelabs/[email protected]
-            #   with:
-            #       username: ${{ secrets.SAUCE_USERNAME }}
-            #       accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-            #       tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-            #       region: us
+            - uses: saucelabs/[email protected]
+              with:
+                  username: ${{ secrets.SAUCE_USERNAME }}
+                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+                  region: us
 
+            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
             # Synthetic shadow only
             - run: LEGACY_BROWSERS=1 yarn test || true
             - run: FORCE_NATIVE_SHADOW_MODE_FOR_TEST=1 yarn test
@@ -127,12 +129,14 @@ jobs:
               run: yarn install --frozen-lockfile
               working-directory: ./
 
-            # - uses: saucelabs/[email protected]
-            #   with:
-            #       username: ${{ secrets.SAUCE_USERNAME }}
-            #       accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
-            #       tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
-            #       region: us
+            - uses: saucelabs/[email protected]
+              with:
+                  username: ${{ secrets.SAUCE_USERNAME }}
+                  accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
+                  tunnelName: ${{ env.SAUCE_TUNNEL_ID }}
+                  region: us
+
+            - run: yarn exec -- playwright install chrome firefox webkit --with-deps
             - run: ENGINE_SERVER=1 yarn test:hydration
             - run: ENGINE_SERVER=1 SHADOW_MODE_OVERRIDE=synthetic yarn test:hydration
             - run: ENGINE_SERVER=1 NODE_ENV_FOR_TEST=production yarn test:hydration

packages/@lwc/engine-core/src/libs/signal-tracker/index.ts

@@ -44,6 +44,19 @@ export function unsubscribeFromSignals(target: object) {
 
 type CallbackFunction = () => void;
 
+/**
+ * A normalized string representation of an error, because browsers behave differently
+ */
+const errorWithStack = (err: unknown): string => {
+    if (typeof err !== 'object' || err === null) {
+        return String(err);
+    }
+    const stack = 'stack' in err ? String(err.stack) : '';
+    const message = 'message' in err ? String(err.message) : '';
+    const constructor = err.constructor.name;
+    return stack.includes(message) ? stack : `${constructor}: ${message}\n${stack}`;
+};
+
 /**
  * This class is used to keep track of the signals associated to a given object.
  * It is used to prevent the LWC engine from subscribing duplicate callbacks multiple times
@@ -67,9 +80,9 @@ class SignalTracker {
             }
         } catch (err: any) {
             logWarnOnce(
-                `Attempted to subscribe to an object that has the shape of a signal but received the following error: ${
-                    err?.stack ?? err
-                }`
+                `Attempted to subscribe to an object that has the shape of a signal but received the following error: ${errorWithStack(
+                    err
+                )}`
             );
         }
     }
@@ -79,9 +92,9 @@ class SignalTracker {
             this.signalToUnsubscribeMap.forEach((unsubscribe) => unsubscribe());
         } catch (err: any) {
             logWarnOnce(
-                `Attempted to call a signal's unsubscribe callback but received the following error: ${
-                    err?.stack ?? err
-                }`
+                `Attempted to call a signal's unsubscribe callback but received the following error: ${errorWithStack(
+                    err
+                )}`
             );
         }
     }

packages/@lwc/integration-not-karma/configs/hydration.js

@@ -1,5 +1,5 @@
 import * as options from '../helpers/options.js';
-import createConfig from './base.js';
+import createConfig from './shared/base-config.js';
 import hydrationTestPlugin from './plugins/serve-hydration.js';
 
 const SHADOW_MODE = options.SHADOW_MODE_OVERRIDE ?? 'native';

packages/@lwc/integration-not-karma/configs/integration.js

@@ -1,6 +1,6 @@
 import { importMapsPlugin } from '@web/dev-server-import-maps';
 import * as options from '../helpers/options.js';
-import createConfig from './base.js';
+import createConfig from './shared/base-config.js';
 import testPlugin from './plugins/serve-integration.js';
 
 const SHADOW_MODE = options.SHADOW_MODE_OVERRIDE ?? 'synthetic';

packages/@lwc/integration-not-karma/configs/plugins/serve-hydration.js

@@ -1,20 +1,16 @@
 import path from 'node:path';
 import vm from 'node:vm';
 import { fileURLToPath } from 'node:url';
+import { readFileSync } from 'node:fs';
 import { rollup } from 'rollup';
 import lwcRollupPlugin from '@lwc/rollup-plugin';
 import { DISABLE_STATIC_CONTENT_OPTIMIZATION, ENGINE_SERVER } from '../../helpers/options.js';
-/** LWC SSR module to use when server-side rendering components. */
-const lwcSsr = await (ENGINE_SERVER
-    ? // Using import('literal') rather than import(variable) so static analysis tools work
-      import('@lwc/engine-server')
-    : import('@lwc/ssr-runtime'));
 
-lwcSsr.setHooks({
-    sanitizeHtmlContent(content) {
-        return content;
-    },
-});
+/** Code for the LWC SSR module. */
+const LWC_SSR = readFileSync(
+    new URL(import.meta.resolve(ENGINE_SERVER ? '@lwc/engine-server' : '@lwc/ssr-runtime')),
+    'utf8'
+);
 
 const ROOT_DIR = path.join(import.meta.dirname, '../..');
 const COMPONENT_NAME = 'x-main';
@@ -68,16 +64,27 @@ async function compileModule(input, targetSSR, format) {
  */
 async function getSsrMarkup(componentEntrypoint, configPath) {
     const componentIife = await compileModule(componentEntrypoint, !ENGINE_SERVER, 'iife');
-    // To minimize the amount of code in the generated script, ideally we'd do `import Component`
-    // and delegate the bundling to the loader. However, that's complicated to configure and using
-    // imports with vm.Script/vm.Module is still experimental, so we use an IIFE for simplicity.
-    // Additionally, we could import LWC, but the framework requires configuration before each test
-    // (setHooks/setFeatureFlagForTest), so instead we configure it once in the top-level context
-    // and inject it as a global variable.
+    // Ideally, we'd be able to do `import Component` and delegate bundling to the loader. We also
+    // need each import of LWC to be isolated, but by all server-side imports share a global state.
+    // We could solve this with the right `vm.Script`/`vm.Module` setup, but that's complicated and
+    // still experimental. Therefore, we just inline everything.
     const script = new vm.Script(
         `(async () => {
-            const {default: config} = await import('./${configPath}');
-            ${componentIife /* var Component = ... */}
+            // node.js / CommonJS setup
+            const process = { env: ${JSON.stringify(process.env)} };
+            const exports = Object.create(null);
+            const LWC = exports;
+
+            // LWC / test setup
+            ${LWC_SSR};
+            LWC.setHooks({ sanitizeHtmlContent: (v) => v });
+            const { default: config } = await import('./${configPath}');
+            config.requiredFeatureFlags?.forEach(ff => {
+                LWC.setFeatureFlagForTest(ff, true);
+            });
+
+            // Component code
+            ${componentIife};
             return LWC.renderComponent(
                 '${COMPONENT_NAME}',
                 Component,
@@ -87,44 +94,32 @@ async function getSsrMarkup(componentEntrypoint, configPath) {
             );
         })()`,
         {
-            filename: `[SSR] ${configPath}`,
+            filename: `(virtual SSR file for) ${configPath}`,
             importModuleDynamically: vm.constants.USE_MAIN_CONTEXT_DEFAULT_LOADER,
         }
     );
 
-    return await script.runInContext(vm.createContext({ LWC: lwcSsr }));
+    return await script.runInNewContext();
 }
 
 /**
  * Hydration test `index.spec.js` files are actually config files, not spec files.
  * This function wraps those configs in the test code to be executed.
  */
 async function wrapHydrationTest(configPath) {
-    const { default: config } = await import(path.join(ROOT_DIR, configPath));
+    const suiteDir = path.dirname(configPath);
+    const componentEntrypoint = path.join(suiteDir, COMPONENT_ENTRYPOINT);
+    const ssrOutput = await getSsrMarkup(componentEntrypoint, configPath);
 
-    try {
-        config.requiredFeatureFlags?.forEach((featureFlag) => {
-            lwcSsr.setFeatureFlagForTest(featureFlag, true);
-        });
-
-        const suiteDir = path.dirname(configPath);
-        const componentEntrypoint = path.join(suiteDir, COMPONENT_ENTRYPOINT);
-        const ssrOutput = await getSsrMarkup(componentEntrypoint, configPath);
-
-        return `
-        import * as LWC from 'lwc';
-        import { runTest } from '/configs/plugins/test-hydration.js';
-        runTest(
-            '/${configPath}?original=1',
-            '/${componentEntrypoint}',
-            ${JSON.stringify(ssrOutput) /* escape quotes */}
-        );
-        `;
-    } finally {
-        config.requiredFeatureFlags?.forEach((featureFlag) => {
-            lwcSsr.setFeatureFlagForTest(featureFlag, false);
-        });
-    }
+    return `
+    import * as LWC from 'lwc';
+    import { runTest } from '/configs/plugins/test-hydration.js';
+    runTest(
+        '/${configPath}?original=1',
+        '/${componentEntrypoint}',
+        ${JSON.stringify(ssrOutput) /* escape quotes */}
+    );
+    `;
 }
 
 /** @type {import('@web/dev-server-core').Plugin} */

packages/@lwc/integration-not-karma/configs/base.js → packages/@lwc/integration-not-karma/configs/shared/base-config.js

@@ -1,6 +1,7 @@
 import { join } from 'node:path';
 import { LWC_VERSION } from '@lwc/shared';
-import { resolvePathOutsideRoot } from '../helpers/utils.js';
+import { resolvePathOutsideRoot } from '../../helpers/utils.js';
+import { getBrowsers } from './browsers.js';
 
 /**
  * We want to convert from parsed options (true/false) to a `process.env` with only strings.
@@ -18,7 +19,7 @@ const envify = (obj) => {
 const pluck = (obj, keys) => Object.fromEntries(keys.map((k) => [k, obj[k]]));
 const maybeImport = (file, condition) => (condition ? `await import('${file}');` : '');
 
-/** @type {() => import("@web/test-runner").TestRunnerConfig} */
+/** @type {(options: typeof import('../../helpers/options.js')) => import("@web/test-runner").TestRunnerConfig} */
 export default (options) => {
     /** `process.env` to inject into test environment. */
     const env = envify({
@@ -37,13 +38,15 @@ export default (options) => {
     });
 
     return {
+        browsers: getBrowsers(options),
+        browserLogs: false,
         // FIXME: Parallelism breaks tests that rely on focus/requestAnimationFrame, because they often
         // time out before they receive focus. But it also makes the full suite take 3x longer to run...
         // Potential workaround: https://github.com/modernweb-dev/web/issues/2588
         concurrency: 1,
-        browserLogs: false,
+        concurrentBrowsers: 3,
         nodeResolve: true,
-        rootDir: join(import.meta.dirname, '..'),
+        rootDir: join(import.meta.dirname, '../..'),
         plugins: [
             {
                 name: 'lwc-base-plugin',
@@ -64,7 +67,8 @@ export default (options) => {
             },
         ],
         testRunnerHtml: (testFramework) =>
-            `<!DOCTYPE html>
+            `
+        <!DOCTYPE html>
         <html>
           <head>
             <script type="module">
@@ -82,6 +86,7 @@ export default (options) => {
             <script type="module" src="./helpers/setup.js"></script>
             <script type="module" src="${testFramework}"></script>
           </head>
-        </html>`,
+        </html>
+        `,
     };
 };

packages/@lwc/integration-not-karma/configs/shared/browsers.js

@@ -0,0 +1,56 @@
+import { playwrightLauncher } from '@web/test-runner-playwright';
+import { createSauceLabsLauncher } from '@web/test-runner-saucelabs';
+
+/** @type {(options: typeof import('../../helpers/options.js')) => import("@web/test-runner").BrowserLauncher[]} */
+export function getBrowsers(options) {
+    if (options.IS_CI) {
+        if (!options.SAUCE_USERNAME || !options.SAUCE_ACCESS_KEY || !options.SAUCE_TUNNEL_ID) {
+            throw new Error(
+                `SAUCE_USERNAME, SAUCE_ACCESS_KEY, and SAUCE_TUNNEL_ID must be configured in CI`
+            );
+        }
+        const sauceLabsLauncher = createSauceLabsLauncher(
+            {
+                user: options.SAUCE_USERNAME,
+                key: options.SAUCE_ACCESS_KEY,
+            },
+            {
+                tunnelName: options.SAUCE_TUNNEL_ID,
+            }
+        );
+        return [
+            sauceLabsLauncher({
+                browserName: 'chrome',
+                browserVersion: 'latest',
+            }),
+            sauceLabsLauncher({
+                browserName: 'firefox',
+                browserVersion: 'latest',
+            }),
+            sauceLabsLauncher({
+                browserName: 'safari',
+                browserVersion: 'latest',
+                platformName: 'macOS 15', // Update this with new Mac releases
+            }),
+            ...(options.LEGACY_BROWSERS
+                ? [
+                      sauceLabsLauncher({
+                          browserName: 'chrome',
+                          browserVersion: 'latest-2',
+                      }),
+                      sauceLabsLauncher({
+                          browserName: 'safari',
+                          browserVersion: 'latest-2',
+                          platformName: 'macOS 13', // Should be 2 behind latest
+                      }),
+                  ]
+                : []),
+        ];
+    } else {
+        return [
+            playwrightLauncher({ product: 'chromium' }),
+            playwrightLauncher({ product: 'firefox' }),
+            playwrightLauncher({ product: 'webkit' }),
+        ];
+    }
+}

packages/@lwc/integration-not-karma/package.json

@@ -4,6 +4,7 @@
     "version": "8.22.4",
     "type": "module",
     "scripts": {
+        "build": "playwright install || true",
         "start": "web-test-runner --manual",
         "test": "web-test-runner --config configs/integration.js",
         "test:hydration": "web-test-runner --config configs/hydration.js"
@@ -21,7 +22,10 @@
         "@web/dev-server-import-maps": "^0.2.1",
         "@web/dev-server-rollup": "^0.6.4",
         "@web/test-runner": "^0.20.2",
-        "chai": "^6.2.0"
+        "@web/test-runner-playwright": "^0.11.1",
+        "@web/test-runner-saucelabs": "^0.13.0",
+        "chai": "^6.2.0",
+        "playwright": "^1.56.0"
     },
     "volta": {
         "extends": "../../../package.json"