tests: start replacing jest with mocha, starting with unit-cli

.github/workflows/unit.yml

@@ -84,8 +84,9 @@ jobs:
     - run: yarn install --frozen-lockfile --network-timeout 1000000
     - run: yarn build-report
 
-    - name: yarn unit-cli
-      run: yarn unit-cli
+    # - name: yarn unit-cli
+    #   run: yarn unit-cli
+    - run: yarn unit-cli-mocha
     - run: yarn diff:sample-json
 
     # Fail if any changes were written to any source files or generated untracked files (ex, from -GA).

jest.config.js

@@ -10,7 +10,6 @@ module.exports = {
   testEnvironment: 'node',
   testMatch: [
     '**/lighthouse-core/**/*-test.js',
-    '**/lighthouse-cli/**/*-test.js',
     '**/report/**/*-test.js',
     '**/lighthouse-core/test/fraggle-rock/**/*-test-pptr.js',
     '**/treemap/**/*-test.js',

lighthouse-cli/test/cli/bin-test.js

@@ -9,47 +9,42 @@
 
 import fs from 'fs';
 
-import {jest} from '@jest/globals';
+import * as td from 'testdouble';
+import jestMock from 'jest-mock';
 
-import {LH_ROOT} from '../../../root.js';
+import {LH_ROOT, readJson} from '../../../root.js';
 
-const mockRunLighthouse = jest.fn();
-
-jest.unstable_mockModule('../../run.js', () => {
-  return {runLighthouse: mockRunLighthouse};
-});
-
-const mockGetFlags = jest.fn();
-jest.unstable_mockModule('../../cli-flags.js', () => {
-  return {getFlags: mockGetFlags};
-});
-
-const mockAskPermission = jest.fn();
-jest.unstable_mockModule('../../sentry-prompt.js', () => {
-  return {askPermission: mockAskPermission};
-});
-
-const mockSentryInit = jest.fn();
-jest.mock('../../../lighthouse-core/lib/sentry.js', () => {
-  return {init: mockSentryInit};
-});
-
-const mockLoggerSetLevel = jest.fn();
-jest.unstable_mockModule('lighthouse-logger', () => {
-  return {default: {setLevel: mockLoggerSetLevel}};
-});
-
-const mockNotify = jest.fn();
-jest.unstable_mockModule('update-notifier', () => {
-  return {default: () => ({notify: mockNotify})};
-});
+const mockRunLighthouse = jestMock.fn();
+const mockGetFlags = jestMock.fn();
+const mockAskPermission = jestMock.fn();
+const mockSentryInit = jestMock.fn();
+const mockLoggerSetLevel = jestMock.fn();
+const mockNotify = jestMock.fn();
 
 /** @type {import('../../bin.js')} */
 let bin;
 beforeAll(async () => {
+  await td.replaceEsm('../../run.js', {
+    runLighthouse: mockRunLighthouse,
+  });
+  await td.replaceEsm('../../cli-flags.js', {
+    getFlags: mockGetFlags,
+  });
+  await td.replaceEsm('../../sentry-prompt.js', {
+    askPermission: mockAskPermission,
+  });
+  await td.replace('../../../lighthouse-core/lib/sentry.js', {
+    init: mockSentryInit,
+  });
+  await td.replaceEsm('lighthouse-logger', undefined, {setLevel: mockLoggerSetLevel});
+  await td.replaceEsm('update-notifier', undefined, () => ({notify: mockNotify}));
   bin = await import('../../bin.js');
 });
 
+afterAll(async () => {
+  await td.reset();
+});
+
 /** @type {LH.CliFlags} */
 let cliFlags;
 
@@ -186,7 +181,7 @@ describe('CLI bin', function() {
       await bin.begin();
 
       expect(getRunLighthouseArgs()[1]).toMatchObject({
-        precomputedLanternData: (await import(lanternDataFile)).default,
+        precomputedLanternData: readJson(lanternDataFile),
         precomputedLanternDataPath: lanternDataFile,
       });
     });

lighthouse-cli/test/cli/cli-flags-test.js

@@ -10,24 +10,21 @@
 import {strict as assert} from 'assert';
 import fs from 'fs';
 
-import yargs from 'yargs';
-
-import {getFlags} from '../../cli-flags.js';
+import {getFlags, getYargsParser} from '../../cli-flags.js';
 import {LH_ROOT} from '../../../root.js';
 
 describe('CLI flags', function() {
   it('all options should have descriptions', () => {
-    getFlags('chrome://version');
-
+    const parser = getYargsParser();
     // @ts-expect-error - getGroups is private
-    const optionGroups = yargs.getGroups();
+    const optionGroups = parser.getGroups();
     /** @type {string[]} */
     const allOptions = [];
     Object.keys(optionGroups).forEach(key => {
       allOptions.push(...optionGroups[key]);
     });
     // @ts-expect-error - getUsageInstance is private
-    const optionsWithDescriptions = Object.keys(yargs.getUsageInstance().getDescriptions());
+    const optionsWithDescriptions = Object.keys(parser.getUsageInstance().getDescriptions());
 
     allOptions.forEach(opt => {
       assert.ok(optionsWithDescriptions.includes(opt), `cli option '${opt}' has no description`);

lighthouse-cli/test/cli/printer-test.js

@@ -11,7 +11,9 @@ import {strict as assert} from 'assert';
 import fs from 'fs';
 
 import * as Printer from '../../printer.js';
-import sampleResults from '../../../lighthouse-core/test/results/sample_v2.json';
+import {readJson} from '../../../root.js';
+
+const sampleResults = readJson('lighthouse-core/test/results/sample_v2.json');
 
 describe('Printer', () => {
   it('accepts valid output paths', () => {

lighthouse-cli/test/cli/run-test.js

@@ -33,6 +33,11 @@ describe('CLI run', function() {
     let fileResults;
 
     beforeAll(async () => {
+      fs.symlinkSync(
+        `${LH_ROOT}/lighthouse-core/test/fixtures/config-plugins/lighthouse-plugin-simple`,
+        `${LH_ROOT}/lighthouse-plugin-simple`
+      );
+
       const url = 'http://localhost:10200/dobetterweb/dbw_tester.html';
       // eslint-disable-next-line max-len
       const samplev2ArtifactsPath = LH_ROOT + '/lighthouse-core/test/results/artifacts/';
@@ -41,8 +46,6 @@ describe('CLI run', function() {
       const flags = getFlags([
         '--output=json',
         `--output-path=${filename}`,
-        // Jest allows us to resolve this module with no setup.
-        // https://github.com/GoogleChrome/lighthouse/pull/13045#discussion_r708690607
         '--plugins=lighthouse-plugin-simple',
         // Use sample artifacts to avoid gathering during a unit test.
         `--audit-mode=${samplev2ArtifactsPath}`,
@@ -62,6 +65,7 @@ describe('CLI run', function() {
 
     afterAll(() => {
       fs.unlinkSync(filename);
+      fs.unlinkSync(`${LH_ROOT}/lighthouse-plugin-simple`);
     });
 
     it('returns results that match the saved results', () => {

lighthouse-core/test/gather/driver/execution-context-test.js

@@ -85,7 +85,7 @@ describe('ExecutionContext', () => {
     expect(executionContext.getContextId()).toEqual(undefined);
   });
 
-  it.todo('should cache native objects in page');
+  it('TODO: should cache native objects in page');
 });
 
 describe('.evaluateAsync', () => {

lighthouse-core/test/mocha-setup.js

@@ -0,0 +1,148 @@
+/**
+ * @license Copyright 2021 The Lighthouse Authors. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
+ */
+'use strict';
+
+/**
+ * @fileoverview
+ * 1) sets global.expect
+ * 2) configures the mocha test runner to use jest-snapshot
+ */
+
+const path = require('path');
+const expect = require('expect');
+const {SnapshotState, toMatchSnapshot, toMatchInlineSnapshot} = require('jest-snapshot');
+
+/** @type {Map<string, SnapshotState['prototype']>} */
+const snapshotStatesByTestFile = new Map();
+
+/**
+ * @param {string} testFile
+ */
+function getSnapshotState(testFile) {
+  // For every test file, persist the same snapshot state object so there is
+  // not a read/write per snapshot access/change, but one per file.
+  let snapshotState = snapshotStatesByTestFile.get(testFile);
+  if (snapshotState) return snapshotState;
+
+  const snapshotDir = path.join(path.dirname(testFile), '__snapshots__');
+  const snapshotFile = path.join(snapshotDir, path.basename(testFile) + '.snap');
+  snapshotState = new SnapshotState(snapshotFile, {
+    updateSnapshot: process.env.SNAPSHOT_UPDATE ? 'all' : 'new',
+    prettierPath: '',
+    snapshotFormat: {},
+  });
+  snapshotStatesByTestFile.set(testFile, snapshotState);
+  return snapshotState;
+}
+
+/**
+ * @param {Mocha.Test} test
+ * @return {string}
+ */
+function makeTestTitle(test) {
+  /** @type {Mocha.Test | Mocha.Suite} */
+  let next = test;
+  const title = [];
+
+  for (;;) {
+    if (!next.parent) {
+      break;
+    }
+
+    title.push(next.title);
+    next = next.parent;
+  }
+
+  return title.reverse().join(' ');
+}
+
+expect.extend({
+  /**
+   * @param {any} actual
+   */
+  toMatchSnapshot(actual) {
+    const test = mochaCurrentTest;
+    if (!test.file) throw new Error('unexpected value');
+
+    const title = makeTestTitle(test);
+    const snapshotState = getSnapshotState(test.file);
+    /** @type {import('jest-snapshot/build/types').Context} */
+    // @ts-expect-error - this is enough for snapshots to work.
+    const context = {snapshotState, currentTestName: title};
+    const matcher = toMatchSnapshot.bind(context);
+
+    return matcher(actual);
+  },
+  /**
+   * @param {any} actual
+   * @param {any} expected
+   */
+  toMatchInlineSnapshot(actual, expected) {
+    const test = mochaCurrentTest;
+    if (!test.file) throw new Error('unexpected value');
+
+    const title = makeTestTitle(test);
+    const snapshotState = getSnapshotState(test.file);
+    /** @type {import('jest-snapshot/build/types').Context} */
+    // @ts-expect-error - this is enough for snapshots to work.
+    const context = {snapshotState, currentTestName: title};
+    const matcher = toMatchInlineSnapshot.bind(context);
+
+    return matcher(actual, expected);
+  },
+});
+
+// @ts-expect-error
+global.expect = expect;
+
+/**
+ * @param {Mocha.HookFunction} mochaFn
+ * @return {jest.Lifecycle}
+ */
+const makeFn = (mochaFn) => (fn, timeout) => {
+  mochaFn(function() {
+    // eslint-disable-next-line no-invalid-this
+    if (timeout !== undefined) this.timeout(timeout);
+
+    /** @type {jest.DoneCallback} */
+    const cb = () => {};
+    cb.fail = (error) => {
+      throw new Error(typeof error === 'string' ? error : error?.message);
+    };
+    return fn(cb);
+  });
+};
+
+const {before, after} = require('mocha');
+global.beforeAll = makeFn(before);
+global.afterAll = makeFn(after);
+
+/** @type {Mocha.Test} */
+let mochaCurrentTest;
+module.exports = {
+  mochaHooks: {
+    /** @this {Mocha.Context} */
+    beforeEach() {
+      if (!this.currentTest) throw new Error('unexpected value');
+
+      // Needed so `expect` extension method can access information about the current test.
+      mochaCurrentTest = this.currentTest;
+    },
+    afterAll() {
+      for (const snapshotState of snapshotStatesByTestFile.values()) {
+        // Jest adds `file://` to inline snapshot paths, and uses its own fs module to read things,
+        // falling back to fs.readFileSync if not defined. node `fs` does not support
+        // protocols in the path specifier, so we remove it here.
+        // @ts-expect-error - private property.
+        for (const snapshot of snapshotState._inlineSnapshots) {
+          snapshot.frame.file = snapshot.frame.file.replace('file://', '');
+        }
+
+        snapshotState.save();
+      }
+    },
+  },
+};

package.json

@@ -36,6 +36,8 @@
     "debug": "node --inspect-brk ./lighthouse-cli/index.js",
     "start": "yarn build-report --standalone && node ./lighthouse-cli/index.js",
     "jest": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js",
+    "____comment": "echo https://github.com/cucumber/cucumber-js/issues/1844",
+    "mocha": "cp node_modules/mocha/bin/mocha node_modules/mocha/bin/mocha.js && node --loader=testdouble node_modules/mocha/bin/mocha.js --require=lighthouse-core/test/mocha-setup.js",
     "test": "yarn diff:sample-json && yarn lint --quiet && yarn unit && yarn type-check",
     "test-bundle": "yarn smoke --runner bundle -j=1 --retries=2 --invert-match forms",
     "test-clients": "yarn jest \"$PWD/clients/\" && yarn jest --testMatch=\"**/clients/test/**/*-test-pptr.js\"",
@@ -47,12 +49,14 @@
     "test-proto": "yarn compile-proto && yarn build-proto-roundtrip",
     "unit-core": "yarn jest \"lighthouse-core\"",
     "unit-cli": "yarn jest \"lighthouse-cli/\"",
+    "unit-cli-mocha": "yarn mocha \"lighthouse-cli/test/**/*-test.js\"",
     "unit-report": "yarn jest \"report/\"",
     "unit-treemap": "yarn jest \"treemap/.*-test.js\"",
     "unit-viewer": "yarn jest \"viewer/.*-test.js\"",
     "unit-flow": "yarn jest \"flow-report/.*-test.[tj]s[x]?\"",
     "unit": "yarn jest",
-    "unit:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run jest --ci .",
+    "unit-mocha": "yarn mocha",
+    "unit:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run jest --ci . && npm run mocha",
     "core-unit": "yarn unit-core",
     "cli-unit": "yarn unit-cli",
     "viewer-unit": "yarn unit-viewer",
@@ -119,8 +123,10 @@
     "@types/lodash.get": "^4.4.6",
     "@types/lodash.isequal": "^4.5.2",
     "@types/lodash.set": "^4.3.6",
+    "@types/mocha": "^9.0.0",
     "@types/node": "*",
     "@types/pako": "^1.0.1",
+    "@types/proxyquire": "^1.3.28",
     "@types/raven": "^2.5.1",
     "@types/resize-observer-browser": "^0.1.1",
     "@types/semver": "^5.5.0",
@@ -156,12 +162,14 @@
     "idb-keyval": "2.2.0",
     "intl-messageformat-parser": "^1.8.1",
     "jest": "27.1.1",
+    "jest-mock": "^27.3.0",
     "jsdom": "^12.2.0",
     "jsonld": "^5.2.0",
     "jsonlint-mod": "^1.7.6",
     "lighthouse-plugin-publisher-ads": "^1.5.4",
     "magic-string": "^0.25.7",
     "mime-types": "^2.1.30",
+    "mocha": "^9.1.3",
     "node-fetch": "^2.6.1",
     "npm-run-posix-or-windows": "^2.0.2",
     "pako": "^2.0.3",
@@ -177,6 +185,7 @@
     "rollup-plugin-terser": "^7.0.2",
     "tabulator-tables": "^4.9.3",
     "terser": "^5.3.8",
+    "testdouble": "^3.16.3",
     "ts-jest": "^27.0.4",
     "typed-query-selector": "^2.6.1",
     "typescript": "^4.5.2",