Implement node:module createRequire API

The `createRequire()` function is used in node.js to create a `require()`
function that can be used from inside an ESM module. This is useful largely
for porting existing CommonJS code to ESM as it allows the ESM module to
keep using `require()` to load other modules.

This is being implemented in the runtime rather than as a polyfill because
it needs to integrate tightly with the module registry implementation.

The implementation is as close to Node.js' as we can currently make it.

Note that the return value of `createRequire()` is a function. In Node.js,
this function has additional properties like `require.resolve(...)`,
`require.main`, `require.extensions`, and `require.cache`. We are not
currently implementing these additional properties but we could in the
future if there is a need for them. (`require.cache` is unlikely to ever
be implemented since workers does not have a module cache the way Node.js
does).

This was implemented because the frameworks team asked for it.

Refs: https://nodejs.org/docs/latest/api/module.html#modulecreaterequirefilename

src/node/internal/module.d.ts

@@ -0,0 +1,6 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+
+export function createRequire(path: string): (specifier: string) => unknown;
+export function isBuiltin(specifier: string): boolean;

src/node/module.ts

@@ -0,0 +1,122 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+/* eslint-disable */
+
+import { default as moduleUtil } from 'node-internal:module';
+import { ERR_INVALID_ARG_VALUE } from 'node-internal:internal_errors';
+
+export function createRequire(
+  path: string | URL
+): (specifier: string) => unknown {
+  // Note that per Node.js' requirements, path must be one of either
+  // an absolute file path or a file URL. We do not currently handle
+  // module specifiers as URLs yet but we'll try to get close.
+
+  path = `${path}`;
+  if (
+    !(path as string).startsWith('/') &&
+    !(path as string).startsWith('file:')
+  ) {
+    throw new ERR_INVALID_ARG_VALUE(
+      'path',
+      path,
+      'The argument must be a file URL object, ' +
+        'a file URL string, or an absolute path string.'
+    );
+  }
+
+  return moduleUtil.createRequire(path as string);
+}
+
+// Indicates only that the given specifier is known to be a
+// Node.js built-in module specifier with or with the the
+// 'node:' prefix. A true return value does not guarantee that
+// the module is actually implemented in the runtime.
+export function isBuiltin(specifier: string): boolean {
+  return moduleUtil.isBuiltin(specifier);
+}
+
+// Intentionally does not include modules with mandatory 'node:'
+// prefix like `node:test`.
+// See: See https://nodejs.org/docs/latest/api/modules.html#built-in-modules-with-mandatory-node-prefix
+// TODO(later): This list duplicates the list that is in
+// workerd/jsg/modules.c++. Later we should source these
+// from the same place so we don't have to maintain two lists.
+export const builtinModules = [
+  '_http_agent',
+  '_http_client',
+  '_http_common',
+  '_http_incoming',
+  '_http_outgoing',
+  '_http_server',
+  '_stream_duplex',
+  '_stream_passthrough',
+  '_stream_readable',
+  '_stream_transform',
+  '_stream_wrap',
+  '_stream_writable',
+  '_tls_common',
+  '_tls_wrap',
+  'assert',
+  'assert/strict',
+  'async_hooks',
+  'buffer',
+  'child_process',
+  'cluster',
+  'console',
+  'constants',
+  'crypto',
+  'dgram',
+  'diagnostics_channel',
+  'dns',
+  'dns/promises',
+  'domain',
+  'events',
+  'fs',
+  'fs/promises',
+  'http',
+  'http2',
+  'https',
+  'inspector',
+  'inspector/promises',
+  'module',
+  'net',
+  'os',
+  'path',
+  'path/posix',
+  'path/win32',
+  'perf_hooks',
+  'process',
+  'punycode',
+  'querystring',
+  'readline',
+  'readline/promises',
+  'repl',
+  'stream',
+  'stream/consumers',
+  'stream/promises',
+  'stream/web',
+  'string_decoder',
+  'sys',
+  'timers',
+  'timers/promises',
+  'tls',
+  'trace_events',
+  'tty',
+  'url',
+  'util',
+  'util/types',
+  'v8',
+  'vm',
+  'wasi',
+  'worker_threads',
+  'zlib',
+];
+Object.freeze(builtinModules);
+
+export default {
+  createRequire,
+  isBuiltin,
+  builtinModules,
+};

src/workerd/api/node/BUILD.bazel

@@ -157,3 +157,9 @@ wd_test(
     args = ["--experimental"],
     data = ["tests/zlib-nodejs-test.js"],
 )
+
+wd_test(
+    src = "tests/module-create-require-test.wd-test",
+    args = ["--experimental"],
+    data = ["tests/module-create-require-test.js"],
+)

src/workerd/api/node/module.c++

@@ -0,0 +1,112 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+#include "module.h"
+#include <workerd/jsg/url.h>
+
+namespace workerd::api::node {
+
+bool ModuleUtil::isBuiltin(kj::String specifier) {
+  return jsg::checkNodeSpecifier(specifier) != kj::none;
+}
+
+jsg::JsValue ModuleUtil::createRequire(jsg::Lock& js, kj::String path) {
+  // Node.js requires that the specifier path is a File URL or an absolute
+  // file path string. To be compliant, we will convert whatever specifier
+  // is into a File URL if possible, then take the path as the actual
+  // specifier to use.
+  auto parsed = JSG_REQUIRE_NONNULL(jsg::Url::tryParse(path.asPtr(), "file:///"_kj), TypeError,
+      "The argument must be a file URL object, "
+      "a file URL string, or an absolute path string.");
+
+  // We do not currently handle specifiers as URLs, so let's treat any
+  // input that has query string params or hash fragments as errors.
+  if (parsed.getSearch().size() > 0 || parsed.getHash().size() > 0) {
+    JSG_FAIL_REQUIRE(
+        Error, "The specifier must not have query string parameters or hash fragments.");
+  }
+
+  // The specifier must be a file: URL
+  JSG_REQUIRE(parsed.getProtocol() == "file:"_kj, TypeError, "The specifier must be a file: URL.");
+
+  return jsg::JsValue(js.wrapReturningFunction(js.v8Context(),
+      [referrer = kj::str(parsed.getPathname())](
+          jsg::Lock& js, const v8::FunctionCallbackInfo<v8::Value>& args) -> v8::Local<v8::Value> {
+    auto registry = jsg::ModuleRegistry::from(js);
+
+    // TODO(soon): This will need to be updated to support the new module registry
+    // when that is fully implemented.
+    JSG_REQUIRE(registry != nullptr, Error, "Module registry not available.");
+
+    auto ref = ([&] {
+      try {
+        return kj::Path::parse(referrer.slice(1));
+      } catch (kj::Exception& e) {
+        JSG_FAIL_REQUIRE(Error, kj::str("Invalid referrer path: ", referrer.slice(1)));
+      }
+    })();
+
+    auto spec = kj::str(args[0]);
+
+    if (jsg::isNodeJsCompatEnabled(js)) {
+      KJ_IF_SOME(nodeSpec, jsg::checkNodeSpecifier(spec)) {
+        spec = kj::mv(nodeSpec);
+      }
+    }
+
+    static const kj::Path kRoot = kj::Path::parse("");
+
+    kj::Path targetPath = ([&] {
+      // If the specifier begins with one of our known prefixes, let's not resolve
+      // it against the referrer.
+      try {
+        if (spec.startsWith("node:") || spec.startsWith("cloudflare:") ||
+            spec.startsWith("workerd:")) {
+          return kj::Path::parse(spec);
+        }
+
+        return ref == kRoot ? kj::Path::parse(spec) : ref.parent().eval(spec);
+      } catch (kj::Exception&) {
+        JSG_FAIL_REQUIRE(Error, kj::str("Invalid specifier path: ", spec));
+      }
+    })();
+
+    // require() is only exposed to worker bundle modules so the resolve here is only
+    // permitted to require worker bundle or built-in modules. Internal modules are
+    // excluded.
+    auto& info = JSG_REQUIRE_NONNULL(
+        registry->resolve(js, targetPath, ref, jsg::ModuleRegistry::ResolveOption::DEFAULT,
+            jsg::ModuleRegistry::ResolveMethod::REQUIRE, spec.asPtr()),
+        Error, "No such module \"", targetPath.toString(), "\".");
+
+    bool isEsm = info.maybeSynthetic == kj::none;
+
+    auto module = info.module.getHandle(js);
+
+    jsg::instantiateModule(js, module);
+    auto handle = jsg::check(module->Evaluate(js.v8Context()));
+    KJ_ASSERT(handle->IsPromise());
+    auto prom = handle.As<v8::Promise>();
+    if (prom->State() == v8::Promise::PromiseState::kPending) {
+      js.runMicrotasks();
+    }
+    JSG_REQUIRE(prom->State() != v8::Promise::PromiseState::kPending, Error,
+        "Module evaluation did not complete synchronously.");
+    if (module->GetStatus() == v8::Module::kErrored) {
+      jsg::throwTunneledException(js.v8Isolate, module->GetException());
+    }
+
+    if (isEsm) {
+      // If the import is an esm module, we will return the namespace object.
+      jsg::JsObject obj(module->GetModuleNamespace().As<v8::Object>());
+      if (obj.get(js, "__cjsUnwrapDefault"_kj) == js.boolean(true)) {
+        return obj.get(js, "default"_kj);
+      }
+      return obj;
+    }
+
+    return jsg::JsValue(js.v8Get(module->GetModuleNamespace().As<v8::Object>(), "default"_kj));
+  }));
+}
+
+}  // namespace workerd::api::node

src/workerd/api/node/module.h

@@ -0,0 +1,30 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+#pragma once
+
+#include <workerd/jsg/jsg.h>
+
+namespace workerd::api::node {
+
+class ModuleUtil final: public jsg::Object {
+public:
+  ModuleUtil() = default;
+  ModuleUtil(jsg::Lock&, const jsg::Url&) {}
+
+  jsg::JsValue createRequire(jsg::Lock& js, kj::String specifier);
+
+  // Returns true if the specifier is a known node.js built-in module specifier.
+  // Ignores whether or not the module actually exists (use process.getBuiltinModule()
+  // for that purpose).
+  bool isBuiltin(kj::String specifier);
+
+  JSG_RESOURCE_TYPE(ModuleUtil) {
+    JSG_METHOD(createRequire);
+    JSG_METHOD(isBuiltin);
+  }
+};
+
+#define EW_NODE_MODULE_ISOLATE_TYPES api::node::ModuleUtil
+
+}  // namespace workerd::api::node

src/workerd/api/node/node.h

@@ -4,6 +4,7 @@
 #include "buffer.h"
 #include "crypto.h"
 #include "diagnostics-channel.h"
+#include "module.h"
 #include "url.h"
 #include "util.h"
 #include "zlib-util.h"
@@ -43,6 +44,7 @@ class CompatibilityFlags: public jsg::Object {
   V(AsyncHooksModule, "node-internal:async_hooks")                                                 \
   V(BufferUtil, "node-internal:buffer")                                                            \
   V(CryptoImpl, "node-internal:crypto")                                                            \
+  V(ModuleUtil, "node-internal:module")                                                            \
   V(UtilModule, "node-internal:util")                                                              \
   V(DiagnosticsChannelModule, "node-internal:diagnostics_channel")                                 \
   V(ZlibUtil, "node-internal:zlib")                                                                \
@@ -137,4 +139,5 @@ kj::Own<jsg::modules::ModuleBundle> getExternalNodeJsCompatModuleBundle(auto fea
 #define EW_NODE_ISOLATE_TYPES                                                                      \
   api::node::CompatibilityFlags, EW_NODE_BUFFER_ISOLATE_TYPES, EW_NODE_CRYPTO_ISOLATE_TYPES,       \
       EW_NODE_DIAGNOSTICCHANNEL_ISOLATE_TYPES, EW_NODE_ASYNCHOOKS_ISOLATE_TYPES,                   \
-      EW_NODE_UTIL_ISOLATE_TYPES, EW_NODE_ZLIB_ISOLATE_TYPES, EW_NODE_URL_ISOLATE_TYPES
+      EW_NODE_UTIL_ISOLATE_TYPES, EW_NODE_ZLIB_ISOLATE_TYPES, EW_NODE_URL_ISOLATE_TYPES,           \
+      EW_NODE_MODULE_ISOLATE_TYPES\

src/workerd/api/node/tests/module-create-require-test.js

@@ -0,0 +1,75 @@
+// Copyright (c) 2017-2022 Cloudflare, Inc.
+// Licensed under the Apache 2.0 license found in the LICENSE file or at:
+//     https://opensource.org/licenses/Apache-2.0
+import { createRequire, isBuiltin, builtinModules } from 'node:module';
+import { ok, strictEqual, throws } from 'node:assert';
+
+export const doTheTest = {
+  async test() {
+    const require = createRequire('/');
+    ok(typeof require === 'function');
+
+    const foo = require('foo');
+    const bar = require('bar');
+    const baz = require('baz');
+    const qux = require('worker/qux');
+
+    strictEqual(foo.default, 1);
+    strictEqual(bar, 2);
+    strictEqual(baz, 3);
+    strictEqual(qux, '4');
+
+    const assert = await import('node:assert');
+    const required = require('node:assert');
+    strictEqual(assert, required);
+
+    throws(() => require('invalid'), {
+      message: 'Module evaluation did not complete synchronously.',
+    });
+
+    throws(() => require('does not exist'));
+    throws(() => createRequire('not a valid path'), {
+      message: /The argument must be a file URL object/,
+    });
+    throws(() => createRequire(new URL('http://example.org')), {
+      message: /The argument must be a file URL object/,
+    });
+
+    // TODO(soon): Later when we when complete the new module registry, query strings
+    // and hash fragments will be allowed when the new registry is being used.
+    throws(() => createRequire('file://test?abc'), {
+      message:
+        'The specifier must not have query string parameters or hash fragments.',
+    });
+    throws(() => createRequire('file://test#123'), {
+      message:
+        'The specifier must not have query string parameters or hash fragments.',
+    });
+
+    // These should not throw...
+    createRequire('file:///');
+    createRequire('file:///tmp');
+    createRequire(new URL('file:///'));
+  },
+};
+
+export const isBuiltinTest = {
+  test() {
+    ok(isBuiltin('fs'));
+    ok(isBuiltin('http'));
+    ok(isBuiltin('https'));
+    ok(isBuiltin('path'));
+    ok(isBuiltin('node:fs'));
+    ok(isBuiltin('node:http'));
+    ok(isBuiltin('node:https'));
+    ok(isBuiltin('node:path'));
+    ok(isBuiltin('node:test'));
+    ok(!isBuiltin('test'));
+    ok(!isBuiltin('worker'));
+    ok(!isBuiltin('worker/qux'));
+
+    builtinModules.forEach((module) => {
+      ok(isBuiltin(module));
+    });
+  },
+};