diff --git a/.github/workflows/check-deploy-docs.yml b/.github/workflows/check-deploy-docs.yml index ca7e578935..b95ab12052 100644 --- a/.github/workflows/check-deploy-docs.yml +++ b/.github/workflows/check-deploy-docs.yml @@ -21,7 +21,7 @@ jobs: - name: Setup `node` uses: actions/setup-node@v3 with: - node-version: '20.10' + node-version: '24' - name: Install `jsdoc` run: npm install -g jsdoc@3.6.7 - name: Install `jsdoc` theme diff --git a/.github/workflows/check-pr.yml b/.github/workflows/check-pr.yml index 629ad2a653..64463e0abb 100644 --- a/.github/workflows/check-pr.yml +++ b/.github/workflows/check-pr.yml @@ -15,7 +15,7 @@ jobs: - name: Setup node uses: actions/setup-node@v3 with: - node-version: '20.10' + node-version: '24' - name: Install `sultan` run: pip3 install sultan - name: Checkout repository diff --git a/.github/workflows/daily-ci-checks.yml b/.github/workflows/daily-ci-checks.yml index b736eb2a9d..14adbf368b 100644 --- a/.github/workflows/daily-ci-checks.yml +++ b/.github/workflows/daily-ci-checks.yml @@ -16,7 +16,7 @@ jobs: - name: Setup `node` uses: actions/setup-node@v3 with: - node-version: '20.10' + node-version: '24' - name: Checkout repository uses: actions/checkout@v4 - name: Cache `lively.next` dependencies @@ -66,7 +66,7 @@ jobs: - name: Setup `node` uses: actions/setup-node@v3 with: - node-version: '20.10' + node-version: '24' - name: Install `eslint` run: npm install eslint@8.57.0 # pin this version as newer ones require us to use eslint@9 which comes with a new config format - name: Install `jsdoc` plugin for `eslint` diff --git a/.github/workflows/pr_labeler.yml b/.github/workflows/pr_labeler.yml index 948275db69..213fc99c8a 100644 --- a/.github/workflows/pr_labeler.yml +++ b/.github/workflows/pr_labeler.yml @@ -17,7 +17,7 @@ jobs: - name: Setup `node` uses: actions/setup-node@v3 with: - node-version: '20.10' + node-version: '24' - name: Checkout repository uses: actions/checkout@v4 - name: Install `octokit` diff --git a/.gitignore b/.gitignore index 5da5fb8806..0e9774a6ae 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,9 @@ robots.txt config.js user-server.sh package-lock.json +/mocha-es6/.cachedImportMap.json +/lively.*/.cachedImportMap.json +/flatn/.cachedImportMap.json /lively.server/usage_log.json /lively.installer.log /lively-system-interface/.lively.next-eval-server-for-test.js diff --git a/Makefile b/Makefile index a6b769ac77..e3a340f586 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clear-esm-cache clear-headless-cache start install hooks loading-screen landing-page clear-freezer-dir clean clean-install +.PHONY: clear-esm-cache clear-headless-cache start install hooks loading-screen landing-page freezer-unified freezer-unified-debug clear-freezer-dir clean clean-install clear-esm-cache: rm -rf esm_cache mkdir esm_cache @@ -7,23 +7,48 @@ clear-headless-cache: rm -r lively.headless/chrome-data-dir mkdir lively.headless/chrome-data-dir && touch lively.headless/chrome-data-dir/.gitkeep -artifacts: classes-runtime landing-page loading-screen +# New unified build - builds both landing-page and loading-screen in one pass +artifacts: classes-runtime freezer-unified classes-runtime: rm -rf lively.server/.module_cache rm -rf lively.classes/build env CI=true npm --prefix lively.classes run build +# Unified build - faster than building landing-page and loading-screen separately +freezer-unified: + rm -rf lively.server/.module_cache + rm -rf lively.freezer/landing-page + rm -rf lively.freezer/loading-screen + env CI=true npm --prefix lively.freezer run build-unified + +freezer-unified-debug: + rm -rf lively.server/.module_cache + rm -rf lively.freezer/landing-page + rm -rf lively.freezer/loading-screen + env CI=true DEBUG=true npm --prefix lively.freezer run build-unified + +# Legacy targets - kept for backward compatibility (but consider using freezer-unified instead) landing-page: rm -rf lively.server/.module_cache rm -rf lively.freezer/landing-page env CI=true npm --prefix lively.freezer run build-landing-page +landing-page-debug: + rm -rf lively.server/.module_cache + rm -rf lively.freezer/landing-page + env CI=true DEBUG=true npm --prefix lively.freezer run build-landing-page + loading-screen: rm -rf lively.server/.module_cache rm -rf lively.freezer/loading-screen env CI=true npm --prefix lively.freezer run build-loading-screen +loading-screen-debug: + rm -rf lively.server/.module_cache + rm -rf lively.freezer/loading-screen + env CI=true DEBUG=true npm --prefix lively.freezer run build-loading-screen + clear-freezer-dir: rm -rf lively.freezer/landing-page rm -rf lively.freezer/loading-screen diff --git a/README.md b/README.md index e3826a4ab7..4bee548504 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,10 @@ You need to install `lively.next` on your system. Currently, MacOS, Linux, and the Linux Subsystem for Windows are supported. Make sure you have the following software installed: -1. `node.js v20.10` +1. `node.js v24` or higher 2. `git`. -We try to require/support the current LTS version of `node`. +We require Node.js v24 or higher for compatibility with modern JavaScript features. For some more advanced development operations (such as bulk testing from the command line and spell checking inside of `lively.next`), you will also need diff --git a/flatn/flatn-cjs.js b/flatn/flatn-cjs.js index f451c7a8ac..1496a6a9df 100644 --- a/flatn/flatn-cjs.js +++ b/flatn/flatn-cjs.js @@ -1,7 +1,5 @@ 'use strict'; -Object.defineProperty(exports, '__esModule', { value: true }); - var path = require('path'); var fs = require('fs'); var util = require('util'); @@ -16,16 +14,6 @@ var _events = require('events'); var child_process = require('child_process'); var os = require('os'); -function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; } - -var path__default = /*#__PURE__*/_interopDefaultLegacy(path); -var fs__default = /*#__PURE__*/_interopDefaultLegacy(fs); -var http__default = /*#__PURE__*/_interopDefaultLegacy(http); -var https__default = /*#__PURE__*/_interopDefaultLegacy(https); -var zlib__default = /*#__PURE__*/_interopDefaultLegacy(zlib); -var Stream__default = /*#__PURE__*/_interopDefaultLegacy(Stream); -var _events__default = /*#__PURE__*/_interopDefaultLegacy(_events); - var commonjsGlobal = typeof globalThis !== 'undefined' ? globalThis : typeof window !== 'undefined' ? window : typeof global !== 'undefined' ? global : typeof self !== 'undefined' ? self : {}; var semver$1 = {exports: {}}; @@ -5462,7 +5450,7 @@ function requirePonyfill_es2018 () { return ponyfill_es2018.exports; } -var _5_6_0 = {}; +var _5_2_1 = {}; var b64 = {}; @@ -5471,123 +5459,114 @@ var hasRequiredB64; function requireB64 () { if (hasRequiredB64) return b64; hasRequiredB64 = 1; - (function (exports) { -(function (exports) { - - var lookup = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; - - var Arr = (typeof Uint8Array !== 'undefined') - ? Uint8Array - : Array; - - var PLUS = '+'.charCodeAt(0); - var SLASH = '/'.charCodeAt(0); - var NUMBER = '0'.charCodeAt(0); - var LOWER = 'a'.charCodeAt(0); - var UPPER = 'A'.charCodeAt(0); - var PLUS_URL_SAFE = '-'.charCodeAt(0); - var SLASH_URL_SAFE = '_'.charCodeAt(0); - - function decode (elt) { - var code = elt.charCodeAt(0); - if (code === PLUS || code === PLUS_URL_SAFE) return 62 // '+' - if (code === SLASH || code === SLASH_URL_SAFE) return 63 // '/' - if (code < NUMBER) return -1 // no match - if (code < NUMBER + 10) return code - NUMBER + 26 + 26 - if (code < UPPER + 26) return code - UPPER - if (code < LOWER + 26) return code - LOWER + 26 - } - function b64ToByteArray (b64) { - var i, j, l, tmp, placeHolders, arr; + b64.toByteArray = toByteArray; + b64.fromByteArray = fromByteArray; - if (b64.length % 4 > 0) { - throw new Error('Invalid string. Length must be a multiple of 4') - } + var lookup = []; + var revLookup = []; + var Arr = typeof Uint8Array !== 'undefined' ? Uint8Array : Array; - // the number of equal signs (place holders) - // if there are two placeholders, than the two characters before it - // represent one byte - // if there is only one, then the three characters before it represent 2 bytes - // this is just a cheap hack to not do indexOf twice - var len = b64.length; - placeHolders = b64.charAt(len - 2) === '=' ? 2 : b64.charAt(len - 1) === '=' ? 1 : 0; + function init () { + var code = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; + for (var i = 0, len = code.length; i < len; ++i) { + lookup[i] = code[i]; + revLookup[code.charCodeAt(i)] = i; + } - // base64 is 4/3 + up to two characters of the original data - arr = new Arr(b64.length * 3 / 4 - placeHolders); + revLookup['-'.charCodeAt(0)] = 62; + revLookup['_'.charCodeAt(0)] = 63; + } - // if there are placeholders, only get up to the last complete 4 chars - l = placeHolders > 0 ? b64.length - 4 : b64.length; + init(); - var L = 0; + function toByteArray (b64) { + var i, j, l, tmp, placeHolders, arr; + var len = b64.length; - function push (v) { - arr[L++] = v; - } + if (len % 4 > 0) { + throw new Error('Invalid string. Length must be a multiple of 4') + } - for (i = 0, j = 0; i < l; i += 4, j += 3) { - tmp = (decode(b64.charAt(i)) << 18) | (decode(b64.charAt(i + 1)) << 12) | (decode(b64.charAt(i + 2)) << 6) | decode(b64.charAt(i + 3)); - push((tmp & 0xFF0000) >> 16); - push((tmp & 0xFF00) >> 8); - push(tmp & 0xFF); - } + // the number of equal signs (place holders) + // if there are two placeholders, than the two characters before it + // represent one byte + // if there is only one, then the three characters before it represent 2 bytes + // this is just a cheap hack to not do indexOf twice + placeHolders = b64[len - 2] === '=' ? 2 : b64[len - 1] === '=' ? 1 : 0; - if (placeHolders === 2) { - tmp = (decode(b64.charAt(i)) << 2) | (decode(b64.charAt(i + 1)) >> 4); - push(tmp & 0xFF); - } else if (placeHolders === 1) { - tmp = (decode(b64.charAt(i)) << 10) | (decode(b64.charAt(i + 1)) << 4) | (decode(b64.charAt(i + 2)) >> 2); - push((tmp >> 8) & 0xFF); - push(tmp & 0xFF); - } + // base64 is 4/3 + up to two characters of the original data + arr = new Arr(len * 3 / 4 - placeHolders); - return arr - } + // if there are placeholders, only get up to the last complete 4 chars + l = placeHolders > 0 ? len - 4 : len; - function uint8ToBase64 (uint8) { - var i; - var extraBytes = uint8.length % 3; // if we have 1 byte left, pad 2 bytes - var output = ''; - var temp, length; + var L = 0; - function encode (num) { - return lookup.charAt(num) - } + for (i = 0, j = 0; i < l; i += 4, j += 3) { + tmp = (revLookup[b64.charCodeAt(i)] << 18) | (revLookup[b64.charCodeAt(i + 1)] << 12) | (revLookup[b64.charCodeAt(i + 2)] << 6) | revLookup[b64.charCodeAt(i + 3)]; + arr[L++] = (tmp >> 16) & 0xFF; + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } - function tripletToBase64 (num) { - return encode(num >> 18 & 0x3F) + encode(num >> 12 & 0x3F) + encode(num >> 6 & 0x3F) + encode(num & 0x3F) - } + if (placeHolders === 2) { + tmp = (revLookup[b64.charCodeAt(i)] << 2) | (revLookup[b64.charCodeAt(i + 1)] >> 4); + arr[L++] = tmp & 0xFF; + } else if (placeHolders === 1) { + tmp = (revLookup[b64.charCodeAt(i)] << 10) | (revLookup[b64.charCodeAt(i + 1)] << 4) | (revLookup[b64.charCodeAt(i + 2)] >> 2); + arr[L++] = (tmp >> 8) & 0xFF; + arr[L++] = tmp & 0xFF; + } - // go through the array every three bytes, we'll deal with trailing stuff later - for (i = 0, length = uint8.length - extraBytes; i < length; i += 3) { - temp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); - output += tripletToBase64(temp); - } + return arr + } - // pad the end with zeros, but make sure to not forget the extra bytes - switch (extraBytes) { - case 1: - temp = uint8[uint8.length - 1]; - output += encode(temp >> 2); - output += encode((temp << 4) & 0x3F); - output += '=='; - break - case 2: - temp = (uint8[uint8.length - 2] << 8) + (uint8[uint8.length - 1]); - output += encode(temp >> 10); - output += encode((temp >> 4) & 0x3F); - output += encode((temp << 2) & 0x3F); - output += '='; - break - } + function tripletToBase64 (num) { + return lookup[num >> 18 & 0x3F] + lookup[num >> 12 & 0x3F] + lookup[num >> 6 & 0x3F] + lookup[num & 0x3F] + } - return output - } + function encodeChunk (uint8, start, end) { + var tmp; + var output = []; + for (var i = start; i < end; i += 3) { + tmp = (uint8[i] << 16) + (uint8[i + 1] << 8) + (uint8[i + 2]); + output.push(tripletToBase64(tmp)); + } + return output.join('') + } + + function fromByteArray (uint8) { + var tmp; + var len = uint8.length; + var extraBytes = len % 3; // if we have 1 byte left, pad 2 bytes + var output = ''; + var parts = []; + var maxChunkLength = 16383; // must be multiple of 3 + + // go through the array every three bytes, we'll deal with trailing stuff later + for (var i = 0, len2 = len - extraBytes; i < len2; i += maxChunkLength) { + parts.push(encodeChunk(uint8, i, (i + maxChunkLength) > len2 ? len2 : (i + maxChunkLength))); + } - exports.toByteArray = b64ToByteArray; - exports.fromByteArray = uint8ToBase64; - }(exports)); -} (b64)); + // pad the end with zeros, but make sure to not forget the extra bytes + if (extraBytes === 1) { + tmp = uint8[len - 1]; + output += lookup[tmp >> 2]; + output += lookup[(tmp << 4) & 0x3F]; + output += '=='; + } else if (extraBytes === 2) { + tmp = (uint8[len - 2] << 8) + (uint8[len - 1]); + output += lookup[tmp >> 10]; + output += lookup[(tmp >> 4) & 0x3F]; + output += lookup[(tmp << 2) & 0x3F]; + output += '='; + } + + parts.push(output); + + return parts.join('') + } return b64; } @@ -5692,19 +5671,15 @@ function require_1_1_4 () { * @license MIT */ -var hasRequired_5_6_0; +var hasRequired_5_2_1; -function require_5_6_0 () { - if (hasRequired_5_6_0) return _5_6_0; - hasRequired_5_6_0 = 1; +function require_5_2_1 () { + if (hasRequired_5_2_1) return _5_2_1; + hasRequired_5_2_1 = 1; (function (exports) { var base64 = requireB64(); var ieee754 = require_1_1_4(); - var customInspectSymbol = - (typeof Symbol === 'function' && typeof Symbol.for === 'function') - ? Symbol.for('nodejs.util.inspect.custom') - : null; exports.Buffer = Buffer; exports.SlowBuffer = SlowBuffer; @@ -5741,9 +5716,7 @@ function require_5_6_0 () { // Can typed array instances can be augmented? try { var arr = new Uint8Array(1); - var proto = { foo: function () { return 42 } }; - Object.setPrototypeOf(proto, Uint8Array.prototype); - Object.setPrototypeOf(arr, proto); + arr.__proto__ = { __proto__: Uint8Array.prototype, foo: function () { return 42 } }; return arr.foo() === 42 } catch (e) { return false @@ -5772,7 +5745,7 @@ function require_5_6_0 () { } // Return an augmented `Uint8Array` instance var buf = new Uint8Array(length); - Object.setPrototypeOf(buf, Buffer.prototype); + buf.__proto__ = Buffer.prototype; return buf } @@ -5799,6 +5772,17 @@ function require_5_6_0 () { return from(arg, encodingOrOffset, length) } + // Fix subarray() in ES2016. See: https://github.com/feross/buffer/pull/97 + if (typeof Symbol !== 'undefined' && Symbol.species != null && + Buffer[Symbol.species] === Buffer) { + Object.defineProperty(Buffer, Symbol.species, { + value: null, + configurable: true, + enumerable: false, + writable: false + }); + } + Buffer.poolSize = 8192; // not used by this implementation function from (value, encodingOrOffset, length) { @@ -5811,7 +5795,7 @@ function require_5_6_0 () { } if (value == null) { - throw new TypeError( + throw TypeError( 'The first argument must be one of type string, Buffer, ArrayBuffer, Array, ' + 'or Array-like Object. Received type ' + (typeof value) ) @@ -5822,12 +5806,6 @@ function require_5_6_0 () { return fromArrayBuffer(value, encodingOrOffset, length) } - if (typeof SharedArrayBuffer !== 'undefined' && - (isInstance(value, SharedArrayBuffer) || - (value && isInstance(value.buffer, SharedArrayBuffer)))) { - return fromArrayBuffer(value, encodingOrOffset, length) - } - if (typeof value === 'number') { throw new TypeError( 'The "value" argument must not be of type number. Received type number' @@ -5869,8 +5847,8 @@ function require_5_6_0 () { // Note: Change prototype *after* Buffer.from is defined to workaround Chrome bug: // https://github.com/feross/buffer/pull/148 - Object.setPrototypeOf(Buffer.prototype, Uint8Array.prototype); - Object.setPrototypeOf(Buffer, Uint8Array); + Buffer.prototype.__proto__ = Uint8Array.prototype; + Buffer.__proto__ = Uint8Array; function assertSize (size) { if (typeof size !== 'number') { @@ -5974,8 +5952,7 @@ function require_5_6_0 () { } // Return an augmented `Uint8Array` instance - Object.setPrototypeOf(buf, Buffer.prototype); - + buf.__proto__ = Buffer.prototype; return buf } @@ -6297,9 +6274,6 @@ function require_5_6_0 () { if (this.length > max) str += ' ... '; return '' }; - if (customInspectSymbol) { - Buffer.prototype[customInspectSymbol] = Buffer.prototype.inspect; - } Buffer.prototype.compare = function compare (target, start, end, thisStart, thisEnd) { if (isInstance(target, Uint8Array)) { @@ -6425,7 +6399,7 @@ function require_5_6_0 () { return Uint8Array.prototype.lastIndexOf.call(buffer, val, byteOffset) } } - return arrayIndexOf(buffer, [val], byteOffset, encoding, dir) + return arrayIndexOf(buffer, [ val ], byteOffset, encoding, dir) } throw new TypeError('val must be string, number or Buffer') @@ -6754,7 +6728,7 @@ function require_5_6_0 () { var out = ''; for (var i = start; i < end; ++i) { - out += hexSliceLookupTable[buf[i]]; + out += toHex(buf[i]); } return out } @@ -6791,8 +6765,7 @@ function require_5_6_0 () { var newBuf = this.subarray(start, end); // Return an augmented `Uint8Array` instance - Object.setPrototypeOf(newBuf, Buffer.prototype); - + newBuf.__proto__ = Buffer.prototype; return newBuf }; @@ -7281,8 +7254,6 @@ function require_5_6_0 () { } } else if (typeof val === 'number') { val = val & 255; - } else if (typeof val === 'boolean') { - val = Number(val); } // Invalid ranges are not set to a default, so can range check early. @@ -7340,6 +7311,11 @@ function require_5_6_0 () { return str } + function toHex (n) { + if (n < 16) return '0' + n.toString(16) + return n.toString(16) + } + function utf8ToBytes (string, units) { units = units || Infinity; var codePoint; @@ -7469,22 +7445,8 @@ function require_5_6_0 () { // For IE11 support return obj !== obj // eslint-disable-line no-self-compare } - - // Create lookup table for `toString('hex')` - // See: https://github.com/feross/buffer/issues/219 - var hexSliceLookupTable = (function () { - var alphabet = '0123456789abcdef'; - var table = new Array(256); - for (var i = 0; i < 16; ++i) { - var i16 = i * 16; - for (var j = 0; j < 16; ++j) { - table[i16 + j] = alphabet[i] + alphabet[j]; - } - } - return table - })(); -} (_5_6_0)); - return _5_6_0; +} (_5_2_1)); + return _5_2_1; } /* c8 ignore start */ @@ -7516,7 +7478,7 @@ if (!globalThis.ReadableStream) { try { // Don't use node: prefix for this, require+node: is not supported until node v14.14 // Only `import()` can use prefix in 12.20 and later - const { Blob } = require_5_6_0(); + const { Blob } = require_5_2_1(); if (Blob && !Blob.prototype.stream) { Blob.prototype.stream = function name (params) { let position = 0; @@ -7541,6 +7503,7 @@ try { /*! fetch-blob. MIT License. Jimmy Wärting */ + // 64 KiB (same size chrome slice theirs blob into Uint8array's) const POOL_SIZE = 65536; @@ -7833,6 +7796,7 @@ const File = _File; /*! formdata-polyfill. MIT License. Jimmy Wärting */ + var {toStringTag:t,iterator:i,hasInstance:h}=Symbol, r=Math.random, m='append,set,get,getAll,delete,keys,values,entries,forEach,constructor'.split(','), @@ -7997,7 +7961,7 @@ const isSameProtocol = (destination, original) => { return orig === dest; }; -const pipeline = util.promisify(Stream__default["default"].pipeline); +const pipeline = util.promisify(Stream.pipeline); const INTERNALS$2 = Symbol('Body internals'); /** @@ -8027,7 +7991,7 @@ class Body { } else if (ArrayBuffer.isView(body)) { // Body is ArrayBufferView body = buffer.Buffer.from(body.buffer, body.byteOffset, body.byteLength); - } else if (body instanceof Stream__default["default"]) ; else if (body instanceof FormData) { + } else if (body instanceof Stream) ; else if (body instanceof FormData) { // Body is FormData body = formDataToBlob(body); boundary = body.type.split('=')[1]; @@ -8040,9 +8004,9 @@ class Body { let stream = body; if (buffer.Buffer.isBuffer(body)) { - stream = Stream__default["default"].Readable.from(body); + stream = Stream.Readable.from(body); } else if (isBlob(body)) { - stream = Stream__default["default"].Readable.from(body.stream()); + stream = Stream.Readable.from(body.stream()); } this[INTERNALS$2] = { @@ -8054,7 +8018,7 @@ class Body { }; this.size = size; - if (body instanceof Stream__default["default"]) { + if (body instanceof Stream) { body.on('error', error_ => { const error = error_ instanceof FetchBaseError ? error_ : @@ -8185,7 +8149,7 @@ async function consumeBody(data) { } /* c8 ignore next 3 */ - if (!(body instanceof Stream__default["default"])) { + if (!(body instanceof Stream)) { return buffer.Buffer.alloc(0); } @@ -8244,7 +8208,7 @@ const clone = (instance, highWaterMark) => { // Check that body is a stream and not form-data object // note: we can't clone the form-data object without having it as a dependency - if ((body instanceof Stream__default["default"]) && (typeof body.getBoundary !== 'function')) { + if ((body instanceof Stream) && (typeof body.getBoundary !== 'function')) { // Tee instance body p1 = new Stream.PassThrough({highWaterMark}); p2 = new Stream.PassThrough({highWaterMark}); @@ -8310,7 +8274,7 @@ const extractContentType = (body, request) => { } // Body is stream - can't really do much about this - if (body instanceof Stream__default["default"]) { + if (body instanceof Stream) { return null; } @@ -8377,9 +8341,10 @@ const writeToStream = async (dest, {body}) => { * Headers class offers convenient helpers */ + /* c8 ignore next 9 */ -const validateHeaderName = typeof http__default["default"].validateHeaderName === 'function' ? - http__default["default"].validateHeaderName : +const validateHeaderName = typeof http.validateHeaderName === 'function' ? + http.validateHeaderName : name => { if (!/^[\^`\-\w!#$%&'*+.|~]+$/.test(name)) { const error = new TypeError(`Header name must be a valid HTTP token [${name}]`); @@ -8389,8 +8354,8 @@ const validateHeaderName = typeof http__default["default"].validateHeaderName == }; /* c8 ignore next 9 */ -const validateHeaderValue = typeof http__default["default"].validateHeaderValue === 'function' ? - http__default["default"].validateHeaderValue : +const validateHeaderValue = typeof http.validateHeaderValue === 'function' ? + http.validateHeaderValue : (name, value) => { if (/[^\t\u0020-\u007E\u0080-\u00FF]/.test(value)) { const error = new TypeError(`Invalid character in header content ["${name}"]`); @@ -8652,6 +8617,7 @@ const isRedirect = code => { * Response class provides content decoding */ + const INTERNALS$1 = Symbol('Response internals'); /** @@ -9141,6 +9107,7 @@ function parseReferrerPolicyFromHeader(headers) { * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ + const INTERNALS = Symbol('Request internals'); /** @@ -9473,6 +9440,7 @@ if (!globalThis.DOMException) { * All spec algorithm step numbers are based on https://fetch.spec.whatwg.org/commit-snapshots/ae716822cb3a61843226cd090eefc6589446c1d2/. */ + const supportedSchemas = new Set(['data:', 'http:', 'https:']); /** @@ -9499,14 +9467,14 @@ async function fetch$1(url, options_) { } // Wrap http.request into fetch - const send = (parsedURL.protocol === 'https:' ? https__default["default"] : http__default["default"]).request; + const send = (parsedURL.protocol === 'https:' ? https : http).request; const {signal} = request; let response = null; const abort = () => { const error = new AbortError('The operation was aborted.'); reject(error); - if (request.body && request.body instanceof Stream__default["default"].Readable) { + if (request.body && request.body instanceof Stream.Readable) { request.body.destroy(error); } @@ -9650,7 +9618,7 @@ async function fetch$1(url, options_) { } // HTTP-redirect fetch step 9 - if (response_.statusCode !== 303 && request.body && options_.body instanceof Stream__default["default"].Readable) { + if (response_.statusCode !== 303 && request.body && options_.body instanceof Stream.Readable) { reject(new FetchError('Cannot follow redirect with body being a readable stream', 'unsupported-redirect')); finalize(); return; @@ -9731,13 +9699,13 @@ async function fetch$1(url, options_) { // by common browsers. // Always using Z_SYNC_FLUSH is what cURL does. const zlibOptions = { - flush: zlib__default["default"].Z_SYNC_FLUSH, - finishFlush: zlib__default["default"].Z_SYNC_FLUSH + flush: zlib.Z_SYNC_FLUSH, + finishFlush: zlib.Z_SYNC_FLUSH }; // For gzip if (codings === 'gzip' || codings === 'x-gzip') { - body = Stream.pipeline(body, zlib__default["default"].createGunzip(zlibOptions), error => { + body = Stream.pipeline(body, zlib.createGunzip(zlibOptions), error => { if (error) { reject(error); } @@ -9759,13 +9727,13 @@ async function fetch$1(url, options_) { raw.once('data', chunk => { // See http://stackoverflow.com/questions/37519828 if ((chunk[0] & 0x0F) === 0x08) { - body = Stream.pipeline(body, zlib__default["default"].createInflate(), error => { + body = Stream.pipeline(body, zlib.createInflate(), error => { if (error) { reject(error); } }); } else { - body = Stream.pipeline(body, zlib__default["default"].createInflateRaw(), error => { + body = Stream.pipeline(body, zlib.createInflateRaw(), error => { if (error) { reject(error); } @@ -9788,7 +9756,7 @@ async function fetch$1(url, options_) { // For br if (codings === 'br') { - body = Stream.pipeline(body, zlib__default["default"].createBrotliDecompress(), error => { + body = Stream.pipeline(body, zlib.createBrotliDecompress(), error => { if (error) { reject(error); } @@ -10281,6 +10249,7 @@ const windowsRootPathRe = /^[a-z]:\/$/i; /* global clearTimeout, setTimeout */ + /** * Throws a well behaved error on purpopse, when accessing unimplemented * functionality. @@ -10329,6 +10298,7 @@ function argumentNames (f) { * @module lively.lang/object */ + // -=-=-=-=-=-=-=-=- // internal helper // -=-=-=-=-=-=-=-=- @@ -13873,6 +13843,8 @@ exports$1.find; exports$1.articlize; /* global btoa,JsDiff */ +/* lively.vm dontTransform: ["btoa"] */ + // -=-=-=-=-=-=-=-=-=-=-=-=- // file system path support @@ -14202,6 +14174,7 @@ class Resource { * @module lively.lang/number */ + /** * Returns wether `x` is between `a` and `b` and keeps `eps` distance from both of them. * @param {number} x - The number that should be between two bounds @@ -14213,12 +14186,13 @@ class Resource { function between (x, a, b, eps) { eps = eps || 0; let min, max; - if (a < b) { min = a, max = b; } else { max = a, min = b; } + { min = a, max = b; } return (max - x + eps >= 0) && (min - x - eps <= 0); } /* global Promise */ + /** * Returns an object that conveniently gives access to the promise itself and * its resolution and rejection callback. This separates the resolve/reject handling @@ -14234,6 +14208,9 @@ function deferred () { } // -=-=-=-=-=-=-=-=-=-=-=-=-=- +// js object path accessor +// -=-=-=-=-=-=-=-=-=-=-=-=-=- + function Path (p, splitter) { if (p instanceof Path) return p; @@ -14531,6 +14508,7 @@ Object.assign(Path.prototype, { * @module lively.lang/graph */ + /** * Sorts graph into an array of arrays. Each "bucket" contains the graph * nodes that have no other incoming nodes than those already visited. This @@ -14618,7 +14596,7 @@ function sortByReference (depGraph, startNode) { /** * An interval defining an upper and a lower bound. * @typedef { number[] } Interval - * @property {number} 0 - The lower bound of the interval. + * @property {number} 0 - The lower bound of the interval. * @property {number} 1 - The upper bound of the interval. */ @@ -14628,6 +14606,7 @@ typeof System !== 'undefined' /* global process */ + /* * A simple node.js-like cross-platform event emitter implementation that can * be used as a mixin. Emitters support the methods: `on(eventName, handlerFunc)`, @@ -14651,7 +14630,7 @@ const isNode$1 = typeof process !== 'undefined' && process.versions && process.v const makeEmitter = isNode$1 ? function (obj, options) { if (obj.on && obj.removeListener) { return obj; } - let events = _events__default["default"]; + let events = _events; if (!events) events = System._nodeRequire('events'); Object.assign(obj, events.EventEmitter.prototype); events.EventEmitter.call(obj); @@ -14711,6 +14690,7 @@ typeof process !== 'undefined' && process.env && typeof process.exit === 'functi /* global System */ + // type EventType = string // type EventTime = number // type Notification = {type: EventType, time: EventTime, ...}; @@ -14775,6 +14755,7 @@ function log (notification) { // Notification -> () /* global fetch, DOMParser, XPathEvaluator, XPathResult, Namespace,System,global,process,XMLHttpRequest,Buffer */ + class XPathQuery { constructor (expression) { this.expression = expression; @@ -15527,15 +15508,8 @@ class ESMResource extends Resource { } getEsmURL () { - let baseUrl; - if (this.url.startsWith('esm://run/npm/')) baseUrl = 'https://cdn.jsdelivr.net/'; - else if (this.url.startsWith('esm://run/')) baseUrl = 'https://esm.run/'; - else if (this.url.startsWith('esm://cache/')) baseUrl = 'https://jspm.dev/'; - else { - const domain = this.url.match(/esm:\/\/([^\/]*)\//)?.[1]; - baseUrl = `https://${domain}/`; - } - return baseUrl; + const domain = this.url.match(/esm:\/\/([^\/]*)\//)?.[1]; + return `https://${domain}/`; } async read () { @@ -16013,7 +15987,7 @@ function isAbsolute (path) { function parentDir (p) { if (p.isResource) return p.parent(); - return path__default["default"].basename(p); + return path.basename(p); } function equalLocation (a, b) { @@ -16023,37 +15997,37 @@ function equalLocation (a, b) { function join (a, b) { if (a.isResource) return a.join(b); - return path__default["default"].join(a, b); + return path.join(a, b); } function normalizePath (p) { if (p.isResource) return p.withRelativePartsResolved(); - return path__default["default"].normalize(p); + return path.normalize(p); } function fs_isDirectory (location) { if (location.isResource) return location.isDirectory(); - return fs__default["default"].statSync(location).isDirectory(); + return fs.statSync(location).isDirectory(); } function fs_exists (location) { if (location.isResource) return location.exists(); - return fs__default["default"].existsSync(location); + return fs.existsSync(location); } function fs_read (location) { if (location.isResource) return location.read(); - return fs__default["default"].readFileSync(location); + return fs.readFileSync(location); } function fs_write (location, content) { if (location.isResource) return location.write(content); - return fs__default["default"].writeFileSync(location, content); + return fs.writeFileSync(location, content); } function fs_readJson (location) { if (location.isResource) return location.exists().then(exists => exists ? location.readJson() : null); - return fs__default["default"].existsSync(location) ? JSON.parse(String(fs_read(location))) : null; + return fs.existsSync(location) ? JSON.parse(String(fs_read(location))) : null; } function fs_writeJson (location, jso) { @@ -16063,7 +16037,7 @@ function fs_writeJson (location, jso) { function fs_dirList (location) { if (location.isResource) return location.dirList(1); - return fs__default["default"].readdirSync(location).map(ea => join(location, ea)); + return fs.readdirSync(location).map(ea => join(location, ea)); } // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- @@ -16221,7 +16195,7 @@ class PackageMap { let key = this.keyFor(packageCollectionDirs, individualPackageDirs, devPackageDirs); return this.cache[key] || (this.cache[key] = this.build(packageCollectionDirs, individualPackageDirs, devPackageDirs), [...packageCollectionDirs, ...individualPackageDirs, ...devPackageDirs].forEach(dir => { - let watcher = fs__default["default"].watch(dir, {persistent: false}, () => { + let watcher = fs.watch(dir, {persistent: false}, () => { delete this.cache[key]; watcher.close(); }); @@ -16608,7 +16582,6 @@ function depGraph (packageSpec, packageMap, dependencyFields = ['dependencies']) } function buildStages (packageSpec, packageMap, dependencyFields) { - let { name, version } = packageSpec; let { deps, resolvedVersions } = depGraph(packageSpec, packageMap); for (let dep in deps) { @@ -16617,7 +16590,7 @@ function buildStages (packageSpec, packageMap, dependencyFields) { } } - return sortByReference(deps, `${name}@${version}`); + return sortByReference(deps); } /* global System,process,__dirname */ @@ -16631,15 +16604,15 @@ let _npmEnv; function npmEnv () { return _npmEnv || (_npmEnv = (() => { let dir; let cacheFile = path.join(tmpdir(), 'npm-env.json'); let env = {}; - if (fs__default["default"].existsSync(cacheFile)) { - let cached = JSON.parse(String(fs__default["default"].readFileSync(cacheFile))); + if (fs.existsSync(cacheFile)) { + let cached = JSON.parse(String(fs.readFileSync(cacheFile))); if (Date.now() - cached.time < 1000 * 60) return cached.env; } try { dir = path.join(tmpdir(), 'npm-test-env-project'); - if (!fs__default["default"].existsSync(dir)) fs__default["default"].mkdirSync(dir); - fs__default["default"].writeFileSync(path.join(dir, 'package.json'), `{"scripts": {"print-env": "${process.env.npm_node_execpath || 'node'} ./print-env.js"}}`); - fs__default["default"].writeFileSync(path.join(dir, 'print-env.js'), 'console.log(JSON.stringify(process.env))'); + if (!fs.existsSync(dir)) fs.mkdirSync(dir); + fs.writeFileSync(path.join(dir, 'package.json'), `{"scripts": {"print-env": "${process.env.npm_node_execpath || 'node'} ./print-env.js"}}`); + fs.writeFileSync(path.join(dir, 'print-env.js'), 'console.log(JSON.stringify(process.env))'); let PATH = process.env.PATH.split(':').filter(ea => ea !== helperBinDir).join(':'); Object.keys(process.env).forEach(ea => { if (ea.toLowerCase().startsWith('npm_config_')) { env[ea] = process.env[ea]; } @@ -16655,12 +16628,12 @@ function npmEnv () { env = {}; } finally { try { - if (fs__default["default"].existsSync(path.join(dir, 'package.json'))) { fs__default["default"].unlinkSync(path.join(dir, 'package.json')); } - fs__default["default"].unlinkSync(path.join(dir, 'print-env.js')); - fs__default["default"].rmdirSync(dir); + if (fs.existsSync(path.join(dir, 'package.json'))) { fs.unlinkSync(path.join(dir, 'package.json')); } + fs.unlinkSync(path.join(dir, 'print-env.js')); + fs.rmdirSync(dir); } catch (err) { } } - fs__default["default"].writeFileSync(cacheFile, JSON.stringify({ time: Date.now(), env })); + fs.writeFileSync(cacheFile, JSON.stringify({ time: Date.now(), env })); return env; })()); } @@ -16683,7 +16656,7 @@ function npmCreateEnvVars (configObj, env = {}, path = 'npm_package') { function linkBins (packageSpecs, linkState = {}, verbose = false) { let linkLocation = path.join(tmpdir(), 'npm-helper-bin-dir'); - if (!fs__default["default"].existsSync(linkLocation)) fs__default["default"].mkdirSync(linkLocation); + if (!fs.existsSync(linkLocation)) fs.mkdirSync(linkLocation); packageSpecs.forEach(({ bin, location }) => { if (location.startsWith('file://')) { location = location.replace(/^file:\/\//, ''); } if (!bin) return; @@ -16692,11 +16665,11 @@ function linkBins (packageSpecs, linkState = {}, verbose = false) { let realFile = bin[linkName]; try { // fs.existsSync follows links, so broken links won't be reported as existing - fs__default["default"].lstatSync(path.join(linkLocation, linkName)); - fs__default["default"].unlinkSync(path.join(linkLocation, linkName)); + fs.lstatSync(path.join(linkLocation, linkName)); + fs.unlinkSync(path.join(linkLocation, linkName)); } catch (err) { } verbose && console.log(`[flatn build] linking ${path.join(location, realFile)} => ${path.join(linkLocation, linkName)}`); - fs__default["default"].symlinkSync(path.join(location, realFile), path.join(linkLocation, linkName)); + fs.symlinkSync(path.join(location, realFile), path.join(linkLocation, linkName)); } linkState[location] = true; }); @@ -16749,7 +16722,7 @@ class BuildProcess { normalizeScripts ({ scripts, location }) { if (!scripts || !scripts.install) { - let hasBindingGyp = fs__default["default"].existsSync(path.join(location, 'binding.gyp')); + let hasBindingGyp = fs.existsSync(path.join(location, 'binding.gyp')); if (hasBindingGyp) { scripts = Object.assign({ install: 'node-gyp rebuild' }, scripts); } @@ -16824,16 +16797,77 @@ class BuildProcess { } } -/* global process, global */ +function resolveExportMapping (mapping, context) { + if (!mapping) throw Error('Cannot resolve undefined mapping!'); + if (typeof mapping === 'string') return mapping; + let adjustedPath; + if (Array.isArray(mapping)) { + for (let subMapping of mapping) { + adjustedPath = resolveExportMapping(subMapping, context); + if (adjustedPath) { + mapping = adjustedPath; + break; + } + } + } + if (typeof mapping === 'object') { + switch (context) { + case 'node-require': adjustedPath = mapping.node || mapping.require || mapping.default; break; + case 'systemjs-browser': + case 'node-import': adjustedPath = mapping.node || mapping.import || mapping.default; break; + case 'module': adjustedPath = mapping.module || mapping.node || mapping.import || mapping.default; break; + default: adjustedPath = mapping.default; + } + return resolveExportMapping(adjustedPath, context); + } + + return adjustedPath; +} + +function resolveImportMapping(name, mapping, context) { + if (!mapping) throw Error('Cannot resolve undefined mapping!'); + mapping = mapping[name]; + if (!mapping) throw Error('Cannot resolve undefined mapping!'); + while (typeof mapping === 'object') { + switch (context) { + case 'node-require': mapping = mapping.node || mapping.require || mapping.default; break; + case 'systemjs-browser': + case 'node-import': mapping = mapping.node || mapping.import || mapping.default; break; + case 'module': mapping = mapping.module || mapping.node || mapping.import || mapping.default; break; + default: mapping = mapping.default; + } + } + if (!mapping) throw Error('Cannot resolve undefined mapping!'); + return mapping; +} -if (!global.fetch) { - Object.assign( - global, - { fetch: fetch$1 }, - ['Response', 'Headers', 'Request'].reduce((all, name) => - Object.assign(all, fetch$1[name]), {})); +function resolveViaImportMap (id, importMap, importer) { + let scope, remapped = importMap.imports?.[id] || null; + if (scope = Object.entries(importMap.scopes || {}) + .filter(([k]) => importer.startsWith(k)) + .sort((a, b) => a[0].length - b[0].length) + .map(([_, scope]) => scope) + .reduce((a, b) => ({ ...a, ...b }), false)) { + if (scope[id]) remapped = scope[id]; + else { + const prefixMapping = Object.keys(scope).find(k => k.endsWith('/') && id.startsWith(k)); + if (prefixMapping) remapped = id.replace(prefixMapping, scope[prefixMapping]); + } + } + if (remapped) { + return remapped; + } } +/* global process, global */ + +// Always use node-fetch since it supports file:// URLs, which native fetch doesn't +Object.assign( + global, + { fetch: fetch$1 }, + ['Response', 'Headers', 'Request'].reduce((all, name) => + Object.assign(all, fetch$1[name]), {})); + const debug = false; function resetPackageMap () { PackageMap._cache = {}; } @@ -16917,7 +16951,7 @@ async function installPackage ( destinationDir = ensurePathFormat(destinationDir); - if (!fs__default["default"].existsSync(destinationDir)) { fs__default["default"].mkdirSync(destinationDir); } + if (!fs.existsSync(destinationDir)) { fs.mkdirSync(destinationDir); } let atIndex = pNameAndVersion.lastIndexOf('@'); if (atIndex === -1) atIndex = pNameAndVersion.length; @@ -17012,12 +17046,12 @@ function addDependencyToPackage ( if (dep) { if (!depVersion || !semver.parse(depVersion, true) || !semver.satisfies(depVersion, depVersionRange, true)) { packageSpec[dependencyField][depName] = depVersionRange; - let config = fs__default["default"].existsSync(path.join(location, 'package.json')) - ? JSON.parse(String(fs__default["default"].readFileSync(path.join(location, 'package.json')))) + let config = fs.existsSync(path.join(location, 'package.json')) + ? JSON.parse(String(fs.readFileSync(path.join(location, 'package.json')))) : { name: depName, version: dep.version }; if (!config[dependencyField]) config[dependencyField] = {}; config[dependencyField][depName] = depVersionRange; - fs__default["default"].writeFileSync(path.join(location, 'package.json'), JSON.stringify(config, null, 2)); + fs.writeFileSync(path.join(location, 'package.json'), JSON.stringify(config, null, 2)); } } return result; @@ -17519,6 +17553,9 @@ exports.npmSearchForVersions = npmSearchForVersions; exports.packageDirsFromEnv = packageDirsFromEnv; exports.parseArgs = _1_2_6; exports.resetPackageMap = resetPackageMap; +exports.resolveExportMapping = resolveExportMapping; +exports.resolveImportMapping = resolveImportMapping; +exports.resolveViaImportMap = resolveViaImportMap; exports.setPackageDirsOfEnv = setPackageDirsOfEnv; exports.tmpdir = tmpdir; exports.untar = untar; diff --git a/flatn/helpers.mjs b/flatn/helpers.mjs new file mode 100644 index 0000000000..37e61956d6 --- /dev/null +++ b/flatn/helpers.mjs @@ -0,0 +1,62 @@ + +export function resolveExportMapping (mapping, context) { + if (!mapping) throw Error('Cannot resolve undefined mapping!'); + if (typeof mapping === 'string') return mapping; + let adjustedPath; + if (Array.isArray(mapping)) { + for (let subMapping of mapping) { + adjustedPath = resolveExportMapping(subMapping, context); + if (adjustedPath) { + mapping = adjustedPath; + break; + } + } + } + if (typeof mapping === 'object') { + switch (context) { + case 'node-require': adjustedPath = mapping.node || mapping.require || mapping.default; break; + case 'systemjs-browser': + case 'node-import': adjustedPath = mapping.node || mapping.import || mapping.default; break; + case 'module': adjustedPath = mapping.module || mapping.node || mapping.import || mapping.default; break; + default: adjustedPath = mapping.default; + } + return resolveExportMapping(adjustedPath, context); + } + + return adjustedPath; +} + +export function resolveImportMapping(name, mapping, context) { + if (!mapping) throw Error('Cannot resolve undefined mapping!'); + mapping = mapping[name]; + if (!mapping) throw Error('Cannot resolve undefined mapping!'); + while (typeof mapping === 'object') { + switch (context) { + case 'node-require': mapping = mapping.node || mapping.require || mapping.default; break; + case 'systemjs-browser': + case 'node-import': mapping = mapping.node || mapping.import || mapping.default; break; + case 'module': mapping = mapping.module || mapping.node || mapping.import || mapping.default; break; + default: mapping = mapping.default; + } + } + if (!mapping) throw Error('Cannot resolve undefined mapping!'); + return mapping; +} + +export function resolveViaImportMap (id, importMap, importer) { + let scope, remapped = importMap.imports?.[id] || null; + if (scope = Object.entries(importMap.scopes || {}) + .filter(([k]) => importer.startsWith(k)) + .sort((a, b) => a[0].length - b[0].length) + .map(([_, scope]) => scope) + .reduce((a, b) => ({ ...a, ...b }), false)) { + if (scope[id]) remapped = scope[id]; + else { + const prefixMapping = Object.keys(scope).find(k => k.endsWith('/') && id.startsWith(k)) + if (prefixMapping) remapped = id.replace(prefixMapping, scope[prefixMapping]); + } + } + if (remapped) { + return remapped; + } +} \ No newline at end of file diff --git a/flatn/index.js b/flatn/index.js index 17850fdd0e..335eb78565 100644 --- a/flatn/index.js +++ b/flatn/index.js @@ -10,14 +10,14 @@ import { packageDownload } from './download.js'; import { PackageMap, PackageSpec } from './package-map.js'; import { BuildProcess } from './build.js'; export * from './util.js'; - -if (!global.fetch) { - Object.assign( - global, - { fetch: node_fetch }, - ['Response', 'Headers', 'Request'].reduce((all, name) => - Object.assign(all, node_fetch[name]), {})); -} +export * from './helpers.mjs'; + +// Always use node-fetch since it supports file:// URLs, which native fetch doesn't +Object.assign( + global, + { fetch: node_fetch }, + ['Response', 'Headers', 'Request'].reduce((all, name) => + Object.assign(all, node_fetch[name]), {})); const debug = false; diff --git a/flatn/module-resolver.js b/flatn/module-resolver.js index 72f6fe4cf4..b27690dbce 100644 --- a/flatn/module-resolver.js +++ b/flatn/module-resolver.js @@ -1,27 +1,35 @@ /* global require,process,__dirname, module */ let path = require('path'); let fs = require('fs'); -let { ensurePackageMap, packageDirsFromEnv } = require('./flatn-cjs.js'); +let { ensurePackageMap, packageDirsFromEnv, resolveExportMapping, resolveViaImportMap } = require('./flatn-cjs.js'); process.execPath = process.argv[0] = path.join(__dirname, 'bin/node'); +const moduleUrlToConfig = new Map(); + /** * Handles the proper base name resolution of @ prefixed package names or * SystemJS specific import mappings. * @param { string } request - The module reference. * @param { object } config - The package config the module belongs to. - * @param { 'node-import'|'node-require'|'system-browser'|'system-node' } context - The resolution context. + * @param { 'node-import'|'node-require'|'systemjs-browser'|'systemjs-node' } context - The resolution context. */ -function resolveBaseName (request, config, context) { +function resolveBaseName (request, config, context, importer) { let map; let baseName = request; if (context.startsWith('systemjs-') && (map = config.systemjs?.map)) { const envName = context === 'systemjs-node' ? 'node' : '~node'; let remapping; if (remapping = map[request]?.[envName] || map[request]) { - baseName = remapping; + if (typeof remapping === 'string') + baseName = remapping; } } - if (baseName.match(/^https?\:\/\//)) return baseName; + if (context.startsWith('systemjs-') && (map = config.systemjs?.importMap)) { + const remapping = resolveViaImportMap(baseName, map, importer); + if (remapping) + baseName = remapping; + } + if (baseName.match(/^https|esm?\:\/\//)) return baseName; if (baseName.startsWith('@')) return baseName.split('/').slice(0, 2).join('/'); return baseName.split('/')[0]; } @@ -56,9 +64,17 @@ function traverseUntilPkgDir (modulePath, cb) { */ function findPackageConfig (modulePath) { let configs = []; + if (moduleUrlToConfig.has(modulePath)) { + return moduleUrlToConfig.get(modulePath); + } traverseUntilPkgDir(modulePath, (dir) => { + let config; if (fs.existsSync(path.join(dir, 'package.json'))) { - configs.push(JSON.parse(fs.readFileSync(path.join(dir, 'package.json')))); + config = JSON.parse(fs.readFileSync(path.join(dir, 'package.json'))); + configs.push(config); + } + if (config && fs.existsSync(path.join(dir, '.cachedImportMap.json'))) { + config.systemjs = { ...config.systemjs, importMap: JSON.parse(fs.readFileSync(path.join(dir, '.cachedImportMap.json'))) }; } }); return configs.reduce(function (configA, configB) { @@ -75,31 +91,6 @@ function depMap (packageConfig) { }, {}); } -function resolveExportMapping(mapping, context) { - if (!mapping) throw Error('Cannot resolve undefined mapping!'); - if (typeof mapping === 'string') return mapping; - let adjustedPath; - if (Array.isArray(mapping)) { - for (let subMapping of mapping) { - adjustedPath = resolveExportMapping(subMapping, context); - if (adjustedPath) { - mapping = adjustedPath; - break; - } - } - } - if (typeof mapping === 'object') { - switch (context) { - case 'node-require': adjustedPath = mapping.node || mapping.require || mapping.default; break; - case 'node-import': adjustedPath = mapping.node || mapping.import || mapping.default; break; - default: adjustedPath = mapping.default; - } - return resolveExportMapping(adjustedPath, context); - } - - return adjustedPath; -} - /** * Given {name, version, path} from resolveFlatPackageToModule, will find the * full path to the module inside of the package, using the module request. @@ -177,7 +168,7 @@ function findModuleInPackage (requesterPackage, basename, request, context) { function flatnResolve (request, parentId = '', context = 'node') { let config = findPackageConfig(parentId); let deps = config ? depMap(config) : {}; - let basename = resolveBaseName(request, config, context); + let basename = resolveBaseName(request, config, context, parentId); let { packageCollectionDirs, individualPackageDirs, devPackageDirs } = packageDirsFromEnv(); let packageMap = ensurePackageMap(packageCollectionDirs, individualPackageDirs, devPackageDirs); let packageFound = packageMap.lookup(basename, deps[basename]) || @@ -194,8 +185,13 @@ function flatnResolve (request, parentId = '', context = 'node') { } if (basename === '@empty') return '@empty'; - if (basename && basename.match(/^https?\:\/\//)) return basename; - if (resolved) return resolved; + if (basename && basename.match(/^(https|esm)?\:\/\//)) { + if (config && !moduleUrlToConfig.has(basename)) moduleUrlToConfig.set(basename, config); + return basename; + } + if (resolved) { + return resolved; + } process.env.FLATN_VERBOSE && console.error(`Failing to require "${request}" from ${parentId}`); return null; } @@ -206,4 +202,4 @@ function findPackagePathForModule (modulePath) { return dir; } -module.exports = { flatnResolve, findModuleInPackage, depMap, findPackageConfig, findPackagePathForModule }; +module.exports = { flatnResolve, findModuleInPackage, depMap, findPackageConfig, findPackagePathForModule, resolveViaImportMap }; diff --git a/flatn/package.json b/flatn/package.json index 7f143ff889..22c42589a8 100644 --- a/flatn/package.json +++ b/flatn/package.json @@ -17,7 +17,7 @@ "node-gyp": "9.0.0", "babel-plugin-transform-es2015-modules-systemjs": "^6.19.0", "node-fetch": "3.2.10", - "rollup": "2.68.0", + "@rollup/wasm-node": "4.27.3", "@rollup/plugin-commonjs": "22.0.0", "minimist": "1.2.6", "semver": "5.7.2", diff --git a/flatn/tools/build-cjs.mjs b/flatn/tools/build-cjs.mjs index 3d89dd4a9d..5323691f81 100644 --- a/flatn/tools/build-cjs.mjs +++ b/flatn/tools/build-cjs.mjs @@ -1,5 +1,5 @@ /* global global, process */ -import { rollup } from 'rollup'; +import { rollup } from '@rollup/wasm-node'; import commonjs from '@rollup/plugin-commonjs'; import { flatnResolve } from '../module-resolver.js'; diff --git a/install.sh b/install.sh index 6a2293c2da..bbe952b6ee 100755 --- a/install.sh +++ b/install.sh @@ -51,7 +51,7 @@ else fi # set the options for all of the following node invocations -export NODE_OPTIONS="--no-experimental-fetch --no-warnings --experimental-modules --loader $lv_next_dir/flatn/resolver.mjs"; +export NODE_OPTIONS="--no-warnings --experimental-modules --loader $lv_next_dir/flatn/resolver.mjs"; node lively.installer/install-with-node.js $PWD \ @@ -64,7 +64,7 @@ fi if [ -z "${CI}" ]; then - env CI=true npm --prefix $lv_next_dir/lively.freezer/ run build-landing-page + env CI=true npm --prefix $lv_next_dir/lively.freezer/ run build-unified +else + env CI=true npm --prefix $lv_next_dir/lively.freezer/ run build-loading-screen fi - -env CI=true npm --prefix $lv_next_dir/lively.freezer/ run build-loading-screen diff --git a/lively.2lively/package.json b/lively.2lively/package.json index 277e545e2b..81868adc0f 100644 --- a/lively.2lively/package.json +++ b/lively.2lively/package.json @@ -26,7 +26,6 @@ "systemjs": { "map": { "socket.io-client": { - "~node": "esm://cache/socket.io-client@4.4.1", "node": "@empty" } }, diff --git a/lively.2lively/tests/l2l-test.js b/lively.2lively/tests/l2l-test.js index 036e266799..55842283e8 100644 --- a/lively.2lively/tests/l2l-test.js +++ b/lively.2lively/tests/l2l-test.js @@ -244,8 +244,8 @@ describe('l2l', function () { tracker.addService('test', (tracker, msg, ackFn, sender) => { /* nothing */ }); tracker.addService('test-2', (tracker, msg, ackFn, sender) => { ackFn('OK'); }); let answer = await client1.sendToAndWait(tracker.id, 'test', {}); - expect(answer).deep.property('data.isError'); - expect(answer).deep.property('data.error').match(/timeout/i); + expect(answer.data).to.have.property('isError'); + expect(answer.data).to.have.property('error').match(/timeout/i); expect(await client1.sendToAndWait(tracker.id, 'test-2', {})).property('data', 'OK'); }); diff --git a/lively.ast/lib/acorn-decorators.mjs b/lively.ast/lib/acorn-decorators.mjs new file mode 100644 index 0000000000..a7b1c590fc --- /dev/null +++ b/lively.ast/lib/acorn-decorators.mjs @@ -0,0 +1,114 @@ +import * as acorn from 'acorn'; + +const getAcorn = Parser => { + if (Parser.acorn) return Parser.acorn; + + if (acorn.version.indexOf('6.') !== 0 && acorn.version.indexOf('6.0.') === 0 && acorn.version.indexOf('7.') !== 0) { + throw new Error(`acorn-private-class-elements requires acorn@^6.1.0 or acorn@7.0.0, not ${acorn.version}`); + } + + // Make sure `Parser` comes from the same acorn as we `require`d, + // otherwise the comparisons fail. + for (let cur = Parser; cur && cur !== acorn.Parser; cur = Object.getPrototypeOf(cur)) { + if (cur !== acorn.Parser) { + throw new Error('acorn-private-class-elements does not support mixing different acorn copies'); + } + } + return acorn; +}; + +/* rms 11.04.22: Mostly derived from from https://github.com/angelozerr/acorn-es7/blob/master/acorn-es7.js and converted to ESM format */ + +/** + * Plugin that enables AcornJS to parse javascript decorators. + * These can be applied to class definitions or field within + * class definitions. + */ +export default function extendParser (Parser) { + // Only load this plugin once. + if (Parser.prototype._parseDecorator) { + return Parser; + } + + const acorn = getAcorn(Parser); + + class DecoratorParser extends Parser { + _parseDecorator () { + let node = this.startNode(); + this.next(); + node.expression = this.parseMaybeUnary(); + return this.finishNode(node, 'Decorator'); + } + + // Parse @ token + getTokenFromCode (code) { + if (code === 64) { + ++this.pos; return this.finishToken(this.decoratorIdentifier); + } + return super.getTokenFromCode(code); + } + + /** + * Override statement parsing to handle top level decorators + * that are attached to class definitions. Currently decorators + * can ONLY be attached to classes. + */ + parseStatement (declaration, topLevel, exports) { + let decorators = []; + switch (this.type) { + case this.decoratorIdentifier: + while (this.type === this.decoratorIdentifier) { + decorators.push(this._parseDecorator()); + } + if (this.type !== acorn.tokTypes._class) { + this.raise(this.start, 'Leading decorators must be attached to a class declaration'); + } + case acorn.tokTypes._class: + if (declaration) this.unexpected(); + let node = super.parseStatement(declaration, topLevel, exports); + node.decorators = decorators; + if (decorators.length) { + node.start = node.decorators[0].start; + } + return node; + } + return super.parseStatement(declaration, topLevel, exports); + } + + parseClass (node, isStatement) { + node = super.parseClass(node, isStatement); + node.decorators = []; + return node; + } + + /** + * Override parsing of class elements to handle decorators + * that are attached to properties or methods. + * @param { boolean } constructorAllowsSuper - Wether or not super is permitted. + */ + parseClassElement (constructorAllowsSuper) { + let decorators = []; + switch (this.type) { + case this.decoratorIdentifier: + while (this.type === this.decoratorIdentifier) { + decorators.push(this._parseDecorator()); + } + if (this.type !== acorn.tokTypes.name && + this.type !== acorn.tokTypes.star) { + this.raise(this.start, 'Inline decorators must be attached to a property declaration'); + } + default: + if (this.type !== this.privateIdentifierToken && !Object.values(acorn.tokTypes).includes(this.type)) break; + let node = super.parseClassElement(constructorAllowsSuper); + node.decorators = decorators; + if (decorators.length) { + node.start = node.decorators[0].start; + } + return node; + } + return super.parseClassElement(constructorAllowsSuper); + } + } + DecoratorParser.prototype.decoratorIdentifier = new acorn.TokenType('decorator-Identifier'); + return DecoratorParser; +} diff --git a/lively.ast/lib/acorn-extension.js b/lively.ast/lib/acorn-extension.js index 67e45b2068..a6059643cb 100644 --- a/lively.ast/lib/acorn-extension.js +++ b/lively.ast/lib/acorn-extension.js @@ -1,6 +1,6 @@ import { obj, arr, string, Path } from 'lively.lang'; import { withMozillaAstDo } from './mozilla-ast-visitor-interface.js'; -import _Decorators from './acorn-decorators.cjs'; +import Decorators from './acorn-decorators.mjs'; import _ClassFields from 'acorn-class-fields'; import _StaticClassFeatures from 'acorn-static-class-features'; import _PrivateMethods from 'acorn-private-methods'; @@ -21,17 +21,15 @@ const isNode = typeof System !== 'undefined' ? System.get('@system-env').node : false; -let Decorators, ClassFields, StaticClassFeatures, PrivateMethods; +let ClassFields, StaticClassFeatures, PrivateMethods; if (isNode) { // we need to utilize the native require here to bypass the source transform of the class // we can not use the native import, since that is asynchronous. // top level import is causing class instrumentation - Decorators = _ClassFields ? _Decorators : System._nodeRequire('lively.ast/lib/acorn-decorators.cjs'); ClassFields = _ClassFields || System._nodeRequire('acorn-class-fields'); StaticClassFeatures = _StaticClassFeatures || System._nodeRequire('acorn-static-class-features'); PrivateMethods = _PrivateMethods || System._nodeRequire('acorn-private-methods'); } else { - Decorators = _Decorators; ClassFields = _ClassFields; StaticClassFeatures = _StaticClassFeatures; PrivateMethods = _PrivateMethods; diff --git a/lively.ast/lib/query.js b/lively.ast/lib/query.js index 8635b8b262..473162596c 100644 --- a/lively.ast/lib/query.js +++ b/lively.ast/lib/query.js @@ -495,31 +495,23 @@ function imports (scope) { return imports; } -function exports (scope, resolve = false) { - if (resolve) resolveReferences(scope); - - const exports = []; - for (const node of scope.exportDecls) { - var exportsStmt = statementOf(scope.node, node); - if (!exportsStmt) continue; - - var from = exportsStmt.source ? exportsStmt.source.value : null; +function handleExportStmt (exportsStmt, scope, node = exportsStmt) { + var from = exportsStmt.source ? exportsStmt.source.value : null; if (exportsStmt.type === 'ExportAllDeclaration') { - exports.push({ + return [{ local: null, exported: '*', imported: '*', fromModule: from, node: node, type: 'all' - }); - continue; + }]; } if (exportsStmt.type === 'ExportDefaultDeclaration') { if (helpers.isDeclaration(exportsStmt.declaration)) { - exports.push({ + return [{ local: exportsStmt.declaration.id ? exportsStmt.declaration.id.name : null, exported: 'default', type: exportsStmt.declaration.type === 'FunctionDeclaration' @@ -531,13 +523,12 @@ function exports (scope, resolve = false) { node: node, decl: exportsStmt.declaration, declId: exportsStmt.declaration.id - }); - continue; + }]; } if (exportsStmt.declaration.type === 'Identifier') { const { decl, declId } = scope.resolvedRefMap.get(exportsStmt.declaration) || {}; - exports.push({ + return [{ local: exportsStmt.declaration.name, exported: 'default', fromModule: null, @@ -545,12 +536,11 @@ function exports (scope, resolve = false) { type: 'id', decl, declId - }); - continue; + }] } // exportsStmt.declaration is an expression - exports.push({ + return [{ local: null, exported: 'default', fromModule: null, @@ -558,12 +548,11 @@ function exports (scope, resolve = false) { type: 'expr', decl: exportsStmt.declaration, declId: exportsStmt.declaration - }); - continue; + }]; } if (exportsStmt.specifiers && exportsStmt.specifiers.length) { - exports.push(...exportsStmt.specifiers.map(exportSpec => { + return exportsStmt.specifiers.map(exportSpec => { let decl, declId; if (from) { // "export { x as y } from 'foo'" is the only case where export @@ -585,12 +574,11 @@ function exports (scope, resolve = false) { decl, declId }; - })); - continue; + }) } if (exportsStmt.declaration && exportsStmt.declaration.declarations) { - exports.push(...exportsStmt.declaration.declarations.map(decl => { + return exportsStmt.declaration.declarations.map(decl => { return { local: decl.id ? decl.id.name : 'default', exported: decl.id ? decl.id.name : 'default', @@ -600,12 +588,11 @@ function exports (scope, resolve = false) { decl: decl, declId: decl.id }; - })); - continue; + }) } if (exportsStmt.declaration) { - exports.push({ + return [{ local: exportsStmt.declaration.id ? exportsStmt.declaration.id.name : 'default', exported: exportsStmt.declaration.id ? exportsStmt.declaration.id.name : 'default', type: exportsStmt.declaration.type === 'FunctionDeclaration' @@ -617,9 +604,21 @@ function exports (scope, resolve = false) { node: node, decl: exportsStmt.declaration, declId: exportsStmt.declaration.id - }); - continue; + }] } + + return []; +} + +function exports (scope, resolve = false) { + if (resolve) resolveReferences(scope); + + const exports = []; + for (const node of scope.exportDecls) { + var exportsStmt = statementOf(scope.node, node); + if (!exportsStmt) continue; + + exports.push(...handleExportStmt(exportsStmt, scope, node)); } return arr.uniqBy(exports, (a, b) => @@ -665,5 +664,6 @@ export { refWithDeclAt, imports, exports, + handleExportStmt, queryNodes }; diff --git a/lively.ast/package.json b/lively.ast/package.json index 3a66dba7fd..9b0e94197d 100644 --- a/lively.ast/package.json +++ b/lively.ast/package.json @@ -32,12 +32,12 @@ "@javascript-obfuscator/estraverse": "5.4.0", "astq": "2.7.5", "escodegen": "2.0.0", - "acorn": "8.0.4", - "acorn-walk": "8.2.0", - "acorn-loose": "8.3.0", - "acorn-private-methods": "1.0.0", "acorn-class-fields": "1.0.0", - "acorn-static-class-features": "1.0.0" + "acorn-loose": "8.0.2", + "acorn-private-methods": "1.0.0", + "acorn-static-class-features": "1.0.0", + "acorn-walk": "8.2.0", + "acorn": "8.0.4" }, "devDependencies": { "acorn": "8.0.4", @@ -58,40 +58,18 @@ "systemjs": { "main": "index.js", "map": { - "escodegen": { - "~node": "esm://cache/escodegen@2.0.0" - }, - "acorn-walk": { - "~node": "esm://cache/acorn-walk@8.2.0" - }, - "acorn-loose": { - "~node": "esm://cache/acorn-loose@8.3.0" - }, - "acorn": { - "~node": "esm://cache/acorn@8.0.4" - }, - "esutils": { - "~node": "esm://cache/esutils@2.0.3" - }, "astq": { - "~node": "esm://cache/astq@2.7.5", "node": "@empty" }, "acorn-private-methods": { - "~node": "esm://cache/acorn-private-methods@1.0.0", "node": "@empty" }, "acorn-class-fields": { - "~node": "esm://cache/acorn-class-fields@1.0.0", "node": "@empty" }, "acorn-static-class-features": { - "~node": "esm://cache/acorn-static-class-features@1.0.0", "node": "@empty" }, - "@javascript-obfuscator/estraverse": { - "~node": "esm://cache/@javascript-obfuscator/estraverse@5.4.0" - }, "util": { "node": "@node/util", "~node": "@empty" diff --git a/lively.classes/class-to-function-transform.js b/lively.classes/class-to-function-transform.js index 88b9462d99..0e328c9bce 100644 --- a/lively.classes/class-to-function-transform.js +++ b/lively.classes/class-to-function-transform.js @@ -344,7 +344,12 @@ function replaceClass (node, state, options) { : classId ? [n.varDecl(classId, constructorTemplate(classId.name, fields, options)), n.varDecl(n.id(tempLivelyClassVar), classId)] : [n.varDecl(n.id(tempLivelyClassVar), constructorTemplate(null, fields, options))], - n.ifStmt(n.funcCall(n.member(n.id('Object'), n.id('isFrozen')), [n.id(tempLivelyClassHolderVar)]), n.block([n.returnStmt(n.id(tempLivelyClassVar))]), null), + n.ifStmt( + n.logicalExpr('||', + n.funcCall(n.member(n.id('Object'), n.id('isFrozen')), [n.id(tempLivelyClassHolderVar)]), + n.funcCall(n.member(n.id('Object'), n.id('isFrozen')), [n.member(n.id(tempLivelyClassVar), n.id('prototype'))])), + n.block([n.returnStmt(n.id(tempLivelyClassVar))]), + null), n.returnStmt( n.funcCall( options.functionNode, diff --git a/lively.classes/index.js b/lively.classes/index.js index 160e015c9a..6a53423efd 100644 --- a/lively.classes/index.js +++ b/lively.classes/index.js @@ -1,4 +1,4 @@ import * as runtime from './runtime.js'; -import { classToFunctionTransform } from './class-to-function-transform.js'; +import { classToFunctionTransform, classToFunctionTransformBabel } from './class-to-function-transform.js'; -export { runtime, classToFunctionTransform }; +export { runtime, classToFunctionTransform, classToFunctionTransformBabel }; diff --git a/lively.classes/package.json b/lively.classes/package.json index 0aad86150a..65698067e6 100644 --- a/lively.classes/package.json +++ b/lively.classes/package.json @@ -2,7 +2,6 @@ "name": "lively.classes", "version": "0.1.20", "description": "EcmaScript 6 classes for live development", - "main": "dist/lively.classes.js", "type": "module", "systemjs": { "main": "index.js" @@ -28,7 +27,7 @@ }, "homepage": "https://github.com/LivelyKernel/lively.classes", "dependencies": { - "rollup": "2.68.0", + "@rollup/wasm-node": "4.27.3", "@rollup/plugin-babel": "5.3.1", "@rollup/plugin-json": "6.0.0", "rollup-plugin-polyfill-node": "0.9.0" diff --git a/lively.classes/runtime.js b/lively.classes/runtime.js index 8a5f17b099..cda84afe9b 100644 --- a/lively.classes/runtime.js +++ b/lively.classes/runtime.js @@ -5,7 +5,7 @@ import { setPrototypeOf } from 'lively.lang/object.js'; import { isNativeFunction } from 'lively.lang/function.js'; const constructorArgMatcher = /\([^\\)]*\)/; -const NEW_ONLY_CLASSES = [Proxy, Map, WeakMap]; +const NEW_ONLY_CLASSES = [Proxy, Map, WeakMap, Set]; if (typeof HTMLElement !== 'undefined') NEW_ONLY_CLASSES.push(HTMLElement); const defaultPropertyDescriptorForGetterSetter = { @@ -105,6 +105,15 @@ function ensureInitializeStub (superclass) { superclass.prototype[initializeSymbol]) return; let wrappedSuperclass; if (NEW_ONLY_CLASSES.includes(superclass)) wrappedSuperclass = wrapNativeClassAsSuper(superclass); + else { + try { + superclass(); + } catch (err) { + if (/class constructor/i.test(err.message)) { + wrappedSuperclass = wrapNativeClassAsSuper(superclass); + } + } + } Object.defineProperty(superclass.prototype, initializeSymbol, { enumerable: false, configurable: true, diff --git a/lively.classes/tests/class-to-function-transform-test.js b/lively.classes/tests/class-to-function-transform-test.js index e5481589c3..fb535e39be 100644 --- a/lively.classes/tests/class-to-function-transform-test.js +++ b/lively.classes/tests/class-to-function-transform-test.js @@ -32,7 +32,7 @@ function classTemplate (className, superClassName, methodString, classMethodStri return this[Symbol.for("lively-instance-initialize")].apply(this, arguments); } };${(useClassHolder || !className) ? '' : '\n var __lively_class__ = Foo;'} - if (Object.isFrozen(__lively_classholder__)) { + if (Object.isFrozen(__lively_classholder__) || Object.isFrozen(__lively_class__.prototype)) { return __lively_class__; } return initializeClass(__lively_class__, superclass, ${methodString}, ${classMethodString}, ${ useClassHolder ? '__lively_classholder__' : 'null'}, ${moduleMeta}${pos}); diff --git a/lively.classes/tools/build-runtime.mjs b/lively.classes/tools/build-runtime.mjs index a96e94a45e..14db332582 100644 --- a/lively.classes/tools/build-runtime.mjs +++ b/lively.classes/tools/build-runtime.mjs @@ -1,5 +1,5 @@ /* global process */ -import { rollup } from 'rollup'; +import { rollup } from '@rollup/wasm-node'; import jsonPlugin from '@rollup/plugin-json'; import { babel } from '@rollup/plugin-babel'; import { lively } from 'lively.freezer/src/plugins/rollup'; diff --git a/lively.classes/tools/build-runtime.sh b/lively.classes/tools/build-runtime.sh index d29c748b21..00f5bdd99d 100755 --- a/lively.classes/tools/build-runtime.sh +++ b/lively.classes/tools/build-runtime.sh @@ -2,4 +2,4 @@ . ../scripts/lively-next-env.sh lively_next_env "$(dirname "$(pwd)")" export FLATN_DEV_PACKAGE_DIRS=$FLATN_DEV_PACKAGE_DIRS:$(pwd); -node --no-experimental-fetch --no-warnings --experimental-import-meta-resolve --experimental-loader ../flatn/resolver.mjs ./tools/build-runtime.mjs +node --no-warnings --experimental-import-meta-resolve --experimental-loader ../flatn/resolver.mjs ./tools/build-runtime.mjs diff --git a/lively.components/package.json b/lively.components/package.json index ad300ad8ef..ea9ecc70ba 100644 --- a/lively.components/package.json +++ b/lively.components/package.json @@ -10,20 +10,14 @@ "lively.halos": "https://github.com/LivelyKernel/lively.halos", "svg-intersections": "0.2.5", "kld-intersections": "^0.4.2", - "lite-youtube-embed": "0.3.0" + "lite-youtube-embed": "0.3.0", + "bowser": "1.4.1", + "three": "0.126" }, "devDependencies": { "mocha-es6": "*" }, "scripts": { "test": "mocha-es6 tests/*-test.js" - }, - "systemjs": { - "map": { - "bowser": "esm://cache/bowser@1.4.1", - "svg-intersections": "esm://cache/svg-intersections", - "kld-intersections": "esm://cache/kld-intersections", - "lite-youtube-embed": "esm://cache/lite-youtube-embed@0.3.0" - } } } \ No newline at end of file diff --git a/lively.components/webgl-canvas.js b/lively.components/webgl-canvas.js index 579a7f945f..731146df95 100644 --- a/lively.components/webgl-canvas.js +++ b/lively.components/webgl-canvas.js @@ -1,4 +1,4 @@ -import THREE from 'esm://cache/three@0.126'; +import THREE from 'three'; import { Canvas } from './canvas.js'; const { Scene, SpotLight, MeshBasicMaterial, BackSide, Color, MeshPhongMaterial, SphereGeometry, Mesh, AmbientLight, WebGLRenderer, PerspectiveCamera, TextureLoader } = THREE; diff --git a/lively.freezer/index.js b/lively.freezer/index.js index c889f40eae..ab5e7dec3e 100644 --- a/lively.freezer/index.js +++ b/lively.freezer/index.js @@ -152,7 +152,7 @@ export async function bundlePart (partOrSnapshot, { requester, useTerser }) { - const jsonPlugin = await System.import('esm://cache/@rollup/plugin-json'); + const jsonPlugin = await System.import('@rollup/plugin-json'); const snapshot = partOrSnapshot.isMorph ? await createMorphSnapshot(partOrSnapshot, { frozenSnapshot: true @@ -195,7 +195,7 @@ export async function bundleModule (moduleId, { htmlConfig, useTerser }) { - const { default: jsonPlugin } = await System.import('esm://cache/@rollup/plugin-json'); + const { default: jsonPlugin } = await System.import('@rollup/plugin-json'); // fixme: maybe its better to make the plugin devoid of state...? const bundle = await rollup({ input: moduleId, @@ -218,7 +218,7 @@ export async function bundleModule (moduleId, { } export async function jspmCompile (url, out, globalName, redirect = {}) { - const jsonPlugin = await System.import('esm://cache/@rollup/plugin-json'); + const jsonPlugin = await System.import('@rollup/plugin-json'); const freezerPlugin = lively({ includePolyfills: false, redirect, @@ -240,7 +240,7 @@ export async function jspmCompile (url, out, globalName, redirect = {}) { } export async function bootstrapLibrary (url, out, asBrowserModule = true, globalName) { - const jsonPlugin = await System.import('esm://cache/@rollup/plugin-json'); + const jsonPlugin = await System.import('@rollup/plugin-json'); const bundle = await rollup({ input: url, plugins: [ diff --git a/lively.freezer/package.json b/lively.freezer/package.json index 832c6f1b8d..741c101598 100644 --- a/lively.freezer/package.json +++ b/lively.freezer/package.json @@ -3,52 +3,41 @@ "version": "0.1.0", "type": "module", "dependencies": { - "@babel/core": "^7.12.3", + "css": "3.0.0", + "bowser": "1.4.1", + "@babel/core": "7.26.10", + "@babel/types": "7.26.0", "@babel/cli": "^7.12.1", - "@babel/preset-env": "^7.12.1", + "@babel/preset-env": "7.27.1", "@babel/plugin-transform-runtime": "^7.12.1", "babel-standalone": "^6.21.1-0", "uglify-es": "^3.3.9", "google-closure-compiler-linux": "^20200927.0.0", "google-closure-compiler-osx": "^20200927.0.0", - "wasm-brotli": "1.0.2", - "rollup": "4.27.3", + "wasm-brotli": "2.0.2", + "@rollup/wasm-node": "4.27.3", + "@rollup/browser": "4.27.3", "@rollup/rollup-linux-x64-gnu": "4.27.3", "@rollup/rollup-darwin-arm64": "4.27.3", - "css": "3.0.0", + "@rollup/plugin-commonjs": "22.0.0", "@rollup/plugin-babel": "5.3.1", "@rollup/plugin-json": "6.0.0", - "rollup-plugin-polyfill-node": "0.9.0" + "rollup-plugin-polyfill-node": "0.9.0", + "js-untar": "2.0.0" }, "scripts": { "build-loading-screen": "./tools/build-loading-screen.sh", - "build-landing-page": "./tools/build-landing-page.sh" + "build-landing-page": "./tools/build-landing-page.sh", + "build-unified": "./tools/build-unified.sh", + "build": "./tools/build-unified.sh" }, "systemjs": { - "meta": { - "https://unpkg.com/wasm-flate@0.1.11-alpha/dist/bootstrap.js": { - "format": "global" - } - }, "map": { - "wasm-flate": "https://unpkg.com/wasm-flate@0.1.11-alpha/dist/bootstrap.js", - "semver": "esm://cache/semver", - "rollup": "esm://cache/rollup@2.68.0", - "@rollup/plugin-json": "esm://cache/@rollup/plugin-json", - "bowser": "esm://cache/bowser@1.4.1", - "@rollup/plugin-commonjs": { - "~node": "esm://cache/@rollup/plugin-commonjs" - }, - "rollup-plugin-polyfill-node": "esm://cache/rollup-plugin-polyfill-node", - "flatn/resolver.mjs": { - "~node": "@empty" - }, - "node:module": { - "~node": "@empty" + "rollup": { + "node": "@rollup/wasm-node", + "~node": "@rollup/browser" }, - "zlib": { - "~node": "@empty" - } + "zlib": { "~node": "@empty" } } }, "lively": { diff --git a/lively.freezer/src/bundler.js b/lively.freezer/src/bundler.js index 8bdbf3ab32..7d79567e3e 100644 --- a/lively.freezer/src/bundler.js +++ b/lively.freezer/src/bundler.js @@ -1,4 +1,6 @@ /* global process */ +import babel from '@babel/core'; +import t from '@babel/types'; import { resource } from 'lively.resources'; import * as ast from 'lively.ast'; import * as classes from 'lively.classes'; @@ -10,6 +12,15 @@ import { ensureComponentDescriptors, replaceExportedNamespaces } from 'lively.source-transform'; +import { + ensureComponentDescriptors as babel_ensureComponentDescriptors, + replaceExportedVarDeclarations as babel_replaceExportedVarDeclarations, + replaceExportedNamespaces as babel_replaceExportedNamespaces, + replaceImportedNamespaces as babel_replaceImportedNamespaces, + rewriteToCaptureTopLevelVariables as babel_rewriteToCaptureTopLevelVariables, + getScopeFromPath, + babelNodes +} from 'lively.source-transform/babel/plugin.js'; import { rewriteToCaptureTopLevelVariables, insertCapturesForFunctionDeclarations, @@ -23,10 +34,13 @@ import { brotli, ROOT_ID, generateLoadHtml, + generateLoadHtmlForEntry, instrumentStaticSystemJS, compileOnServer } from './util/helpers.js'; import { joinPath, ensureFolder } from 'lively.lang/string.js'; +import { resolveViaImportMap } from 'flatn/helpers.mjs'; + const separator = `__${'Separator'}__`; // obscure formatting to prevent breaking builds when this files in included @@ -49,7 +63,7 @@ const CLASS_INSTRUMENTATION_MODULES = [ 'https://jspm.dev/npm:rollup@2.28.2' // this contains a bunch of class definitions which right now screws up the closure compiler ]; -const ESM_CDNS = ['jspm.dev', 'jspm.io', 'skypack.dev', 'esm://cache', 'esm://run', /esm:\/\/([^\/]*)\//]; +const ESM_CDNS = [/esm:\/\/([^\/]*)\//]; // fixme: Why is a blacklist nessecary if there is a whitelist? const CLASS_INSTRUMENTATION_MODULES_EXCLUSION = ['lively.lang']; @@ -57,44 +71,88 @@ const CLASS_INSTRUMENTATION_MODULES_EXCLUSION = ['lively.lang']; // rsm 30.7.24: There are packages which when bundled will crash any build reliably. We exclude these from any build by default. const ALWAYS_EXCLUDED_MODULES = ['mermaid-it-markdown']; -const ADVANCED_EXCLUDED_MODULES = [ - 'lively.ast', - 'lively.vm', - 'lively.ide', - 'lively.modules', - 'babel-plugin-transform-jsx', - 'lively-system-interface', - 'lively.storage', - 'lively.collab', - 'localconfig.js' -]; - const baseURL = typeof System !== 'undefined' ? System.baseURL : ensureFolder(process.env.lv_next_dir || process.cwd()); -export function bulletProofNamespaces (code) { +export function bulletProofNamespaces (code, chunkFileName, isResurrectionBuild, sourceMap = false) { + if (sourceMap) { + let { code: transformedCode, map } = babel.transform(code, { + sourceMaps: true, + comments: true, + compact: true, + plugins: [() => ({ + visitor: { + Program (path) { + const functionBody = path.get('body.0.expression.arguments.1.body') + const [useStrictDirective] = functionBody.get('directives'); + if (useStrictDirective) { + useStrictDirective.remove(); + functionBody.unshiftContainer('body', babel.parse("var __contextModule__ = typeof module !== 'undefined' ? module : arguments[1];").program.body[0]); + } + if (isResurrectionBuild) { + path.get('body.0.expression.callee.object').replaceWith(t.Identifier('BootstrapSystem')); + path.unshiftContainer('body', babel.parse(`BootstrapSystem._currentFile = "${chunkFileName}";`).program.body[0]); + } + }, + VariableDeclaration (path) { + let hasPureComment = false; + for (const declarator of path.node.declarations) { + const init = declarator.init; + if (!init) continue; + const leading = init.leadingComments; + if (Array.isArray(leading)) { + hasPureComment = leading.some(comment => { + return comment.value.trim() === "#__PURE__"; + }); + + if (hasPureComment) { + break; + } + } + } + if (!hasPureComment) return; + try { + const matchingGetter = path.get('declarations.0.init.arguments.0.properties').find(({ node: prop }) => prop?.key?.name === 'default' && prop.kind === 'get'); + const [returnStmt] = matchingGetter.get('body.body'); + if (!returnStmt?.isReturnStatement()) return; + matchingGetter.get('body').unshiftContainer('body', babel.parse(`if (typeof ${returnStmt.node.argument.name} === 'undefined') throw new Error('Module not yet initialized!');`).program.body[0]) + } catch (err) { + + } + } + } + })] + }); + return { code: transformedCode, map } + } + let rewrites = []; let parsed = ast.parse(code, { withComments: true }); const pureComments = parsed.allComments.filter(c => c.text === '#__PURE__'); - if (pureComments.length === 0) return null; - parsed = ast.ReplaceVisitor.run(parsed, (node) => { - if (node.type === 'VariableDeclaration') { - const matchingComment = pureComments.find(c => node.start < c.start && c.end > node.end); - if (!matchingComment) return node; - const matchingGetter = node.declarations[0]?.init?.arguments?.[0]?.properties?.find(prop => prop.key.name === 'default' && prop.kind === 'get'); - if (!matchingGetter) return node; - const getterBody = matchingGetter.value.body; - const [returnStmt] = getterBody.body; - rewrites.push([getterBody, `\nif (typeof ${returnStmt.argument.name} === 'undefined') throw new Error('Module not yet initialized!');\n`]) - } - return node; - }); + if (pureComments.length > 0) { + ast.ReplaceVisitor.run(parsed, (node) => { + if (node.type === 'VariableDeclaration') { + const matchingComment = pureComments.find(c => node.start < c.start && c.end > node.end); + if (!matchingComment) return node; + const matchingGetter = node.declarations[0]?.init?.arguments?.[0]?.properties?.find(prop => prop.key.name === 'default' && prop.kind === 'get'); + if (!matchingGetter) return node; + const getterBody = matchingGetter.value.body; + const [returnStmt] = getterBody.body; + rewrites.push([getterBody, `\nif (typeof ${returnStmt.argument.name} === 'undefined') throw new Error('Module not yet initialized!');\n`]); + } + return node; + }); + } if (rewrites.length > 0) { arr.sortBy(rewrites, ([node]) => node.start).forEach(([node, snippet]) => { code = code.slice(0, node.start + 1) + snippet + code.slice(node.start + 1); }); - return code; } - return null; + if (isResurrectionBuild) { + // this messes up the source map + code = code.replace('System.register', `BootstrapSystem._currentFile = "${chunkFileName}";\nBootstrapSystem.register`); + } + code = code.replace("'use strict';", "var __contextModule__ = typeof module !== 'undefined' ? module : arguments[1];\n"); + return { code }; } /** @@ -151,7 +209,6 @@ function resolutionId (id, importer) { * @returns { boolean } Wether or not the module was served from an ESM CDN. */ function isCdnImport (id, importer, resolver) { - if (ESM_CDNS.find(cdn => id.match(cdn) || importer.match(cdn)) && importer && importer !== ROOT_ID) { const { url } = resource(resolver.ensureFileFormat(importer)).root(); // get the cdn host root return ESM_CDNS.find(cdn => url.match(cdn)); @@ -178,7 +235,8 @@ export default class LivelyRollup { compress = true, minify = true, captureModuleScope = true, - verbose = false + verbose = false, + sourceMap = false }) { this.verbose = verbose; // wether or not to log the warnings to the console that happen during build this.resolver = resolver; // resolves the modules to the respective urls, for either client or browser @@ -195,6 +253,7 @@ export default class LivelyRollup { this.includeLivelyAssets = includeLivelyAssets; // If set to true, will include the default fonts and css from lively.next into the bundle. Disabling this is probably a bad idea. this.compress = compress; // If true, this will perform custom compression of the files to brotli and gzip. this.minify = minify; // If true, will invoke the google closure minification to further reduce source code size. + this.sourceMap = sourceMap; this.globalMap = {}; // accumulates the package -> url mappings that are provided by each of the packages this.modulesWithDynamicLoads = new Set(); // collection of all modules that include System.import() @@ -205,6 +264,8 @@ export default class LivelyRollup { this.customFontFiles = []; this.projectsInBundle = new Set(); this.moduleToPkg = new Map(); + this.moduleSources = {}; + this.entryPaths = null; // for multi-entry builds, stores the original entry paths this.resolver.setStatus({ label: 'Freezing in Progress' }); } @@ -233,11 +294,11 @@ export default class LivelyRollup { * @param { string } path - The relative path to be imported. */ // FIXME: the reason this is async is because we still keep the browser resolver around... - async resolveRelativeImport (moduleId, path) { + resolveRelativeImport (moduleId, path) { if (!path.startsWith('.')) return this.resolver.normalizeFileName(path); // how to achieve that without the nasty file handle - return await this.resolver.normalizeFileName( - string.joinPath(await this.resolver.normalizeFileName(moduleId), '..', path)); + return this.resolver.normalizeFileName( + string.joinPath(this.resolver.normalizeFileName(moduleId), '..', path)); } /** @@ -245,7 +306,7 @@ export default class LivelyRollup { * @param { string } id - The id of the module. */ normalizedId (id) { - return id.replace(baseURL, '').replace('local://lively-object-modules/', '').replace('local_projects/', '').replace('https://jspm.dev/', 'esm://cache/'); + return id.replace(baseURL, '').replace('local://lively-object-modules/', '').replace('local_projects/', ''); } /** @@ -257,40 +318,64 @@ export default class LivelyRollup { */ getTransformOptions (modId, parsedSource) { if (modId === '@empty.js') return {}; + const parsedGlobals = parsedSource.scope?.globals && Object.keys(parsedSource.scope?.globals) || GlobalInjector.getGlobals(null, parsedSource); let version, name; - const pkg = this.resolver.resolvePackage(modId); + const pkg = this.resolver.resolvePackage(modId, this.getResolutionContext()) || this.moduleToPkg.get(modId); if (pkg) { name = pkg.name; version = pkg.version; + } else if (modId.startsWith('esm://')) { + [name, version] = resource(modId).path().slice(1).split('@'); + if (version) version = version.split('/')[0]; } else { // assuming the module comes from jspm version = modId.split('@')[1]; name = modId.split('npm:')[1].split('@')[0]; } - const classToFunction = { - classHolder: ast.parse(`((lively.FreezerRuntime || lively.frozenModules).recorderFor("${this.normalizedId(modId)}", __contextModule__))`), - functionNode: { type: 'Identifier', name: 'initializeES6ClassForLively' }, - transform: classes.classToFunctionTransform, - currentModuleAccessor: ast.parse(`({ - pathInPackage: () => { - return "${this.resolver.pathInPackageFor(modId)}" + const classToFunction = this.sourceMap ? { + classHolder: babel.parse(`((lively.FreezerRuntime || lively.frozenModules).recorderFor("${this.normalizedId(modId)}", __contextModule__))`).program.body[0].expression, + functionNode: t.Identifier('initializeES6ClassForLively'), + transform: (path, options) => { + classes.classToFunctionTransformBabel(path, {}, options); }, - unsubscribeFromToplevelDefinitionChanges: () => () => {}, - subscribeToToplevelDefinitionChanges: () => () => {}, - package: () => { - return { - name: "${name}", - version: "${version}" + nodes: babelNodes, + currentModuleAccessor: babel.parse(`({ + pathInPackage: () => { + return "${this.resolver.pathInPackageFor(modId)}" + }, + unsubscribeFromToplevelDefinitionChanges: () => () => {}, + subscribeToToplevelDefinitionChanges: () => () => {}, + package: () => { + return { + name: "${name}", + version: "${version}" + } } - } - })`).body[0].expression - }; + })`).program.body[0].expression + } : { + classHolder: ast.parse(`((lively.FreezerRuntime || lively.frozenModules).recorderFor("${this.normalizedId(modId)}", __contextModule__))`).body[0].expression, + functionNode: { type: 'Identifier', name: 'initializeES6ClassForLively' }, + transform: classes.classToFunctionTransform, + currentModuleAccessor: ast.parse(`({ + pathInPackage: () => { + return "${this.resolver.pathInPackageFor(modId)}" + }, + unsubscribeFromToplevelDefinitionChanges: () => () => {}, + subscribeToToplevelDefinitionChanges: () => () => {}, + package: () => { + return { + name: "${name}", + version: "${version}" + } + } + })`).body[0].expression + }; return { - captureImports: false, // we do not need to support inline evals within bundled modules, + captureImports: this.sourceMap, // for the babel transform, we need to capture imports as well exclude: [ 'System', '__contextModule__', - ...this.resolver.dontTransform(modId, [...ast.query.knownGlobals, ...GlobalInjector.getGlobals(null, parsedSource)]), + ...this.resolver.dontTransform(modId, [...ast.query.knownGlobals, ...parsedGlobals]), ...arr.range(0, 50).map(i => `__captured${i}__`) ], classToFunction @@ -311,17 +396,33 @@ export default class LivelyRollup { } } + async getRootModuleForEntry (entryPath) { + // Generate a synthetic root module for a specific entry point + // This is used in multi-entry builds where each entry needs its own wrapper + if (!this.autoRun) { + return `export * from "${entryPath}";`; + } + return await this.synthesizeMainModuleForEntry(entryPath); + } + /** * Returns the source code of a synthesized modules that creates plain * morphic world and then calls the configured main method with that * world as the argument. */ async synthesizeMainModule () { - let mainModuleSource = await resource(this.resolver.ensureFileFormat(await this.resolver.normalizeFileName('lively.freezer/src/util/main-module.js'))).read(); + let mainModuleSource = await resource(this.resolver.ensureFileFormat(this.resolver.normalizeFileName('lively.freezer/src/util/main-module.js'))).read(); mainModuleSource = mainModuleSource.replaceAll('TRACE', this.isResurrectionBuild ? 'true' : 'false'); return mainModuleSource.replace('prepare()', `const { main, WORLD_CLASS = World, TITLE } = await System.import('${this.rootModuleId}')`); } + async synthesizeMainModuleForEntry (entryPath) { + // Generate a synthetic main module for a specific entry point + let mainModuleSource = await resource(this.resolver.ensureFileFormat(this.resolver.normalizeFileName('lively.freezer/src/util/main-module.js'))).read(); + mainModuleSource = mainModuleSource.replaceAll('TRACE', this.isResurrectionBuild ? 'true' : 'false'); + return mainModuleSource.replace('prepare()', `const { main, WORLD_CLASS = World, TITLE } = await System.import('${entryPath}')`); + } + /** * Determines if a given module requires the scope to be captured. * Capturing the scope means that the module gets exposed to the static runtime @@ -422,13 +523,31 @@ export default class LivelyRollup { }); } + babel_instrumentDynamicLoads (path) { + const self = this; + path.traverse({ + CallExpression (path) { + if (path.get('callee').type === 'MemberExpression' && path.get('arguments').length === 1) { + const { property, object } = path.get('callee').node; + if (property.name === 'import' && object.name === 'System') { + try { + const resolvedImport = eval(path.get('arguments')[0].getSource()); + if (resolvedImport) self.hasDynamicImports = true; + path.replaceWith(babel.parse(`import("${resolvedImport}")`).program.body[0].expression); + } catch (err) { + } + } + } + } + }); + } + /** * A custom transform() callback for RollupJS. * @param { string } source - The source code of a module. * @param { string } id - The id of the module to be transformed. */ async transform (source, id) { - const originalSource = source; if (id.startsWith('\0') || id.endsWith('.json') || this.excludedModules.find(m => id.startsWith(m))) { return source; } @@ -450,18 +569,64 @@ export default class LivelyRollup { return `projectAsset(\'${newName}\')`; }; - source = source.replaceAll(projectAssetRegex, assetNameRewriter); + source = source.replaceAll(projectAssetRegex, assetNameRewriter); // needs to be performed in a way to preserve sourcemap + } + + // FIXME: here we need to move over to a babel transform and also pass over the sourcemap + // The easiest way to achieve thatis via an inline visitor that basically performs the same steps as below within babel. + + const needsLoadInstrumentation = this.needsDynamicLoadTransform(source); + const self = this; + + if (this.sourceMap) { + + function inlinePlugin () { + return { + visitor: { + Program (path, state) { + const source = self.moduleSources[id]; + if (!source) return; + if (id === ROOT_ID && !needsLoadInstrumentation) return; + if (needsLoadInstrumentation) { + self.babel_instrumentDynamicLoads(path, state, id); + } + if (id === ROOT_ID) return; + // this capturing stuff needs to behave differently when we have dynamic imports. Why?? + const instrumentClasses = self.needsClassInstrumentation(id, source); + if (instrumentClasses || self.needsScopeToBeCaptured(id, null, source)) { + const sourceHash = string.hashCode(source); // why cant we use the original source here? because other plugins already scrambled the code... + self.babel_captureScope(path, id, sourceHash, instrumentClasses); + } + } + } + }; + } + + const { code, map } = babel.transform(source, { + sourceMaps: true, + compact: true, + comments: false, + plugins: [inlinePlugin] + }); + + return { code, map }; + } let parsed = ast.parse(source); - if (this.needsDynamicLoadTransform(source)) { + if (needsLoadInstrumentation) { parsed = this.instrumentDynamicLoads(parsed, id); } - if (id === ROOT_ID) return ast.stringify(parsed); - // this capturing stuff needs to behave differently when we have dynamic imports. Why?? + // Root modules (both single-entry ROOT_ID and multi-entry synthetic modules) + // only need dynamic load instrumentation, not scope capturing + if (id === ROOT_ID || id.startsWith('__rootModule__:')) { + return ast.stringify(parsed); + } + const instrumentClasses = this.needsClassInstrumentation(id, source); + if (this.needsScopeToBeCaptured(id, null, source) || instrumentClasses) { const sourceHash = string.hashCode(await this.resolver.load(id)); parsed = await this.captureScope(parsed, id, sourceHash, instrumentClasses); @@ -475,9 +640,11 @@ export default class LivelyRollup { * @param { string } id - The module id to be resolved. * @param { string } importer - The module id that is importing said module. */ - async resolveId (id, importer) { + resolveId (id, importer) { if (this.resolved[resolutionId(id, importer)]) return this.resolved[resolutionId(id, importer)]; if (id === ROOT_ID) return id; + // Handle synthetic root modules for multi-entry builds + if (id.startsWith('__rootModule__:')) return id; // handle standalone if (!importer) return this.resolver.resolveModuleId(id, importer, this.getResolutionContext()); @@ -490,46 +657,31 @@ export default class LivelyRollup { } } - const importingPackage = this.resolver.resolvePackage(importer) || this.moduleToPkg.get(importer); + const importingPackage = this.resolver.resolvePackage(importer, this.getResolutionContext()) || this.moduleToPkg.get(importer); // honor the systemjs options within the package config const { map: mapping, importMap } = importingPackage?.systemjs || {}; - if (importMap) { - let remapped; - if (remapped = importMap.imports?.[id]) { - id = remapped; - } - let scope, prefix; - if (scope = Object.entries(importMap.scopes) - .filter(([k, v]) => importer.startsWith(k)) - .sort((a, b) => a[0].length - b[0].length) - .map(([prefix, scope]) => scope) - .reduce((a, b) => ({ ...a, ...b }), false)) { - remapped = scope[id]; - } - if (remapped) { - id = remapped; - } - } - this.moduleToPkg.set(id, importingPackage); if (mapping) { - this.globalMap = { ...this.globalMap, ...mapping }; - if (mapping[id] || this.globalMap[id]) { - if (!mapping[id] && this.globalMap[id]) { - console.warn(`[freezer] No mapping for "${id}" provided by package "${importingPackage.name}". Guessing "${this.globalMap[id]}" based on past resolutions. Please consider adding a map entry to this package config in oder to make the package definition sound and work independently of the current setup!`); // eslint-disable-line no-console - } + let remapped = mapping[id]; + if (remapped) { if (this.excludedModules.includes(id)) return id; - let remapped = mapping[id] || this.globalMap[id]; const ctx = this.asBrowserModule ? '~node' : 'node'; if (remapped[ctx]) remapped = remapped[ctx]; if (typeof remapped === 'string') id = remapped; } } + if (importMap) { + let remapped = resolveViaImportMap(id, importMap, importer) + if (remapped) id = remapped; + } + + this.moduleToPkg.set(id, importingPackage); + let absolutePath; if (id.startsWith('.')) { // handle some kind of relative import try { - absolutePath = await this.resolveRelativeImport(importer, id); + absolutePath = this.resolveRelativeImport(importer, id); if (this.belongsToExcludedPackage(absolutePath)) return null; return this.resolved[resolutionId(id, importer)] = absolutePath; } catch (err) { @@ -547,7 +699,7 @@ export default class LivelyRollup { belongsToExcludedPackage (id) { if (id === null) return true; - const pkg = this.resolver.resolvePackage(id); + const pkg = this.resolver.resolvePackage(id, this.getResolutionContext()); if (pkg && this.excludedModules.includes(pkg.name)) { return true; } @@ -579,7 +731,12 @@ export default class LivelyRollup { * @param { string } id - The module id to getch the source code for. * @returns { string } The source code. */ - async load (id) { + + async load(id) { + return this.moduleSources[id] = await this.perform_load(id); + } + + async perform_load (id) { if (this.excludedModules.find(m => id.startsWith(m))) { if (id === 'lively.ast') { return ` @@ -598,7 +755,19 @@ export default class LivelyRollup { const res = await this.getRootModule(); return res; } - const pkg = this.resolver.resolvePackage(id); + + // Handle entry-specific root modules for multi-entry builds + // Format: __rootModule__: + // Look up the actual entry path from the entryPaths mapping + if (id.startsWith('__rootModule__:')) { + const entryPath = this.entryPaths[id]; + if (!entryPath) { + throw new Error(`No entry path found for root module ID: ${id}`); + } + const res = await this.getRootModuleForEntry(entryPath); + return res; + } + const pkg = this.resolver.resolvePackage(id, this.getResolutionContext()); if (pkg && this.excludedModules.includes(pkg.name) && !id.endsWith('.json')) { return ''; @@ -616,7 +785,9 @@ export default class LivelyRollup { async buildStart (plugin) { this.resolver.setStatus({ status: 'Bundling...' }); await this.resolver.whenReady(); - if (this.autoRun) { + // Only emit ROOT_ID chunk for single-entry builds with autoRun + // For multi-entry builds, we use the actual entry points instead + if (this.autoRun && this.rootModuleId) { plugin.emitFile({ type: 'chunk', id: ROOT_ID @@ -749,6 +920,139 @@ export default class LivelyRollup { return instrumented; } + babel_captureScope (path, id, hashCode, instrumentClasses) { + let classRuntimeImport = ''; + const recorderName = '__varRecorder__'; + + const exports = []; + const self = this; + + const scope = { resolvedRefMap: new Map(), decls: [] }; + + Object.values(path.scope.bindings).map(binding => { + let decl = binding.path.node; + if (decl.type === 'ImportSpecifier' || decl.type === 'ImportDefaultSpecifier') decl = binding.path.parent; + scope.decls.push([decl, binding.identifier]); // this data format is just weird af? + binding.referencePaths.forEach(ref => { + scope.resolvedRefMap.set(ref.node, { decl, declId: binding.identifier, ref }); + }); + }); + + path.traverse({ + ExportDeclaration (path) { + for (let exp of ast.query.handleExportStmt(path.node, scope)) { + if (exp.local && exp.exported !== 'default' && exp.exported !== exp.local) { + // retrieve all the exports of the module + exports.push(JSON.stringify('__rename__' + exp.local + '->' + exp.exported)); + continue; + } + if (exp.exported === '*') { + // retrieve all the exports of the module + exports.push(JSON.stringify('__reexport__' + self.normalizedId(self.resolveId(exp.fromModule, id)))); + continue; + } + if (exp.exported === 'default' && exp.local) { + exports.push(JSON.stringify('__default__' + exp.local)); // in order to capture this + } + exports.push(JSON.stringify(exp.exported)); + } + } + }) + + const localLivelyVar = Object.keys(path.scope.references).includes('lively'); + const recorderString = this.captureModuleScope + ? `${localLivelyVar ? GLOBAL_FETCH : ''} const ${recorderName} = (${localLivelyVar ? 'G.' : ''}lively.FreezerRuntime || ${localLivelyVar ? 'G.' : ''}lively.frozenModules).recorderFor("${this.normalizedId(id)}", __contextModule__);\n` + : ''; + const moduleHash = `${recorderName}.__module_hash__ = ${hashCode};\n`; + const moduleExports = `${recorderName}.__module_exports__ = ${recorderName}.__module_exports__ || [${exports.join(',')}];\n`; + const captureObj = t.Identifier(recorderName); + const opts = this.getTransformOptions(this.resolver.resolveModuleId(id), path); + const currentModuleAccessor = opts.classToFunction.currentModuleAccessor; + + if (instrumentClasses) { + classRuntimeImport = `import { initializeClass as initializeES6ClassForLively } from "${this.isResurrectionBuild ? 'livelyClassesRuntime.js' : 'lively.classes/runtime.js'}";\n`; + } else { + opts.classToFunction = false; + } + + const normalizedId = this.normalizedId(id); + if (this.isComponentModule(id)) { + babel_ensureComponentDescriptors(path, normalizedId, { recorderName }); + } + + let defaultExport = ''; + if (this.captureModuleScope) { + babel_replaceExportedVarDeclarations(path, normalizedId, { recorderName }); + if (this.isResurrectionBuild) { + babel_replaceImportedNamespaces(path, id, this, opts); + babel_replaceExportedNamespaces(path, id, this, opts); + } + path.scope.crawl(); + Object.assign(scope, getScopeFromPath(path)); + babel_rewriteToCaptureTopLevelVariables(path, { + ...opts, + scope, + captureObj, + // declarationWrapper: t.MemberExpression(captureObj, t.StringLiteral(normalizedId + '__define__'), true), + currentModuleAccessor + }); + + const imports = []; + const toBeReplaced = []; + + path.traverse({ + ImportDeclaration (path) { + arr.pushIfNotIncluded(imports, path); + }, + ExportDefaultDeclaration (path) { + let exp; + switch (path.get('declaration').type) { + case 'Literal': + exp = path.get('declaration.raw').node; + break; + case 'Identifier': + exp = path.get('declaration.name').node; + break; + case 'ClassDeclaration': + case 'FunctionDeclaration': + exp = path.get('declaration.id.name').node; + break; + } + if (exp) defaultExport = `${captureObj.name}.default = ${exp};\n`; + } + }); + + for (const stmts of Object.values(arr.groupBy(imports, imp => imp.node.source.value))) { + const toBeMerged = stmts.filter(stmt => stmt.get('specifiers').every(spec => spec.type === 'ImportSpecifier')); + if (toBeMerged.length > 1) { + // merge statements + // fixme: if specifiers are not named, these can not be merged + // fixme: properly handle default export + const mergedSpecifiers = arr.uniqBy( + toBeMerged.map(stmt => stmt.node.specifiers).flat(), + (spec1, spec2) => + spec1.type === 'ImportSpecifier' && + spec2.type === 'ImportSpecifier' && + spec1.imported.name === spec2.imported.name && + spec1.local.name === spec2.local.name + ); + toBeMerged[0].set('specifiers', mergedSpecifiers); + toBeMerged.slice(1).map(stmt => { + stmt.remove(); + }); + } + } + } + + path.unshiftContainer('body', [ + ...babel.parse(recorderString).program.body, + ...babel.parse(this.isResurrectionBuild ? moduleHash + moduleExports : '').program.body, + ...babel.parse(classRuntimeImport).program.body + ]); + + path.pushContainer('body', babel.parse(defaultExport).program.body); + } + /** * Automatically generates a variable name from a module id. * This variable name can be used for storing the module scope @@ -819,9 +1123,9 @@ export default class LivelyRollup { // however that does not allow us to transition to the dynamic lively.modules system // so we can only utilize s.js in case we do not want to resurrect if (this.needsOldSystem) { - code += await resource(this.resolver.ensureFileFormat(await this.resolver.normalizeFileName('lively.freezer/src/util/system.0.21.js'))).read(); + code += await resource(this.resolver.ensureFileFormat(this.resolver.normalizeFileName('lively.freezer/src/util/system.0.21.js'))).read(); } else { - code += await resource(this.resolver.ensureFileFormat(await this.resolver.normalizeFileName('lively.freezer/src/util/system.6.js'))).read(); + code += await resource(this.resolver.ensureFileFormat(this.resolver.normalizeFileName('lively.freezer/src/util/system.6.js'))).read(); } // stub the globals code += `(${instrumentStaticSystemJS.toString()})(System);\n`; @@ -852,9 +1156,9 @@ export default class LivelyRollup { async getRuntimeCode () { const includePolyfills = this.includePolyfills && this.asBrowserModule; - let runtimeCode = await resource(this.resolver.ensureFileFormat(await this.resolver.normalizeFileName('lively.freezer/src/util/runtime.js'))).read(); - const regeneratorSource = await resource(this.resolver.ensureFileFormat(await this.resolver.normalizeFileName('lively.freezer/src/util/regenerator-runtime.js'))).read(); - const polyfills = includePolyfills ? await resource(this.resolver.ensureFileFormat(await this.resolver.normalizeFileName('lively.freezer/deps/fetch.umd.js'))).read() : ''; + let runtimeCode = await resource(this.resolver.ensureFileFormat(this.resolver.normalizeFileName('lively.freezer/src/util/runtime.js'))).read(); + const regeneratorSource = await resource(this.resolver.ensureFileFormat(this.resolver.normalizeFileName('lively.freezer/src/util/regenerator-runtime.js'))).read(); + const polyfills = includePolyfills ? await resource(this.resolver.ensureFileFormat(this.resolver.normalizeFileName('lively.freezer/deps/fetch.umd.js'))).read() : ''; runtimeCode = `(${runtimeCode.slice(0, -1).replace('export ', '')})();\n`; if (!this.hasDynamicImports) { // If there are no dynamic imports, we compile without systemjs and @@ -882,62 +1186,52 @@ export default class LivelyRollup { }, importMap, this.resolver, modules, this.isResurrectionBuild); } + async generateIndexHtmlForEntry (importMap, modules, entryFileName) { + return await generateLoadHtmlForEntry({ + ...this.autoRun || {}, + head: (this.autoRun?.head || '') + this.generateAssetPreloads() + }, importMap, this.resolver, modules, this.isResurrectionBuild, entryFileName); + } + async generateBundle (plugin, bundle, depsCode, importMap, opts) { const modules = Object.values(bundle); - modules.forEach(chunk => { - if (chunk.code) { - if (this.isResurrectionBuild) { - chunk.code = chunk.code.replace('System.register', `BootstrapSystem._currentFile = "${chunk.fileName}";\nBootstrapSystem.register`); + if (this.minify && opts.format !== 'esm' && !this.sourceMap) { + // Parallelize minification per chunk instead of concatenating all chunks + this.resolver.setStatus({ status: 'Minifying chunks in parallel...', progress: 0.5 }); + + await Promise.all(modules.map(async (chunk, i) => { + try { + const { min: minifiedCode } = await compileOnServer(chunk.code, this.resolver, this.useTerser); + chunk.code = minifiedCode.replace("'use strict';", ''); + } catch (err) { + // If minification fails for a chunk, keep the original code + console.warn(`Minification failed for chunk ${i}: ${err.message}`); } - chunk.code = chunk.code.replace("'use strict'", "var __contextModule__ = typeof module !== 'undefined' ? module : arguments[1];\n"); - } - }); - if (this.minify && opts.format !== 'esm') { - modules.forEach((chunk, i) => { - chunk.instrumentedCode = `"${separator}",${i};\n` + chunk.code; - }); - let codeToMinify = modules.map(chunk => chunk.instrumentedCode).join('\n'); - const { min: minfiedCode } = await compileOnServer(codeToMinify, this.resolver, this.useTerser); - let compiledSnippets = minfiedCode.split(new RegExp(`"${separator}";\n?`)); - const adjustedSnippets = new Map(); // ensure order - modules.forEach((snippet, i) => { - adjustedSnippets.set(i, snippet.code); // populate with original source in case the transpiler kicked the chunk away - }); - compiledSnippets.forEach((compiledSnippet, i) => { - const hit = compiledSnippet.match(/^[0-9]+;\n?/); - if (!hit) return; // fixme: google closure seems to add some weird polyfill stuff... - const hint = Number(hit[0].replace(/;\n?/, '')); - adjustedSnippets.set(hint, compiledSnippet.replace(/^[0-9]+;\n?/, '')); - }); - const polyfills = compiledSnippets[0]; - compiledSnippets = [...adjustedSnippets.values()]; - compiledSnippets[0] = polyfills + compiledSnippets[0]; - for (const [snippet, compiled] of arr.zip(modules, compiledSnippets)) { - snippet.code = compiled.replace("'use strict';", ''); - } // override the code attribute + })); } if (this.isResurrectionBuild) { plugin.emitFile({ type: 'asset', fileName: 'livelyClassesRuntime.js', - source: await this.resolver.load(await this.resolver.normalizeFileName('lively.classes/build/runtime.js')) + source: await this.resolver.load(this.resolver.normalizeFileName('lively.classes/build/runtime.js')) }); } if (this.compress) { - for (let chunk of modules) { - plugin.emitFile({ + // Parallelize compression - gzip and brotli for all chunks + await Promise.all(modules.flatMap(chunk => [ + gzip(chunk.code).then(source => plugin.emitFile({ type: 'asset', fileName: chunk.fileName + '.gz', - source: await gzip(chunk.code) - }); - plugin.emitFile({ + source + })), + brotli(chunk.code).then(source => plugin.emitFile({ type: 'asset', fileName: chunk.fileName + '.br', - source: await brotli(chunk.code) - }); - } + source + })) + ])); } const morphicUrl = this.resolver.ensureFileFormat(this.resolver.decanonicalizeFileName('lively.morphic').replace('index.js', '')); @@ -945,27 +1239,35 @@ export default class LivelyRollup { const fontBundleDir = resource(config.css.fontBundle).parent(); const fontFiles = await fontBundleDir.dirList(); - for (let file of fontFiles) { + // Parallelize font file loading + const fontPromises = fontFiles.map(async (file) => { file.beBinary(); let source = await file.read(); - if (source instanceof ArrayBuffer) source = new Uint8Array(source); // this fucks up font files... - plugin.emitFile({ + if (source instanceof ArrayBuffer) source = new Uint8Array(source); + return { type: 'asset', fileName: joinPath(fontBundleDir.url.replace(morphicUrl, ''), file.name()), source - }); - } + }; + }); const assetDir = resource(config.css.fontBundle).parent().parent(); const morphicCSS = assetDir.join('morphic.css'); morphicCSS.beBinary(); - let source = await morphicCSS.read(); - if (source instanceof ArrayBuffer) source = new Uint8Array(source); // this fucks up font files... - plugin.emitFile({ - type: 'asset', - fileName: joinPath(assetDir.url.replace(morphicUrl, ''), 'morphic.css'), - source + + // Load morphic.css in parallel with fonts + const morphicCSSPromise = morphicCSS.read().then(source => { + if (source instanceof ArrayBuffer) source = new Uint8Array(source); + return { + type: 'asset', + fileName: joinPath(assetDir.url.replace(morphicUrl, ''), 'morphic.css'), + source + }; }); + + // Wait for all assets to load and emit them + const allAssets = await Promise.all([...fontPromises, morphicCSSPromise]); + allAssets.forEach(asset => plugin.emitFile(asset)); } const livelyDir = resource(morphicUrl).join('..').withRelativePartsResolved(); @@ -973,32 +1275,55 @@ export default class LivelyRollup { let bundledProjectCSS = ''; let bundledProjectFontCSS = ''; // In contrast to `assets`, we cannot tell which CSS and font files are actually used. We need to collect them for the project to be bundled and all its dependencies. - for (let [project] of this.projectsInBundle.entries()) { + + // Process all projects in parallel + const projectPromises = Array.from(this.projectsInBundle.entries()).map(async ([project]) => { const indexCSSFile = projectsDir.join(project).join('index.css'); - const indexCSSContents = await indexCSSFile.read(); - bundledProjectCSS = indexCSSContents + '\n' + bundledProjectCSS; const fontCSSFile = projectsDir.join(project).join('fonts.css'); - // Inside of the projects, font files are in the assets folder, with font.css being a hierarchy above. - // Inside of the bundles, font.css itself is part of the assets folder. - const fontCSSContents = (await fontCSSFile.read()).replaceAll(/\.\/assets\//g, './'); - bundledProjectFontCSS = fontCSSContents + '\n' + bundledProjectFontCSS; - const assetDir = await projectsDir.join(project).join('assets'); - // Each project can have multiple font files + // Load both CSS files in parallel + const [indexCSSContents, fontCSSContents] = await Promise.all([ + indexCSSFile.read(), + fontCSSFile.read().then(content => content.replaceAll(/\.\/assets\//g, './')) + ]); + + const assetDir = projectsDir.join(project).join('assets'); + let fontAssets = []; + + // Check if asset directory exists and load font files if (await assetDir.exists()) { const fontFiles = (await assetDir.dirList()).filter(f => f.url.includes('woff2')); - this.customFontFiles.push(...fontFiles); - for (let file of fontFiles) { + + // Parallelize font file reading + fontAssets = await Promise.all(fontFiles.map(async (file) => { file.beBinary(); let source = await file.read(); if (source instanceof ArrayBuffer) source = new Uint8Array(source); - plugin.emitFile({ - type: 'asset', - fileName: joinPath('assets', file.name()), - source - }); - } + return { + file, + asset: { + type: 'asset', + fileName: joinPath('assets', file.name()), + source + } + }; + })); } + + return { indexCSSContents, fontCSSContents, fontAssets }; + }); + + const projectResults = await Promise.all(projectPromises); + + // Concatenate CSS in order and emit font assets + for (const { indexCSSContents, fontCSSContents, fontAssets } of projectResults) { + bundledProjectCSS = indexCSSContents + '\n' + bundledProjectCSS; + bundledProjectFontCSS = fontCSSContents + '\n' + bundledProjectFontCSS; + + fontAssets.forEach(({ file, asset }) => { + this.customFontFiles.push(file); + plugin.emitFile(asset); + }); } const bundledCSS = bundledProjectFontCSS + '\n' + bundledProjectCSS; @@ -1008,18 +1333,24 @@ export default class LivelyRollup { source: bundledCSS }); - for (let asset of this.projectAssets) { + // Parallelize project asset loading + const assetPromises = this.projectAssets.map(async (asset) => { const file = resource(projectsDir).join(asset.project).join('assets').join(`${asset.oldName}`); - if (file.isDirectory()) continue; + if (file.isDirectory()) return null; file.beBinary(); let source = await file.read(); if (source instanceof ArrayBuffer) source = new Uint8Array(source); - plugin.emitFile({ + return { type: 'asset', fileName: joinPath('assets', `${asset.newName}`), source - }); - } + }; + }); + + const loadedAssets = await Promise.all(assetPromises); + loadedAssets.forEach(asset => { + if (asset) plugin.emitFile(asset); + }); // add the blank import file to make systemjs happy plugin.emitFile({ type: 'asset', @@ -1034,11 +1365,32 @@ export default class LivelyRollup { fileName: 'deps.js', source: depsCode }); - plugin.emitFile({ - type: 'asset', - fileName: 'index.html', - source: await this.generateIndexHtml(importMap, modules) - }); + + // For multi-entry builds, generate an index.html for each entry point + const entryModules = modules.filter(m => m.isEntry); + if (entryModules.length > 1) { + // Generate separate HTML for each entry point + for (const entryModule of entryModules) { + plugin.emitFile({ + type: 'asset', + fileName: `index-${entryModule.name}.html`, + source: await this.generateIndexHtmlForEntry(importMap, modules, entryModule.fileName) + }); + } + // Also generate a default index.html pointing to the first entry + plugin.emitFile({ + type: 'asset', + fileName: 'index.html', + source: await this.generateIndexHtmlForEntry(importMap, modules, entryModules[0].fileName) + }); + } else { + // Single entry point - original behavior + plugin.emitFile({ + type: 'asset', + fileName: 'index.html', + source: await this.generateIndexHtml(importMap, modules) + }); + } } this.resolver.finish(); diff --git a/lively.freezer/src/plugins/rollup.js b/lively.freezer/src/plugins/rollup.js index b82a069c06..476548b0a6 100644 --- a/lively.freezer/src/plugins/rollup.js +++ b/lively.freezer/src/plugins/rollup.js @@ -1,6 +1,7 @@ +/* global process */ import LivelyRollup, { customWarn, bulletProofNamespaces } from '../bundler.js'; import { ROOT_ID } from '../util/helpers.js'; -import { obj, arr } from 'lively.lang'; +import { obj, arr, string } from 'lively.lang'; /** * Checks wether or not a given module is an internal module @@ -70,13 +71,31 @@ export function lively (args) { ? arr.pushIfNotIncluded(opts.external, 'livelyClassesRuntime.js') : (opts.external = ['livelyClassesRuntime.js']); } - if (bundler.snapshot || !!bundler.autoRun) { + // Handle autoRun for single-entry builds only + // For multi-entry builds (when input is an object), skip autoRun logic + const isMultiEntry = obj.isObject(opts.input) && !obj.isString(opts.input); + if ((bundler.snapshot || !!bundler.autoRun) && !isMultiEntry) { // since we are supposed to resolve from the snapshot, we set the input // to be the synthesized module. // delete the input bundler.rootModuleId = opts.input; delete opts.input; // emit the root module + } else if (bundler.autoRun && isMultiEntry) { + // For multi-entry builds, create synthetic root modules with hash-based names + // Store mapping from hash to entry path instead of embedding path in name + const syntheticEntries = {}; + const entryPathsMap = {}; + for (const [name, entryPath] of Object.entries(opts.input)) { + // Generate a hash for this entry path instead of embedding the path directly + // This avoids issues with slashes in module IDs breaking resolution + const entryHash = string.hashCode(entryPath).toString(36); + const rootModuleId = `__rootModule__:${entryHash}`; + syntheticEntries[name] = rootModuleId; + entryPathsMap[rootModuleId] = entryPath; + } + bundler.entryPaths = entryPathsMap; + opts.input = syntheticEntries; } if (bundler.excludedModules.length > 0) { opts.shimMissingExports = true; // since we are asked to exclude some of the lively modules, we set this flag to true. Can we isolate this?? @@ -104,13 +123,13 @@ export function lively (args) { opts.globals = { ...opts.globals, ...globals }; } } + opts.chunkFileNames = (chunk) => { + return `${chunk.name.replace('!cjs', '_CJS_')}-[hash].js`; + } return opts; }, - renderChunk(code) { - if (code.includes('get default ()')) { - return bulletProofNamespaces(code); - } - return null; + renderChunk(code, chunk) { + return bulletProofNamespaces(code, chunk.fileName, bundler.isResurrectionBuild, bundler.sourceMap); // this completely messes up the source mapping }, renderDynamicImport: () => { bundler.hasDynamicImports = true; // set flag to handle dynamic imports diff --git a/lively.freezer/src/resolvers/node.cjs b/lively.freezer/src/resolvers/node.cjs index 7386759be8..6366cb98d2 100644 --- a/lively.freezer/src/resolvers/node.cjs +++ b/lively.freezer/src/resolvers/node.cjs @@ -1,7 +1,7 @@ /* global process, require, module */ const { findPackageConfig } = require('flatn/flatn-cjs.js'); const babel = require('@babel/core'); -const { flatnResolve, findPackagePathForModule } = require('flatn/module-resolver.js'); +const { flatnResolve, findPackagePathForModule, findPackageConfig: findPackageConfigBrowser } = require('flatn/module-resolver.js'); const path = require('node:path'); const fs = require('node:fs'); const { builtinModules } = require('node:module'); @@ -72,7 +72,7 @@ function detectFormatFromSource (source) { } -async function normalizeFileName (fileName) { +function normalizeFileName (fileName) { if (isAlreadyResolved(fileName)) return fileName; return require.resolve(fileName); } @@ -88,11 +88,11 @@ function decanonicalizeFileName (fileName) { return url; } -function resolvePackage (moduleName) { +function resolvePackage (moduleName, context) { // if the moduleName is from a ESM cdn, we cannot determine the // package based on the module path if (isCdnImport(moduleName)) return; - return findPackageConfig(moduleName); + return context === 'systemjs-browser' ? findPackageConfigBrowser(moduleName) : findPackageConfig(moduleName); } function dontTransform (moduleId, knownGlobals) { @@ -173,12 +173,6 @@ function supportingPlugins(context = 'node', self) { return code.replaceAll(/\s(System|this)._nodeRequire\(/g, ' require('); } }, - context == 'browser' && { - name: 'node-prefix-remover', - resolveId(id, importer, options) { - return this.resolve(id.replace('node:', ''), importer, { skipSelf: true, ...options }); - } - }, context == 'node' && { // source-map and related packages are written in AMD format // we transform this here to ESM in order to be properly consumed by rollup. diff --git a/lively.freezer/src/util/bootstrap.js b/lively.freezer/src/util/bootstrap.js index 995497ac03..c0ce9978be 100644 --- a/lively.freezer/src/util/bootstrap.js +++ b/lively.freezer/src/util/bootstrap.js @@ -11,7 +11,7 @@ import { Project } from 'lively.project/project.js'; import { pathForBrowserHistory } from 'lively.morphic/helpers.js'; import { installLinter } from 'lively.ide/js/linter.js'; import { setupBabelTranspiler } from 'lively.source-transform/babel/plugin.js'; -import untar from 'esm://cache/js-untar'; +import untar from 'js-untar'; import bowser from 'bowser'; if (bowser.safari) { diff --git a/lively.freezer/src/util/helpers.js b/lively.freezer/src/util/helpers.js index 9c5f9ac206..1b477c9a4e 100644 --- a/lively.freezer/src/util/helpers.js +++ b/lively.freezer/src/util/helpers.js @@ -124,8 +124,6 @@ export async function getConfig (resolver) { } export async function compileOnServer (code, resolver, useTerser) { - const transpilationSpeed = 100000; - const compressionSpeed = 150000; const { cwd, tmp, min, presetPath, babelPath, babelConfig, pathToGoogleClosure, preprocessViaBabel } = await getConfig(resolver); tmp.onProgress = (evt) => { // set progress of loading indicator @@ -147,19 +145,14 @@ export async function compileOnServer (code, resolver, useTerser) { presets: [[presetPath, { modules: false }]] }); c = await resolver.spawn({ command: `${babelPath} -o tmp.es5.js tmp.js`, cwd }); - resolver.setStatus({ status: 'Transpiling source...', progress: 0.01 }); - for (const i of arr.range(0, code.length / transpilationSpeed)) { - await promise.delay(400); - if (c.status.startsWith('exited')) break; - resolver.setStatus({ status: 'Transpiling source', progress: (i + 1) / (code.length / transpilationSpeed) }); - } + resolver.setStatus({ status: 'Transpiling source...', progress: 0.5 }); await promise.waitFor(100 * 1000, () => c.status.startsWith('exited')); c = await resolver.spawn({ command: 'mv tmp.es5.js tmp.js', cwd }); await promise.waitFor(100 * 1000, () => c.status.startsWith('exited')); babelConfig.remove(); } - resolver.setStatus({ status: 'Minifying source files...', progress: 0.01 }); + resolver.setStatus({ status: 'Minifying source files...', progress: 0.5 }); if (useTerser) { c = await resolver.spawn({ @@ -173,11 +166,6 @@ export async function compileOnServer (code, resolver, useTerser) { cwd }); } - for (const i of arr.range(0, code.length / compressionSpeed)) { - await promise.delay(400); - if (c.status.startsWith('exited')) break; - resolver.setStatus({ status: 'Minifying source files ', progress: (i + 1) / (code.length / compressionSpeed) }); - } await promise.waitFor(100 * 1000, () => c.status.startsWith('exited')); if (c.stderr && c.exitCode !== 0) { resolver.finish(); @@ -313,6 +301,47 @@ export async function generateLoadHtml (htmlConfig, importMap, resolver, modules .replace('__CRAWLER_HTML__', crawler); } +export async function generateLoadHtmlForEntry (htmlConfig, importMap, resolver, modules, isResurrectionBuild, entryFileName) { + const htmlTemplate = await resource(resolver.ensureFileFormat(resolver.decanonicalizeFileName('lively.freezer/src/util/load-template.html'))).read(); + // Use the specified entry point instead of searching for one + const entryPoint = entryFileName; + // fixme: this only makes sense for auto run builds + const loadCode = ` + window.frozenPart = { + renderFrozenPart: (domNode, baseURL) => { + if (baseURL) System.config( { baseURL }); + if (!baseURL) baseURL = './'; + System.config({ + meta: { + ${ + modules.map(snippet => + `[baseURL + '${snippet.fileName}']: {format: "system", nonce: "lively" }` + ).join(',\n') // makes sure that compressed modules are still recognized as such + } + } + }); + ${isResurrectionBuild ? 'window.BootstrapSystem = System;' : ''} + System.trace = ${isResurrectionBuild ? 'true' : 'false'}; + System.import("./${entryPoint}").then(m => { m.renderFrozenPart(domNode); }); + } + } + `; + let title = htmlConfig.title || 'lively.next app'; + let head = htmlConfig.head || ''; + let load = htmlConfig.load || ''; + let crawler = htmlConfig.crawler || ''; + // extract stuff from the source code + if (importMap) { + head += importMap; + } + head += ''; + return htmlTemplate + .replace('__TITLE_TAG__', title) + .replace('__HEAD_HTML__', head) + .replace('__LOADING_HTML__', load) + .replace('__CRAWLER_HTML__', crawler); +} + /** * Handles the writing of all the files of a finished frozen bundle. * @param { object } frozen - A finished build. diff --git a/lively.freezer/src/util/regenerator-runtime.js b/lively.freezer/src/util/regenerator-runtime.js index 4112c33112..06247a9d82 100644 --- a/lively.freezer/src/util/regenerator-runtime.js +++ b/lively.freezer/src/util/regenerator-runtime.js @@ -7,8 +7,6 @@ */ let runtime = (function (exports) { - 'use strict'; - let Op = Object.prototype; let hasOwn = Op.hasOwnProperty; let undefined; // More compressible than void 0. diff --git a/lively.freezer/src/util/runtime.js b/lively.freezer/src/util/runtime.js index 25dbf13a02..bea409cd8a 100644 --- a/lively.freezer/src/util/runtime.js +++ b/lively.freezer/src/util/runtime.js @@ -590,7 +590,8 @@ export function runtimeDefinition () { const [local, exported] = exp.replace('__rename__', '').split('->'); exports[exported] = rec[local]; } else if (exp.startsWith('__reexport__')) Object.assign(exports, this.exportsOf(exp.replace('__reexport__', ''))); - else exports[exp] = rec[exp]; + else if (exp.startsWith('__default__')) exports.default = rec[exp.replace('__default__', '')] ; + else if (exp in rec) exports[exp] = rec[exp]; } return exports; }, diff --git a/lively.freezer/src/util/system.0.21.js b/lively.freezer/src/util/system.0.21.js index 6aec4adf47..a86fe2440f 100644 --- a/lively.freezer/src/util/system.0.21.js +++ b/lively.freezer/src/util/system.0.21.js @@ -3963,8 +3963,14 @@ SystemJSLoader.prototype.global = envGlobal; SystemJSLoader.prototype.import = function () { - return RegisterLoader.prototype.import.apply(this, arguments) - .then(function (m) { + const System = this; + let [moduleName] = arguments; + if (moduleName.startsWith('./')) { + moduleName = System.baseURL + moduleName.slice(2); + } + return RegisterLoader.prototype.import.apply(this, arguments).then(function (m) { + if (!m) m = System.loads[moduleName].exports; + if (!m) m = System.REGISTER_INTERNAL.records[moduleName].module; return '__useDefault' in m ? m.__useDefault : m; }); }; diff --git a/lively.freezer/tools/build-landing-page.sh b/lively.freezer/tools/build-landing-page.sh index 68044bf4c4..369a4c766d 100755 --- a/lively.freezer/tools/build-landing-page.sh +++ b/lively.freezer/tools/build-landing-page.sh @@ -8,4 +8,4 @@ done . ../scripts/lively-next-env.sh lively_next_env "$(dirname "$(pwd)")" export FLATN_DEV_PACKAGE_DIRS=$FLATN_DEV_PACKAGE_DIRS:$(pwd); -node --no-experimental-fetch --no-warnings --experimental-import-meta-resolve --experimental-loader ../flatn/resolver.mjs ./tools/build.landing-page.mjs $verbose +node --inspect --no-warnings --experimental-import-meta-resolve --experimental-loader ../flatn/resolver.mjs ./tools/build.landing-page.mjs $verbose diff --git a/lively.freezer/tools/build-loading-screen.sh b/lively.freezer/tools/build-loading-screen.sh index 7d6307ce0f..6cb18b1f44 100755 --- a/lively.freezer/tools/build-loading-screen.sh +++ b/lively.freezer/tools/build-loading-screen.sh @@ -8,4 +8,4 @@ done . ../scripts/lively-next-env.sh lively_next_env "$(dirname "$(pwd)")" export FLATN_DEV_PACKAGE_DIRS=$FLATN_DEV_PACKAGE_DIRS:$(pwd); -node --no-warnings --no-experimental-fetch --experimental-import-meta-resolve --experimental-loader ../flatn/resolver.mjs ./tools/build.loading-screen.mjs $verbose +node --inspect --no-warnings --experimental-import-meta-resolve --experimental-loader ../flatn/resolver.mjs ./tools/build.loading-screen.mjs $verbose diff --git a/lively.freezer/tools/build-unified.sh b/lively.freezer/tools/build-unified.sh new file mode 100755 index 0000000000..06c2b7ee65 --- /dev/null +++ b/lively.freezer/tools/build-unified.sh @@ -0,0 +1,11 @@ +#!/bin/bash +while [ "$#" -gt 0 ]; do + case "$1" in + --verbose) verbose="--verbose"; shift 1;; + -*) echo "unknown option: $1" >&2; exit 1;; + esac +done +. ../scripts/lively-next-env.sh +lively_next_env "$(dirname "$(pwd)")" +export FLATN_DEV_PACKAGE_DIRS=$FLATN_DEV_PACKAGE_DIRS:$(pwd); +node --inspect --no-warnings --experimental-import-meta-resolve --experimental-loader ../flatn/resolver.mjs ./tools/build.unified.mjs $verbose diff --git a/lively.freezer/tools/build.landing-page.mjs b/lively.freezer/tools/build.landing-page.mjs index 69650b1934..08ab1945ab 100644 --- a/lively.freezer/tools/build.landing-page.mjs +++ b/lively.freezer/tools/build.landing-page.mjs @@ -1,5 +1,5 @@ /* global process */ -import { rollup } from 'rollup'; +import { rollup } from '@rollup/wasm-node'; import jsonPlugin from '@rollup/plugin-json'; import { babel } from '@rollup/plugin-babel'; import { lively } from 'lively.freezer/src/plugins/rollup'; @@ -8,7 +8,7 @@ import PresetEnv from '@babel/preset-env'; const verbose = process.argv[2] === '--verbose'; const minify = !process.env.CI; - +const sourceMap = !!process.env.DEBUG; try { const build = await rollup({ input: './src/landing-page.cp.js', @@ -25,6 +25,7 @@ try { }, minify, verbose, + sourceMap, isResurrectionBuild: true, asBrowserModule: true, excludedModules: [ @@ -42,8 +43,13 @@ try { }), jsonPlugin({ exclude: [/https\:\/\/jspm.dev\/.*\.json/, /esm\:\/\/cache\/.*\.json/]}), babel({ - babelHelpers: 'bundled', - presets: [PresetEnv] + babelHelpers: 'bundled', + presets: [ + [PresetEnv, + { + "targets": "> 3%, not dead" + }] + ] }) ] }); @@ -51,6 +57,7 @@ try { await build.write({ format: 'system', dir: 'landing-page', + sourcemap: sourceMap ? 'inline' : false, globals: { chai: 'chai', mocha: 'mocha', diff --git a/lively.freezer/tools/build.loading-screen.mjs b/lively.freezer/tools/build.loading-screen.mjs index 3a181b986f..e0e4c16ac1 100644 --- a/lively.freezer/tools/build.loading-screen.mjs +++ b/lively.freezer/tools/build.loading-screen.mjs @@ -1,13 +1,14 @@ /* global process */ -import { rollup } from 'rollup'; +import { rollup } from '@rollup/wasm-node'; import jsonPlugin from '@rollup/plugin-json'; import { babel } from '@rollup/plugin-babel'; import { lively } from 'lively.freezer/src/plugins/rollup'; import resolver from 'lively.freezer/src/resolvers/node.cjs'; import PresetEnv from '@babel/preset-env'; -const verbose = process.argv[2] === '--verbose'; +const verbose = true; // process.argv[2] === '--verbose'; const minify = !process.env.CI; +const sourceMap = !!process.env.DEBUG; try { const build = await rollup({ input: './src/loading-screen.cp.js', @@ -22,17 +23,17 @@ try { ` }, + sourceMap, minify, verbose, isResurrectionBuild: true, asBrowserModule: true, excludedModules: [ 'mocha', 'chai', 'picomatch', // references old lgtg that breaks the build - 'path-is-absolute', 'fs.realpath', 'rollup', // has a dist file that cant be parsed by rollup + 'path-is-absolute', 'fs.realpath', // has a dist file that cant be parsed by rollup '@babel/preset-env', '@babel/plugin-syntax-import-meta', '@rollup/plugin-json', - '@rollup/plugin-commonjs', 'rollup-plugin-polyfill-node', 'babel-plugin-transform-es2015-modules-systemjs' ], @@ -41,7 +42,12 @@ try { jsonPlugin({ exclude: [/https\:\/\/jspm.dev\/.*\.json/, /esm\:\/\/cache\/.*\.json/]}), babel({ babelHelpers: 'bundled', - presets: [PresetEnv] + presets: [ + [PresetEnv, + { + "targets": "> 3%, not dead" + }] + ] }) ] }); @@ -49,6 +55,7 @@ try { await build.write({ format: 'system', dir: 'loading-screen', + sourcemap: sourceMap ? 'inline' : false, globals: { chai: 'chai', mocha: 'mocha', diff --git a/lively.freezer/tools/build.unified.mjs b/lively.freezer/tools/build.unified.mjs new file mode 100644 index 0000000000..e98e491876 --- /dev/null +++ b/lively.freezer/tools/build.unified.mjs @@ -0,0 +1,134 @@ +/* global process */ +import { rollup } from '@rollup/wasm-node'; +import jsonPlugin from '@rollup/plugin-json'; +import { babel } from '@rollup/plugin-babel'; +import { lively } from 'lively.freezer/src/plugins/rollup'; +import resolver from 'lively.freezer/src/resolvers/node.cjs'; +import PresetEnv from '@babel/preset-env'; + +const verbose = process.argv[2] === '--verbose'; +const minify = !process.env.CI; +const sourceMap = !!process.env.DEBUG; + +// Combine excluded modules from both builds to ensure compatibility +const commonExcludedModules = [ + 'chai', 'mocha', // references old lgtg that breaks the build + 'rollup', // has a dist file that cant be parsed by rollup + 'picomatch', 'path-is-absolute', 'fs.realpath', // from loading-screen build + // other stuff that is only needed by rollup + '@babel/preset-env', + '@babel/plugin-syntax-import-meta', + '@rollup/plugin-json', + '@rollup/plugin-commonjs', + 'rollup-plugin-polyfill-node', + 'babel-plugin-transform-es2015-modules-systemjs' +]; + +const commonAutoRunConfig = { + title: 'lively.next', + head: ` + + + ` +}; + +// Common plugins configuration +const commonPlugins = [ + jsonPlugin({ exclude: [/https\:\/\/jspm.dev\/.*\.json/, /esm\:\/\/cache\/.*\.json/] }), + babel({ + babelHelpers: 'bundled', + presets: [ + [PresetEnv, { + "targets": "> 3%, not dead" + }] + ] + }) +]; + +try { + console.log('[lively.freezer] Starting unified build for landing-page and loading-screen...'); + + // Single rollup build with multiple entry points + // Rollup will automatically share module parsing, transformation, and resolution + const build = await rollup({ + input: { + 'landing-page': './src/landing-page.cp.js', + 'loading-screen': './src/loading-screen.cp.js' + }, + shimMissingExports: true, + external: ['chai', 'mocha'], + plugins: [ + lively({ + // Note: For multi-entry builds, autoRun config is used for HTML generation + // but the rootModule synthesis is skipped (handled by rollup plugin) + autoRun: commonAutoRunConfig, + minify, + verbose, + sourceMap, + isResurrectionBuild: true, + asBrowserModule: true, + excludedModules: commonExcludedModules, + resolver + }), + ...commonPlugins + ] + }); + + console.log('[lively.freezer] Build complete, writing outputs...'); + + // Write landing-page output + await build.write({ + format: 'system', + dir: 'landing-page', + entryFileNames: '[name].js', + chunkFileNames: '[name]-[hash].js', + sourcemap: sourceMap ? 'inline' : false, + globals: { + chai: 'chai', + mocha: 'mocha', + }, + }); + console.log('[lively.freezer] ✓ Landing page written to landing-page/'); + + // Write loading-screen output + // Note: We need a second write() call to output to a different directory + // The build is already done, so this is just writing the same build to a different location + await build.write({ + format: 'system', + dir: 'loading-screen', + entryFileNames: '[name].js', + chunkFileNames: '[name]-[hash].js', + sourcemap: sourceMap ? 'inline' : false, + globals: { + chai: 'chai', + mocha: 'mocha', + } + }); + console.log('[lively.freezer] ✓ Loading screen written to loading-screen/'); + + // Post-process: Copy the correct index.html for each directory + // The build generates index-landing-page.html and index-loading-screen.html + // We copy the appropriate one to index.html in each directory + const fs = await import('fs/promises'); + + try { + await fs.copyFile('landing-page/index-landing-page.html', 'landing-page/index.html'); + console.log('[lively.freezer] ✓ Set landing-page/index.html to load landing-page entry'); + } catch (err) { + console.warn('[lively.freezer] ⚠ Could not copy landing-page index.html:', err.message); + } + + try { + await fs.copyFile('loading-screen/index-loading-screen.html', 'loading-screen/index.html'); + console.log('[lively.freezer] ✓ Set loading-screen/index.html to load loading-screen entry'); + } catch (err) { + console.warn('[lively.freezer] ⚠ Could not copy loading-screen index.html:', err.message); + } + + console.log('[lively.freezer] ✓ Unified build complete!'); + +} catch (err) { + console.error('[lively.freezer] Build failed:'); + console.error(err); + process.exit(1); +} diff --git a/lively.git/index.js b/lively.git/index.js index 552adf2409..5c610b75d8 100644 --- a/lively.git/index.js +++ b/lively.git/index.js @@ -1,6 +1,6 @@ /* global fetch */ import { currentUserToken } from 'lively.user'; -import { default as libsod } from 'esm://cache/libsodium-wrappers'; +import { default as libsod } from 'libsodium-wrappers'; // TODO: This could all use some sensible error handling -lh 15.02.2024 diff --git a/lively.git/package.json b/lively.git/package.json index 5e0ef99da7..ce4c37193b 100644 --- a/lively.git/package.json +++ b/lively.git/package.json @@ -9,7 +9,6 @@ "author": "Robert Krahn ", "license": "MIT", "main": "index.js", - "systemjs": {}, "lively": { "main": "index.js", "packageMap": { @@ -26,7 +25,8 @@ "js-github": "^1.0.0" }, "dependencies": { - "lively.resources": "^0.1.20", - "lively.storage": "^0.1.7" + "lively.resources": "*", + "lively.storage": "*", + "libsodium-wrappers": "0.7.15" } } diff --git a/lively.halos/layout.js b/lively.halos/layout.js index ecb7ed89c5..8eb415749b 100644 --- a/lively.halos/layout.js +++ b/lively.halos/layout.js @@ -3,7 +3,8 @@ import { Morph, Path, Text, - part + part, + TilingLayout } from 'lively.morphic'; import { Color, pt, Rectangle } from 'lively.graphics'; import { arr } from 'lively.lang'; @@ -360,7 +361,7 @@ class RowHalo extends AxisHalo { fetchExtent () { return pt(40, this.targetAxis.length - 10); } getMenuOffset (menu) { return this.targetAxis.length > menu.height ? pt(2, 5) : pt(26, 10); } - getMenuLayout () { return new VerticalLayout(); } + getMenuLayout () { return new TilingLayout({ axis: 'column' }); } getResizeCursor () { return 'row-resize'; } @@ -399,7 +400,7 @@ class ColumnHalo extends AxisHalo { fetchExtent () { return pt(this.targetAxis.length - 10, 40); } getMenuOffset (menu) { return this.targetAxis.length > menu.width ? pt(5, 3) : pt(8, 26); } - getMenuLayout () { return new HorizontalLayout(); } + getMenuLayout () { return new TilingLayout(); } getResizeCursor () { return 'col-resize'; } diff --git a/lively.halos/morph.js b/lively.halos/morph.js index b81d198425..1a089290f4 100644 --- a/lively.halos/morph.js +++ b/lively.halos/morph.js @@ -124,7 +124,7 @@ export class MorphHighlighter extends Morph { show () { if (this.target && this.target.layout && this.showLayout) { this.layoutHalo = - this.layoutHalo || this.world().showLayoutHaloFor(this.target, this.pointerId); + this.layoutHalo || $world.showLayoutHaloFor(this.target, this.pointerId); if (this.layoutHalo && this.layoutHalo.previewDrop) { this.visible = false; this.alignWithHalo(); diff --git a/lively.halos/package.json b/lively.halos/package.json index 34eb874ae0..dd37ab434f 100644 --- a/lively.halos/package.json +++ b/lively.halos/package.json @@ -18,10 +18,5 @@ }, "scripts": { "test": "mocha-es6 tests/*-test.js" - }, - "systemjs": { - "map": { - "svg-intersections": "lively.morphic/dist/svg-intersections.js" - } } } \ No newline at end of file diff --git a/lively.ide/components/reconciliation.js b/lively.ide/components/reconciliation.js index 98598bc85f..4699590575 100644 --- a/lively.ide/components/reconciliation.js +++ b/lively.ide/components/reconciliation.js @@ -649,8 +649,6 @@ export class Reconciliation { const [openEditor] = openEditors = this.getEligibleSourceEditors(modId, sourceCode); if (openEditor) sourceCode = openEditor.textString; - // FIXME: cache the AST node and transform them with a source mods library that understands how to patch the ast - // This can be done with: import { print, parse } from 'esm://cache/recast@0.21.5' const parsedModule = parse(sourceCode); const scope = query.topLevelDeclsAndRefs(parsedModule).scope; const parsedComponent = descr.getASTNode(parsedModule); diff --git a/lively.ide/html/editor-plugin.js b/lively.ide/html/editor-plugin.js index a5c21be68d..c1815207d9 100644 --- a/lively.ide/html/editor-plugin.js +++ b/lively.ide/html/editor-plugin.js @@ -8,25 +8,14 @@ import { import { localInterface, systemInterfaceNamed } from 'lively-system-interface'; import HTMLNavigator from './navigator.js'; import HTMLChecker from './checker.js'; -import * as parse5 from 'esm://cache/parse5@7.0.0'; +import * as parse5 from 'parse5'; import { IFrameMorph } from 'lively.morphic'; import { string } from 'lively.lang'; import { Color } from 'lively.graphics'; - -// export async function tidyHtml(htmlSrc) { -// let {stdout} = await runCommand("tidy --indent", {stdin: htmlSrc}).whenDone(); -// stdout = stdout.replace(/\s*]+>/, ""); -// return stdout; -// } +import js_beautify from 'js-beautify'; export async function tidyHtml (htmlSrc) { - let beautify; - try { - ({ default: beautify } = await lively.modules.module('esm://cache/js-beautify').load({ format: 'global' })); - } catch (err) { - ({ default: beautify } = await lively.modules.module('esm://cache/js-beautify').load({ format: 'global' })); - } - return beautify.html(htmlSrc); + return js_beautify.html(htmlSrc); } let commands = [ @@ -335,6 +324,8 @@ export default class HTMLEditorPlugin extends CodeMirrorEnabledEditorPlugin { return endpoint.runEval(code, env); } + setFormat () {} + get parser () { return parse5; } parse () { diff --git a/lively.ide/js/linter.js b/lively.ide/js/linter.js index 08e19d98fb..941ad6238e 100644 --- a/lively.ide/js/linter.js +++ b/lively.ide/js/linter.js @@ -2,8 +2,8 @@ * Methods for running eslint on JS code as well as the linter configuration used in lively.next. */ -import config from 'esm://cache/eslint-config-standard@16.0.3'; -import eslint from 'esm://cache/eslint@7.32.0'; +import config from 'eslint-config-standard'; +import eslint from 'eslint'; const rules = { // These are all rules from the default ruleset that are fixable diff --git a/lively.ide/js/objecteditor/index.js b/lively.ide/js/objecteditor/index.js index c64316173c..4e81611653 100644 --- a/lively.ide/js/objecteditor/index.js +++ b/lively.ide/js/objecteditor/index.js @@ -1722,7 +1722,7 @@ export class ImportControllerModel extends ViewModel { let importStmt = 'import ... from "module";'; if (importStyle === 'jspm') { - let jspmModule = await this.world().filterableListPrompt('Browse NPM', [], { + let npmModule = await this.world().filterableListPrompt('Browse NPM', [], { requester, onFilter: fun.debounce(500, async (param) => { const list = param.target.owner; // gets a MethodCallChange as parameter @@ -1732,11 +1732,67 @@ export class ImportControllerModel extends ViewModel { }), fuzzy: true }); - if (jspmModule.status !== 'accepted') return; - [jspmModule] = jspmModule.selected; - const { version, name } = jspmModule.package; - // fixme: use custom esm://cache mechanism per default here? - importStmt = `import ... from "https://jspm.dev/${name}@${version}";`; + if (npmModule.status !== 'accepted') return; + [npmModule] = npmModule.selected; + const { version, name } = npmModule.package; + + // Add package to package.json and regenerate import map + const li = LoadingIndicator.open('Adding package to project...'); + try { + const { pkgName, pkgUrl } = await editor.withContextDo(async (ctx) => { + const pkg = ctx.selectedModule.package(); + if (!pkg) { + throw new Error('No package found for the current module'); + } + const packageJsonUrl = pkg.url + '/package.json'; + const packageJsonResource = resource(packageJsonUrl); + + // Read current package.json + let packageConfig = await packageJsonResource.readJson(); + + // Add to dependencies + if (!packageConfig.dependencies) packageConfig.dependencies = {}; + packageConfig.dependencies[name] = `^${version}`; + + // Write updated package.json + await packageJsonResource.writeJson(packageConfig, true); + + return { pkgName: pkg.name, pkgUrl: pkg.url }; + }, { name, version }); + + // Trigger server to regenerate import map and fetch it + li.label = 'Regenerating import map...'; + const importMapUrl = resource(System.baseURL).join(`/import-map.json?projectName=${encodeURIComponent(pkgName)}`).url; + const importMapResponse = await resource(importMapUrl).readJson(); + + if (!importMapResponse) { + throw new Error('Failed to retrieve import map from server'); + } + + // Write the cached import map to the package + li.label = 'Applying import map...'; + const cachedImportMapResource = resource(pkgUrl).join('.cachedImportMap.json'); + await cachedImportMapResource.writeJson(importMapResponse, true); + + // Update the package config with the new import map + await editor.withContextDo(async (ctx) => { + const pkg = ctx.selectedModule.package(); + if (!pkg.systemjs) pkg.systemjs = {}; + pkg.systemjs.importMap = importMapResponse; + // Also update the package config object + if (!pkg.config.systemjs) pkg.config.systemjs = {}; + pkg.config.systemjs.importMap = importMapResponse; + }, { importMapResponse }); + + li.remove(); + requester.setStatusMessage(`Added ${name}@${version} to package.json and updated import map`); + } catch (err) { + li.remove(); + requester.showError(new Error(`Failed to add package: ${err.message}`)); + return; + } + + importStmt = `import ... from "${name}";`; importStyle = 'free text'; // transition to free text mode } diff --git a/lively.ide/md/compiler.js b/lively.ide/md/compiler.js index 4f76b8fd91..b142346559 100644 --- a/lively.ide/md/compiler.js +++ b/lively.ide/md/compiler.js @@ -1,9 +1,9 @@ -import markdownIt from 'esm://cache/markdown-it@12.3.2'; -import markdownCheckbox from 'esm://cache/markdown-it-checkbox@1.1.0'; -import markdownCaption from 'esm://cache/markdown-it-implicit-figures'; -import { html5Media } from 'esm://cache/markdown-it-html5-media'; +import markdownIt from 'markdown-it'; +import markdownCheckbox from 'markdown-it-checkbox'; +import markdownCaption from 'markdown-it-implicit-figures'; +import { html5Media } from 'markdown-it-html5-media'; -import markdownAttrs from 'esm://cache/markdown-it-attrs'; +import markdownAttrs from 'markdown-it-attrs'; import { string } from 'lively.lang'; import markdownMermaid from 'mermaid-it-markdown'; diff --git a/lively.ide/md/morphs.js b/lively.ide/md/morphs.js index ee481d9dcb..396d8e1c7c 100644 --- a/lively.ide/md/morphs.js +++ b/lively.ide/md/morphs.js @@ -5,9 +5,9 @@ import { connect } from 'lively.bindings/index.js'; import { fun, promise } from 'lively.lang/index.js'; import { rect, Color } from 'lively.graphics/index.js'; import { ExpressionSerializer } from 'lively.serializer2'; -import hljs from 'esm://cache/highlight.js@11.9.0/lib/core'; -import javascript from 'esm://cache/highlight.js@11.9.0/lib/languages/javascript'; -import shell from 'esm://cache/highlight.js@11.9.0/lib/languages/shell'; +import hljs from 'highlight.js/lib/core'; +import javascript from 'highlight.js/lib/languages/javascript'; +import shell from 'highlight.js/lib/languages/shell'; import { module } from 'lively.modules/index.js'; hljs.registerLanguage('javascript', javascript); diff --git a/lively.ide/package.json b/lively.ide/package.json index 57d6a2cd31..eaaf9f3d87 100644 --- a/lively.ide/package.json +++ b/lively.ide/package.json @@ -22,18 +22,34 @@ "lively.resources": "https://github.com/LivelyKernel/lively.resources", "lively.shell": "https://github.com/LivelyKernel/lively.shell", "lively.lang": "^1.0.0", - "prettier": "1.18.2" - }, - "systemjs": { - "map": { - "semver": "esm://cache/semver", - "mermaid-it-markdown": "esm://run/mermaid-it-markdown" - } + "prettier": "1.18.2", + "semver": "5.3.0", + "mermaid-it-markdown": "1.0.8", + "diff": "5.0.0", + "parse5": "7.0.0", + "eslint": "7.32.0", + "eslint-config-standard": "16.0.3", + "markdown-it": "12.3.2", + "markdown-it-checkbox": "1.1.0", + "markdown-it-implicit-figures": "0.12.0", + "markdown-it-html5-media": "0.7.1", + "markdown-it-attrs": "4.3.1", + "highlight.js": "11.9.0", + "sinon-chai": "4.0.0", + "sinon": "21.0.0", + "js-beautify": "^1.15.4" }, "devDependencies": { "mocha-es6": "^0.5" }, "scripts": { "test": "mocha-es6 tests/{,diff/,js/,shell/}*-test.js" + }, + "systemjs": { + "map": { + "highlight.js/lib/core": "https://ga.jspm.io/npm:highlight.js@11.11.1/es/core.js", + "highlight.js/lib/languages/javascript": "https://ga.jspm.io/npm:highlight.js@11.11.1/es/languages/javascript.js", + "highlight.js/lib/languages/shell": "https://ga.jspm.io/npm:highlight.js@11.11.1/es/languages/shell.js" + } } } \ No newline at end of file diff --git a/lively.ide/test-runner.js b/lively.ide/test-runner.js index 778b00d630..8b249082aa 100644 --- a/lively.ide/test-runner.js +++ b/lively.ide/test-runner.js @@ -13,7 +13,7 @@ import { resource } from 'lively.resources/index.js'; import { packagesConfig } from 'lively.modules/src/packages/package.js'; import { localInterface } from 'lively-system-interface'; -import * as jsDiff from 'esm://cache/diff@5.0.0'; +import * as jsDiff from 'diff'; import { browserForFile } from './js/browser/ui.cp.js'; export function testsFromSource (sourceOrAst) { diff --git a/lively.ide/tests/components/reconciliation-test.js b/lively.ide/tests/components/reconciliation-test.js index 61352be9f9..0f8d66146b 100644 --- a/lively.ide/tests/components/reconciliation-test.js +++ b/lively.ide/tests/components/reconciliation-test.js @@ -1,17 +1,13 @@ /* global it, describe, beforeEach, afterEach, after, System */ import { expect, chai } from 'mocha-es6'; -import sinonChai from 'esm://cache/sinon-chai'; -import sinon from 'esm://cache/sinon'; +import sinonChai from 'sinon-chai'; +import sinon from 'sinon'; import { createFiles, resource } from 'lively.resources'; import module from 'lively.modules/src/module.js'; -import { getSystem, PackageRegistry, removeSystem } from 'lively.modules'; -import { importPackage } from 'lively.modules/src/packages/package.js'; import { Color, pt, rect } from 'lively.graphics'; -import { morph, component, add, without, TilingLayout, Label, part } from 'lively.morphic'; -import { GlobalInjector } from 'lively.modules/src/import-modification.js'; +import { morph, add, without, TilingLayout, Label, part } from 'lively.morphic'; import { Reconciliation } from '../../components/reconciliation.js'; import { promise } from 'lively.lang'; -import { getPathFromMorphToMaster } from '../../components/helpers.js'; chai.use(sinonChai); diff --git a/lively.ide/world-commands.js b/lively.ide/world-commands.js index f4e3d007de..80e3219e31 100644 --- a/lively.ide/world-commands.js +++ b/lively.ide/world-commands.js @@ -695,7 +695,7 @@ const commands = [ ({ a, b, format } = findFormat(a, b)); // eslint-disable-line no-use-before-define } else { a = String(a); b = String(b); } - const diff = await System.import('esm://cache/diff@5.0.0'); + const diff = await System.import('diff'); let diffed; diff --git a/lively.installer/install.js b/lively.installer/install.js index 75966a1c07..64618f06ba 100644 --- a/lively.installer/install.js +++ b/lively.installer/install.js @@ -26,7 +26,8 @@ export async function install(baseDir, dependenciesDir, verbose) { step6_setupObjectDB = true, step6_syncWithObjectDB = false, step7_setupAssets = true, - step8_runPackageBuildScripts = false; + step8_runPackageBuildScripts = false, + step9_createImportMap = true; try { @@ -161,8 +162,9 @@ export async function install(baseDir, dependenciesDir, verbose) { // ObjectDB init // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + const System = await setupSystem(baseDir); + if (step6_setupObjectDB) { - await setupSystem(baseDir); await setupObjectDB(baseDir, packageMap); } @@ -170,7 +172,6 @@ export async function install(baseDir, dependenciesDir, verbose) { // ObjectDB sync // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- if (step6_syncWithObjectDB) { - await setupSystem(baseDir); await replicateObjectDB(baseDir); } @@ -215,6 +216,17 @@ export async function install(baseDir, dependenciesDir, verbose) { } } + // the paths need to be updated now, so that we can properly resolve installed deps + if (step9_createImportMap) { + const { Generator } = await import('@jspm/generator'); + System.set('@jspm_generator', System.newModule({ default: Generator })); + const { generateImportMap } = await System.import('lively.server/plugins/lib-lookup.js'); + for (let p of packages) { + console.log(`generating import map of ${p.name}`); + await generateImportMap(p.name); + } + } + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- pBar && pBar.remove(); diff --git a/lively.installer/package.json b/lively.installer/package.json index b6d9c0411e..b8edd0e5eb 100644 --- a/lively.installer/package.json +++ b/lively.installer/package.json @@ -31,14 +31,13 @@ "lively.lang": "*", "flatn": "*", "lively.freezer": "*", - "systemjs": "^0.21.6", + "@jspm/generator": "2.6.1", "@rollup/plugin-json": "4.1.0", - "rollup": "^2.70.2", + "@rollup/wasm-node": "4.27.3", "rollup-plugin-polyfill-node": "0.9.0", - "babel-runtime": "6.20.0", "acorn": "8.0.4", "@buxlabs/amd-to-es6": "0.16.3", - "@babel/core": "^7.12.0", + "@babel/core": "7.26.10", "@babel/plugin-transform-modules-systemjs": "7.18.0", "@babel/plugin-transform-modules-commonjs": "7.14.0" } diff --git a/lively.lang/interval.js b/lively.lang/interval.js index 1e8c764459..d775f970cb 100644 --- a/lively.lang/interval.js +++ b/lively.lang/interval.js @@ -13,7 +13,7 @@ import { timeToRunN } from './function.js'; /** * An interval defining an upper and a lower bound. * @typedef { number[] } Interval - * @property {number} 0 - The lower bound of the interval. + * @property {number} 0 - The lower bound of the interval. * @property {number} 1 - The upper bound of the interval. */ @@ -91,7 +91,7 @@ function compare (a, b) { * interval.coalesce([3,6], [4,5]) // => [3,6] */ function coalesce (interval1, interval2, optMergeCallback) { - const cmpResult = this.compare(interval1, interval2); + const cmpResult = compare(interval1, interval2); let temp; switch (cmpResult) { case -3: @@ -134,7 +134,7 @@ function coalesceOverlapping (intervals, optMergeCallback) { } condensed.push(ival); } - return this.sort(condensed); + return sort(condensed); } /** @@ -181,7 +181,7 @@ function mergeOverlapping (intervalsA, intervalsB, mergeFunc) { function intervalsInRangeDo (start, end, intervals, iterator, mergeFunc, context) { context = context || GLOB; // need to be sorted for the algorithm below - intervals = this.sort(intervals); + intervals = sort(intervals); let nextInterval; const collected = []; // merged intervals are already sorted, simply "negate" the interval array; while ((nextInterval = intervals.shift())) { diff --git a/lively.modules/package.json b/lively.modules/package.json index 9192c80e29..9e2096224a 100644 --- a/lively.modules/package.json +++ b/lively.modules/package.json @@ -13,10 +13,6 @@ "systemjs": { "main": "./index.js", "map": { - "semver": { - "~node": "esm://cache/semver@7.3.7", - "node": "semver" - }, "fs": { "node": "@node/fs", "~node": "@empty" @@ -32,19 +28,17 @@ "babel-regenerator-runtime": "^6.5.0", "babel-standalone": "^6.21.1-0", "@babel/standalone": "^7.18.3", - "lively.ast": "^0.11", - "lively.lang": "^1", - "lively.notifications": "*", - "lively.resources": "*", - "lively.storage": "*", - "lively.vm": "*", "semver": "^5.3.0", "systemjs": "^0.21.6", - "systemjs-plugin-babel": "0.0.13", "uglify-es": "^3.1.1", "brotli": "^1.3.2", "node-fetch": "^3.2.4", - "tar-fs": "^3.0.5" + "lively.ast": "^0.11", + "lively.lang": "^1", + "lively.notifications": "*", + "lively.resources": "*", + "lively.storage": "*", + "lively.vm": "*" }, "devDependencies": { "babel-plugin-external-helpers": "^6.8.0", diff --git a/lively.modules/src/cache.js b/lively.modules/src/cache.js index db2fe18f7f..3b5aac8545 100644 --- a/lively.modules/src/cache.js +++ b/lively.modules/src/cache.js @@ -76,18 +76,24 @@ export class NodeModuleTranslationCache extends ModuleTranslationCache { if (!await r.exists()) return null; const { birthtime: timestamp } = await r.stat(); const source = await r.read(); - const hash = await this.moduleCacheDir.join(fpath + '/.hash_' + fname).read(); - return { source, timestamp, hash }; + const hash = await this.moduleCacheDir.join(fpath).join('.hash_' + fname).read(); + const sourceMap = await this.moduleCacheDir.join(fpath).join('.source_map_' + fname).readJson(); + const exports = await this.moduleCacheDir.join(fpath).join('.exports_' + fname).readJson(); + return { source, timestamp, hash, sourceMap, exports }; } - async cacheModuleSource (moduleId, hash, source) { + async cacheModuleSource (moduleId, hash, source, exports = [], sourceMap = {}) { if (moduleId.endsWith('package.json')) return; moduleId = moduleId.replace('file://', ''); const fname = this.getFileName(moduleId); const fpath = moduleId.replace(fname, ''); await this.ensurePath(fpath); await this.moduleCacheDir.join(moduleId).write(source); - await this.moduleCacheDir.join(fpath + '/.hash_' + fname).write(hash); + await this.moduleCacheDir.join(fpath).join('.hash_' + fname).write(hash); + await this.moduleCacheDir.join(fpath).join('.source_map_' + fname).writeJson(sourceMap); + await this.moduleCacheDir.join(fpath).join('.exports_' + fname).writeJson(exports.map(({ + type, exported, local, fromModule + }) => ({ type, exported, local, fromModule }))); } async deleteCachedData (moduleId) { diff --git a/lively.modules/src/instrumentation.js b/lively.modules/src/instrumentation.js index c5be678bb0..fe7c1e5b0f 100644 --- a/lively.modules/src/instrumentation.js +++ b/lively.modules/src/instrumentation.js @@ -1,15 +1,10 @@ /* global System */ -import { parse, nodes, isValidIdentifier } from 'lively.ast'; +import { nodes, isValidIdentifier } from 'lively.ast'; const { funcCall, member, literal } = nodes; import { evalCodeTransform, evalCodeTransformOfSystemRegisterSetters } from 'lively.vm'; import { string, obj, properties } from 'lively.lang'; import { classToFunctionTransform } from 'lively.classes'; -import { - install as installHook, - remove as removeHook, - isInstalled as isHookInstalled -} from './hooks.js'; import module, { detectModuleFormat } from './module.js'; import { BrowserModuleTranslationCache, NodeModuleTranslationCache } from './cache.js'; @@ -130,7 +125,27 @@ export function prepareTranslatedCodeForSetterCapture (System, source, moduleId, } } -function getCachedNodejsModule (System, load) { +function isMarkedForNodeRequire (System, load) { + // Check if this module's package is marked to use System._nodeRequire() + const env = System.get('@lively-env'); + if (!env.nodeRequirePackages || env.nodeRequirePackages.size === 0) return null; + + const { packageRegistry } = env; + if (!packageRegistry) return null; + + const pkg = packageRegistry.findPackageHavingURL(load.name); + if (!pkg) return null; + + // Check if this package is in the nodeRequirePackages set + for (const pkgName of env.nodeRequirePackages) { + if (pkg.name === pkgName) { + return pkgName; // Return the package name to require + } + } + return null; +} + +function getCachedNodejsModule (System, load, markedPackageName) { // On nodejs we might run alongside normal node modules. To not load those // twice we have this little hack... try { @@ -138,7 +153,19 @@ function getCachedNodejsModule (System, load) { const id = Module._resolveFilename(load.name .replace(/^file:\/\//, '') // unix .replace(/^\/([a-z]:\/)/i, '$1')); // windows - const nodeModule = Module._cache[id]; + let nodeModule = Module._cache[id]; + + // If not in cache but marked for node require, pre-load it + if (!nodeModule && markedPackageName) { + System.debug && console.log('[lively.modules getCachedNodejsModule] pre-loading %s via System._nodeRequire', markedPackageName); + try { + System._nodeRequire(id); + nodeModule = Module._cache[id]; + } catch (e) { + System.debug && console.log('[lively.modules getCachedNodejsModule] failed to pre-load %s: %s', markedPackageName, e.message); + } + } + return nodeModule; } catch (e) { System.debug && console.log('[lively.modules getCachedNodejsModule] %s unknown to nodejs', load.name); @@ -146,10 +173,10 @@ function getCachedNodejsModule (System, load) { return null; } -function addNodejsWrapperSource (System, load) { +function addNodejsWrapperSource (System, load, markedPackageName) { // On nodejs we might run alongside normal node modules. To not load those // twice we have this little hack... - const m = getCachedNodejsModule(System, load); + const m = getCachedNodejsModule(System, load, markedPackageName); if (m) { load.metadata.format = 'esm'; load.source = `var exports = System._nodeRequire('${m.id}'); export default exports;\n` + @@ -175,15 +202,20 @@ export async function customTranslate (load) { const System = this; const debug = System.debug; const meta = load.metadata; + + // Check if this module should be loaded via System._nodeRequire() + const markedForNodeRequire = isNode && isMarkedForNodeRequire(System, load); + const ignored = (meta && meta.hasOwnProperty('instrument') && !meta.instrument) || exceptions.some(exc => exc(load.name)); - if (ignored) { + // Only ignore if explicitly marked as ignored AND not marked for nodeRequire + if (ignored && !markedForNodeRequire) { debug && console.log('[lively.modules customTranslate ignoring] %s', load.name); return load.source; } - if (isNode && addNodejsWrapperSource(System, load)) { + if (isNode && addNodejsWrapperSource(System, load, markedForNodeRequire)) { debug && console.log('[lively.modules] loaded %s from nodejs cache', load.name); return load.source; } @@ -245,6 +277,8 @@ export async function customTranslate (load) { '[lively.modules customTranslate] loaded %s from filesystem cache after %sms', load.name, Date.now() - start); + + if (stored.sourceMap) meta.sourceMap = stored.sourceMap; return Promise.resolve(stored.source); } } diff --git a/lively.modules/src/module.js b/lively.modules/src/module.js index d027b786a7..d1b74c547e 100644 --- a/lively.modules/src/module.js +++ b/lively.modules/src/module.js @@ -17,7 +17,7 @@ export const detectModuleFormat = (function () { const esmFormatCommentRegExp = /['"]format (esm|es6)['"];/; const cjsFormatCommentRegExp = /['"]format cjs['"];/; // Stolen from SystemJS - const esmRegEx = /(^\s*|[}\);\n]\s*)(import\s*(['"]|(\*\s*as\s+)?[^"'\(\)\n;]+\s+from\s*['"]|\{)|export\s+\*\s+from\s+["']|export\s*(\{|default|function|class|var|const|let|async\s+function))/; + const esmRegEx = /(^\s*|[}\);\n]\s*)(import\s*(['"]|(\*\s*as\s+)?[^"'\(\)\n;]+\s+from\s*['"]|\{)|export\s*\*\s*from\s*["']|export\s*(\{|default|function|class|var|const|let|async\s+function))/; return (source, metadata) => { if (metadata && metadata.format) { @@ -71,10 +71,12 @@ export async function updateBundledModules (system, modulesToUpdate) { let mod = S['__lively.modules__loadedModules'][m]; if (!mod) { S.delete(m); + system.SUPPRESS_DEFINE_ERRORS = true; await S.import(m); mod = S['__lively.modules__loadedModules'][m] || module(S, m); } await mod.reload(); + system.SUPPRESS_DEFINE_ERRORS = false; S['__lively.modules__loadedModules'][m] = mod; // ensure module stays here even when the source and initialization are skipped. } // finally update the frozen records that require update @@ -546,6 +548,12 @@ class ModuleInterface { value: undefined }, + process: { + configurable: true, + writable: true, + value: S.global.process + }, + System: { configurable: true, writable: true, value: S }, __currentLivelyModule: { value: self }, diff --git a/lively.modules/src/packages/configuration.js b/lively.modules/src/packages/configuration.js index 1d7ce44839..063dfcc03e 100644 --- a/lively.modules/src/packages/configuration.js +++ b/lively.modules/src/packages/configuration.js @@ -37,6 +37,7 @@ export default class PackageConfiguration { meta: { 'package.json': { format: 'json' }, '*.cjs': { defaultExtension: false }, + '*.mjs': { defaultExtension: false }, ...sysConfig.meta }, configured: true @@ -76,6 +77,13 @@ export default class PackageConfiguration { if (sysConfig.globalmap) { System.config({ map: sysConfig.globalmap }); } if (sysConfig.babelOptions) { System.config({ babelOptions: sysConfig.babelOptions }); } if (sysConfig.meta) { System.config({ meta: sysConfig.meta }); } + // Only apply nodeRequirePackages in Node.js environment + if (sysConfig.nodeRequirePackages && System._nodeRequire) { + // Store packages that should be loaded via System._nodeRequire() + const env = System.get('@lively-env'); + if (!env.nodeRequirePackages) env.nodeRequirePackages = new Set(); + sysConfig.nodeRequirePackages.forEach(pkg => env.nodeRequirePackages.add(pkg)); + } } // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- diff --git a/lively.modules/src/packages/package-registry.js b/lively.modules/src/packages/package-registry.js index 0f1855c726..e00ea9b1da 100644 --- a/lively.modules/src/packages/package-registry.js +++ b/lively.modules/src/packages/package-registry.js @@ -274,7 +274,6 @@ export class PackageRegistry { // does url identify a resource inside pkg, maybe pkg.url === url? if (url.isResource) url = url.url; if (url.endsWith('/')) url = url.slice(0, -1); - if (this.moduleUrlToPkg.has(url)) return this.moduleUrlToPkg.get(url); let penaltySoFar = Infinity; let found = null; let { byURL } = this; for (let pkgURL in byURL) { if (url.indexOf(pkgURL) !== 0) continue; @@ -283,6 +282,7 @@ export class PackageRegistry { penaltySoFar = penalty; found = byURL[pkgURL]; } + if (!found && this.moduleUrlToPkg.has(url)) return this.moduleUrlToPkg.get(url); if (found) { this.moduleUrlToPkg.set(url, found); } diff --git a/lively.modules/src/packages/package.js b/lively.modules/src/packages/package.js index 3da527dbd2..a527542bcd 100644 --- a/lively.modules/src/packages/package.js +++ b/lively.modules/src/packages/package.js @@ -142,7 +142,7 @@ class Package { this.version = config.version; this.dependencies = config.dependencies || {}; this.devDependencies = config.devDependencies || {}; - this.main = config.main || 'index.js'; + this.main = config.main; this.systemjs = config.systemjs; this.lively = config.lively; this.author = config.author; @@ -329,10 +329,15 @@ class Package { try { let config = System.get(packageConfigURL) || await System.import(packageConfigURL); - if (config.__useDefault) config = config.default; + const importMap = await this.hasResource('.cachedImportMap.json') && await resource(url + '/.cachedImportMap.json').readJson(); let packageConfigPaths = [...System.packageConfigPaths]; arr.pushIfNotIncluded(packageConfigPaths, packageConfigURL); // to inform systemjs that there is a config System.config({ packageConfigPaths }); + if (config.__useDefault) config = config.default; + if (importMap) { + if (!config.systemjs) config.systemjs = {}; + config.systemjs.importMap = importMap; + } return config; } catch (err) { console.log('[lively.modules package] Unable loading package config %s for package: ', packageConfigURL, err); // eslint-disable-line no-console diff --git a/lively.modules/src/resource.js b/lively.modules/src/resource.js index 8e274c4fec..fa3c79118d 100644 --- a/lively.modules/src/resource.js +++ b/lively.modules/src/resource.js @@ -1,6 +1,5 @@ import { resource, Resource } from 'lively.resources'; import { ESMResource } from 'lively.resources/src/esm-resource.js'; -import { promise } from 'lively.lang'; import { install as installHook, remove as removeHook, @@ -50,7 +49,7 @@ export async function fetchResource (proceed, load) { if (!System.get('@system-env').node && useCache && indexdb && cache) { const stored = await cache.fetchStoredModuleSource(load.name); let normalizedName = load.name.replace(System.baseURL, '/'); - if (normalizedName.startsWith('esm://cache')) { normalizedName = '/esm_cache/' + ESMResource.normalize(normalizedName).join('/'); } + if (normalizedName.startsWith('esm://')) { normalizedName = '/esm_cache/' + ESMResource.normalize(normalizedName).join('/'); } if (stored && (jsFileHashMap?.[normalizedName] === Number.parseInt(stored.hash))) { load.metadata.instrument = false; // skip instrumentation if (stored.sourceMap) load.metadata.sourceMap = JSON.parse(stored.sourceMap); diff --git a/lively.modules/src/system.js b/lively.modules/src/system.js index 4cb11fe659..9abb72d3a6 100644 --- a/lively.modules/src/system.js +++ b/lively.modules/src/system.js @@ -8,6 +8,7 @@ import { wrapResource, fetchResource } from './resource.js'; import { emit } from 'lively.notifications'; import { join, urlResolve } from './url-helpers.js'; import { resource } from 'lively.resources'; +import { resolveExportMapping, resolveImportMapping, resolveViaImportMap } from 'flatn/helpers.mjs'; // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- const isNode = System.get('@system-env').node; @@ -163,7 +164,8 @@ function livelySystemEnv (System) { notificationSubscribers: System['__lively.modules__notificationSubscribers'] || (System['__lively.modules__notificationSubscribers'] = {}), options: System['__lively.modules__options'] || (System['__lively.modules__options'] = obj.deepCopy(defaultOptions)), onLoadCallbacks: System['__lively.modules__onLoadCallbacks'] || (System['__lively.modules__onLoadCallbacks'] = []), - modulePackageMapCache: System['__lively.modules__modulePackageMapCache'] + modulePackageMapCache: System['__lively.modules__modulePackageMapCache'], + nodeRequirePackages: System['__lively.modules__nodeRequirePackages'] || (System['__lively.modules__nodeRequirePackages'] = new Set()) }; } @@ -217,6 +219,8 @@ function prepareSystem (System, config) { ? config.useModuleTranslationCache : !urlQuery().noModuleCache; System.useModuleTranslationCache = useModuleTranslationCache; + + System.importMapCache = new Map(); if (config._nodeRequire) System._nodeRequire = config._nodeRequire; @@ -360,7 +364,7 @@ function urlQuery () { // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- const dotSlashStartRe = /^\.?\//; const trailingSlashRe = /\/$/; -const jsExtRe = /\.js$/; +const jsExtRe = /(\.js|\.mjs)$/; const cjsExtRe = /\.cjs$/; const jsxExtRe = /\.jsx$/; const nodeExtRe = /\.node$/; @@ -387,29 +391,19 @@ function preNormalize (System, name, parent) { const { packageRegistry } = System.get('@lively-env'); if (packageRegistry) { let importMap, mappedObject, packageURL; - const pkg = parent && packageRegistry.findPackageHavingURL(parent); + let pkg = parent && packageRegistry.findPackageHavingURL(parent); if (pkg) { - let map, systemjs; - ({ map, url: packageURL, systemjs } = pkg); - importMap = !isNode && systemjs?.importMap; // only works in the browser - mappedObject = map?.[name] || System.map[name]; - } - - if (importMap) { - let remapped = importMap.imports?.[name]; - let scope, prefix; - if (scope = Object.entries(importMap.scopes) - .filter(([k, v]) => parent.startsWith(k)) - .sort((a, b) => a[0].length - b[0].length) - .map(([prefix, scope]) => scope) - .reduce((a, b) => ({ ...a, ...b }), false)) { - if (scope[name]) remapped = scope[name]; - } - if (remapped) { - name = remapped; - if (mappedObject) mappedObject = name; - packageRegistry.moduleUrlToPkg.set(name, pkg); + let map, systemjs, config; + ({ map, url: packageURL, systemjs, config } = pkg); + if (config.imports) { + try { + name = resolveImportMapping(name, config.imports, 'module'); + if (name.startsWith('.')) name = urlResolve(join(packageURL, name)); + } catch (err) {} } + if (config.optionalDependencies?.[name]) return '@empty'; + importMap = !isNode && systemjs?.importMap; // only works in the browser + mappedObject = map?.[name] || isNode && System.map[name]; // only consider the global map if no local importMap } if (mappedObject) { @@ -423,6 +417,16 @@ function preNormalize (System, name, parent) { if (name.startsWith('.')) name = urlResolve(join(packageURL, name)); } + if (importMap) { + let remapped = resolveViaImportMap(name, importMap, parent); + if (remapped) { + name = remapped; + packageRegistry.moduleUrlToPkg.set(name, pkg); + } + } + + if (!importMap && name.startsWith('node:')) name = '@node/' + name.slice(5); // some jspm bullshit + let resolved = packageRegistry.resolvePath(name, parent); if (resolved) { if (resolved.endsWith('/') && !name.endsWith('/')) resolved = resolved.slice(0, -1); @@ -430,8 +434,28 @@ function preNormalize (System, name, parent) { name = resolved; } - if (pkg && importMap && !packageRegistry.moduleUrlToPkg.get(name)) { - packageRegistry.moduleUrlToPkg.set(name, pkg); + if (importMap && !System.importMapCache.get(name)) + System.importMapCache.set(name, importMap); + + if (pkg && importMap && !packageRegistry.moduleUrlToPkg.has(name)) { + packageRegistry.moduleUrlToPkg.set(name, pkg); // orphaned ESM modules, that do not have their own packages + } else if (pkg = packageRegistry.findPackageWithURL(name)) { + // check if the exports exports are defined here, and adjust them now! + let { exports: exportMappings } = pkg.config + + if (exportMappings?.['.']) { // only in case the request was root + let adjustedPath = resolveExportMapping(exportMappings['.'], 'module'); + name = join(pkg.url, adjustedPath); + } + } else if (pkg = packageRegistry.findPackageHavingURL(name)) { + // for cases where we import a package via subpath like packageName/subpath + // which are not captured properly by packageWithURL + let { exports: exportMappings } = pkg.config + const subPath = name.replace(pkg.url, '.'); + if (exportMappings?.[subPath]) { + let adjustedPath = resolveExportMapping(exportMappings[subPath], 'module'); + name = join(pkg.url, adjustedPath); + } } } @@ -442,7 +466,7 @@ function preNormalize (System, name, parent) { function postNormalize (System, normalizeResult, isSync) { // console.log(`> [postNormalize] ${normalizeResult}`); // lookup package main - const base = normalizeResult.replace(jsExtRe, ''); + const base = normalizeResult.replace('/index.js', '') // rk 2017-05-13: FIXME, we currently use a form like // System.decanonicalize("lively.lang/") to figure out the package base path... @@ -454,10 +478,10 @@ function postNormalize (System, normalizeResult, isSync) { const { packageRegistry } = System.get('@lively-env'); if (packageRegistry) { const referencedPackage = packageRegistry.findPackageWithURL(base); + if (referencedPackage) { - let main = (referencedPackage.main || 'index.js').replace(dotSlashStartRe, ''); - let withMain = base.replace(trailingSlashRe, '') + '/' + main; - // console.log(`>> [postNormalize] ${withMain} (main 1)`); + let { main = 'index.js' } = referencedPackage.config + let withMain = join(referencedPackage.url, main.replace(dotSlashStartRe, '')); return withMain; } } else { @@ -465,7 +489,6 @@ function postNormalize (System, normalizeResult, isSync) { let main = System.CONFIG.packages[base].main; if (main) { let withMain = base.replace(trailingSlashRe, '') + '/' + main.replace(dotSlashStartRe, ''); - // console.log(`>> [postNormalize] ${withMain} (main 2)`); return withMain; } } @@ -485,6 +508,40 @@ async function checkExistence (url, System) { return System._fileCheckMap[url] = await resource(url).exists(); } +async function finalizeNormalization (System, name, normalized) { + const isNodePath = normalized.startsWith('file:'); + if ( + // Make sure we did not ask for a js or jsx file in the initial query. + !jsExtRe.test(name) && + !jsxExtRe.test(name) && + !cjsExtRe.test(name) && + // Make sure SystemJS has not yet resolved to a json or node module. + // If this happens, the resolution algorithm most likely has already + // figured out things and we assume that it has come up with a reasonable + // answer. + !jsonExtRe.test(normalized) && + !nodeModRe.test(normalized) && + !nodeExtRe.test(normalized) && + // Make sure that the module as not been loaded. + !(System.loads && System.loads[normalized]) && + !normalized.startsWith('node:') + ) { + if (jsExtRe.test(normalized)) { + if (await checkExistence(normalized, System)) return normalized; + const indexjs = normalized.replace('.js', '/index.js'); + if (await checkExistence(indexjs, System) || !isNodePath) return indexjs; + return normalized.replace('.js', '/index.node'); + } else if (!normalized.startsWith('esm://') && !normalized.includes('jspm.dev') && normalized !== '@empty') { + if (await checkExistence(normalized + '.js', System)) return normalized + '.js'; + if (await checkExistence(normalized + '/index.js', System)) return normalized + '/index.js'; + } + } + + if (jsxJsExtRe.test(normalized)) normalized = normalized.replace('.jsx.js', '.jsx'); + + return normalized; +} + async function normalizeHook (proceed, name, parent, parentAddress) { const System = this; if (System.transpiler !== 'lively.transpiler.babel') return await proceed(name, parent, true); @@ -498,40 +555,16 @@ async function normalizeHook (proceed, name, parent, parentAddress) { } if (name === 'lively.fetch') return name; if (name === '@system-env') return name; - if (name.startsWith('node:')) name = '@node/' + name.slice(5); // some jspm bullshit const stage1 = preNormalize(System, name, parent); const stage2 = await proceed(stage1, parent, true); let stage3 = postNormalize(System, stage2 || stage1, false); - const isNodePath = stage3.startsWith('file:'); + stage3 = await finalizeNormalization(System, name, stage3); System.debug && console.log(`[normalize] ${name} => ${stage3}`); - if ( - // Make sure we did not ask for a js or jsx file in the initial query. - !jsExtRe.test(name) && - !jsxExtRe.test(name) && - !cjsExtRe.test(name) && - // Make sure SystemJS has not yet resolved to a json or node module. - // If this happens, the resolution algorithm most likely has already - // figured out things and we assume that it has come up with a reasonable - // answer. - !jsonExtRe.test(stage3) && - !nodeModRe.test(stage3) && - !nodeExtRe.test(stage3) && - // Make sure that the module as not been loaded. - !(System.loads && System.loads[stage3]) && - !stage3.startsWith('node:') - ) { - if (jsExtRe.test(stage3)) { - if (await checkExistence(stage3, System)) return stage3; - const indexjs = stage3.replace('.js', '/index.js'); - if (await checkExistence(indexjs, System) || !isNodePath) return indexjs; - return stage3.replace('.js', '/index.node'); - } else if (!stage3.startsWith('esm://') && !stage3.includes('jspm.dev') && stage3 !== '@empty') { - if (await checkExistence(stage3 + '.js', System)) return stage3 + '.js'; - if (await checkExistence(stage3 + '/index.js', System)) return stage3 + '/index.js'; - } + + if (System.METADATA[stage2] && !System.METADATA[stage3]) { + System.METADATA[stage3] = System.METADATA[stage2]; } - if (jsxJsExtRe.test(stage3)) stage3 = stage3.replace('.jsx.js', '.jsx'); return stage3; } @@ -544,10 +577,13 @@ function decanonicalizeHook (proceed, name, parent, isPlugin) { } let stage2 = proceed(stage1, parent, isPlugin); if (stage1.endsWith('/')) { - const main = this.CONFIG.packages[stage1.replace(/\/*$/, '')]?.main; + const isNodePath = stage2.startsWith('file:'); + let main = this.CONFIG.packages[(isNodePath ? 'file://' : '') + stage1.replace(/\/*$/, '')]?.main || 'index.js'; + if (main?.startsWith('./')) main = main.replace('./', ''); // SystemJS 0.21 has appended the main module, which is something we do not like // if we decanonicalize a '/' terminated url if (stage2.endsWith(main)) stage2 = stage2.replace(main, ''); + else if (!stage2.endsWith('/')) stage2 += '/'; } let stage3 = postNormalize(System, stage2, true); if (plugin) stage3 += plugin; diff --git a/lively.morphic/html-morph.js b/lively.morphic/html-morph.js index 7637834f39..42160ffb34 100644 --- a/lively.morphic/html-morph.js +++ b/lively.morphic/html-morph.js @@ -3,7 +3,7 @@ import { promise, arr } from 'lively.lang'; import { pt, Rectangle } from 'lively.graphics'; import { Morph } from './morph.js'; import { addOrChangeCSSDeclaration } from './rendering/dom-helper.js'; -import css from 'esm://cache/css@3.0.0'; +import css from 'css'; // Usage: // var htmlMorph = $world.addMorph(new HTMLMorph({position: pt(10,10)})); diff --git a/lively.morphic/layout.js b/lively.morphic/layout.js index c2e7d4522d..0c04b873e2 100644 --- a/lively.morphic/layout.js +++ b/lively.morphic/layout.js @@ -1,7 +1,7 @@ -import { pt, Transform, Point, Rectangle, rect } from 'lively.graphics'; +import { pt, Point, Rectangle, rect } from 'lively.graphics'; import { arr, promise, Closure, num, obj, fun } from 'lively.lang'; import { once, signal } from 'lively.bindings'; -import { loadYoga } from 'yoga-layout/dist/src/load.js'; +import { loadYoga } from 'yoga-layout/load'; let Yoga, _yoga, yogaConfig, ALIGN, ALIGN_CSS, JUSTIFY, JUSTIFY_CSS; if (!Yoga) { diff --git a/lively.morphic/morphicdb/tools.js b/lively.morphic/morphicdb/tools.js index 11a354a5ab..01ff86403d 100644 --- a/lively.morphic/morphicdb/tools.js +++ b/lively.morphic/morphicdb/tools.js @@ -1,6 +1,6 @@ import { Rectangle, pt, rect, Color } from 'lively.graphics'; import { connect, noUpdate, signal } from 'lively.bindings'; -import { ConstraintLayout, Icon, HorizontalLayout, VerticalLayout, Morph, part } from 'lively.morphic'; +import { ConstraintLayout, Icon, TilingLayout, Morph, part } from 'lively.morphic'; import { Checkbox } from 'lively.components'; import MorphicDB from './db.js'; @@ -315,7 +315,7 @@ class MorphicDBWidget extends Morph { ? [] : [{ name: 'controls', - layout: new HorizontalLayout({ spacing: 2 }), + layout: new TilingLayout({ spacing: 2 }), fill: null, submorphs: controls }] @@ -367,7 +367,7 @@ export class MorphicDBList extends Morph { let { layout, itemSettings, dbInfos, showAddButton, showCloseButton, showOKButton } = this; - if (!layout) { layout = this.layout = new VerticalLayout({ align: 'center', spacing: 4 }); } + if (!layout) { layout = this.layout = new TilingLayout({ axis: 'column', align: 'center', spacing: 4 }); } this.submorphs = dbInfos.map(info => new MorphicDBWidget({ ...itemSettings, selected: info.selected, dbInfo: info })); this.submorphs.forEach(ea => connect(ea, 'selected', this, 'onDBSelected')); @@ -377,7 +377,7 @@ export class MorphicDBList extends Morph { this.addMorph({ name: 'buttons', fill: null, - layout: new HorizontalLayout({ spacing: 5 }), + layout: new TilingLayout({ spacing: 5 }), submorphs: [okBtn, addBtn].filter(Boolean) }); addBtn && connect(addBtn, 'fire', this, 'interactivelyAddDB'); diff --git a/lively.morphic/package.json b/lively.morphic/package.json index 9155710eb8..3c1ce02b06 100644 --- a/lively.morphic/package.json +++ b/lively.morphic/package.json @@ -22,7 +22,10 @@ "vdom-parser": "^1.4.1", "web-animations-js": "^2.3.1", "flubber": "^0.4.2", - "bezier-easing": "^2.1.0" + "bezier-easing": "^2.1.0", + "css": "3.0.0", + "dom-to-image-more": "2.8.0", + "yoga-layout": "3.2.0" }, "devDependencies": { "babel-plugin-external-helpers": "^6.8.0", @@ -42,13 +45,7 @@ "systemjs": { "main": "index.js", "map": { - "bezier-easing": "esm://cache/bezier-easing", - "flubber": "esm://cache/flubber", - "jsdiff": "esm://cache/jsdiff", - "dom-to-image": "esm://cache/dom-to-image", - "bowser": "esm://cache/bowser@1.4.1", - "webfontloader": "esm://cache/webfontloader", - "yoga-layout/dist/src/load.js": "esm://run/yoga-layout@3.0.4/dist/src/load.js" + "yoga-layout/load": "https://ga.jspm.io/npm:yoga-layout@3.2.1/dist/src/load.js" } }, "scripts": { diff --git a/lively.morphic/rendering/morph-to-image.js b/lively.morphic/rendering/morph-to-image.js index f27d3caa09..85a97598dc 100644 --- a/lively.morphic/rendering/morph-to-image.js +++ b/lively.morphic/rendering/morph-to-image.js @@ -1,5 +1,5 @@ /* global System,Map */ -import domToImage from 'esm://cache/dom-to-image-more@2.8.0'; +import domToImage from 'dom-to-image-more'; /* diff --git a/lively.project/package.json b/lively.project/package.json index 33fa0f92a4..b675a72cef 100644 --- a/lively.project/package.json +++ b/lively.project/package.json @@ -1,4 +1,7 @@ { "name": "lively.project", - "version": "0.1.0" + "version": "0.1.0", + "dependencies": { + "semver": "^5.3.0" + } } \ No newline at end of file diff --git a/lively.project/project.js b/lively.project/project.js index c765c6d6b0..a241a1b059 100644 --- a/lively.project/project.js +++ b/lively.project/project.js @@ -28,7 +28,7 @@ import { promise } from 'lively.lang'; import { setupLively2Lively, setupLivelyShell } from 'lively.morphic/world-loading.js'; import { GitHubAPIWrapper } from 'lively.git'; import { generateKeyPair } from 'lively.git/js-keygen/js-keygen.js'; -import * as semver from 'esm://cache/semver'; +import * as semver from 'semver'; export const repositoryOwnerAndNameRegex = /\.com\/(.+)\/(.*)/; const fontCSSWarningString = `/*\nDO NOT CHANGE THE CONTENTS OF THIS FILE! @@ -135,7 +135,7 @@ export class Project { // TODO: How do we manage the necessity to upgrade this nicely? dependencies: { '@rollup/plugin-json': '4.1.0', - rollup: '^2.70.2', + '@rollup/wasm-node': '4.27.3', 'rollup-plugin-export-default': '1.4.0', 'rollup-plugin-polyfill-node': '0.9.0' }, diff --git a/lively.project/templates/build-shell.js b/lively.project/templates/build-shell.js index 89bd74d92a..655c3e378d 100644 --- a/lively.project/templates/build-shell.js +++ b/lively.project/templates/build-shell.js @@ -9,5 +9,5 @@ done . ../../scripts/lively-next-env.sh lively_next_env "$(dirname "$(dirname "$(pwd)")")" export FLATN_DEV_PACKAGE_DIRS=$FLATN_DEV_PACKAGE_DIRS:$(pwd); -node --no-experimental-fetch --no-warnings --experimental-import-meta-resolve --experimental-loader ../../flatn/resolver.mjs ./tools/build.mjs $verbose +node --no-warnings --experimental-import-meta-resolve --experimental-loader ../../flatn/resolver.mjs ./tools/build.mjs $verbose ` diff --git a/lively.project/templates/build.js b/lively.project/templates/build.js index 9d9d0d951a..7897bba1a5 100644 --- a/lively.project/templates/build.js +++ b/lively.project/templates/build.js @@ -1,5 +1,5 @@ export const buildScript = `/* global process */ -import { rollup } from 'rollup'; +import { rollup } from '@rollup/wasm-node'; import jsonPlugin from '@rollup/plugin-json'; import { babel } from '@rollup/plugin-babel'; import { lively } from 'lively.freezer/src/plugins/rollup'; diff --git a/lively.resources/package.json b/lively.resources/package.json index 1877239fbe..32ee1ef559 100644 --- a/lively.resources/package.json +++ b/lively.resources/package.json @@ -9,11 +9,7 @@ "map": { "fs": { "node": "@node/fs", - "~node": "esm://cache/fs@0.0.2" - }, - "fetch-ponyfill": { - "node": "https://jspm.dev/fetch-ponyfill", - "~node": "fetch-ponyfill/build/fetch-browser.js" + "~node": "@empty" } } }, diff --git a/lively.resources/src/esm-resource.js b/lively.resources/src/esm-resource.js index 018ddff9e3..b4b3346f79 100644 --- a/lively.resources/src/esm-resource.js +++ b/lively.resources/src/esm-resource.js @@ -41,15 +41,8 @@ export class ESMResource extends Resource { } getEsmURL () { - let baseUrl; - if (this.url.startsWith('esm://run/npm/')) baseUrl = 'https://cdn.jsdelivr.net/'; - else if (this.url.startsWith('esm://run/')) baseUrl = 'https://esm.run/'; - else if (this.url.startsWith('esm://cache/')) baseUrl = 'https://jspm.dev/'; - else { - const domain = this.url.match(/esm:\/\/([^\/]*)\//)?.[1]; - baseUrl = `https://${domain}/`; - } - return baseUrl; + const domain = this.url.match(/esm:\/\/([^\/]*)\//)?.[1]; + return `https://${domain}/`; } async read () { diff --git a/lively.serializer2/object-extensions.js b/lively.serializer2/object-extensions.js index 280371a380..ed06bd7a7d 100644 --- a/lively.serializer2/object-extensions.js +++ b/lively.serializer2/object-extensions.js @@ -116,3 +116,24 @@ Object.defineProperty(Set.prototype, '__deserialize__', { } } }); + +if (hasSystem) { + const originalDefineProperties = Object.defineProperties; + + // Global flag to toggle behavior + System.SUPPRESS_DEFINE_ERRORS = false; + + // Override + Object.defineProperties = function (obj, props) { + if (System.SUPPRESS_DEFINE_ERRORS) { + try { + return originalDefineProperties(obj, props); + } catch (err) { + if (!err.message.includes('Cannot redefine property')) throw err; + return obj; // or a fallback behavior + } + } else { + return originalDefineProperties(obj, props); + } + }; +} \ No newline at end of file diff --git a/lively.serializer2/package.json b/lively.serializer2/package.json index 1637274a2a..0b491be2e2 100644 --- a/lively.serializer2/package.json +++ b/lively.serializer2/package.json @@ -7,11 +7,6 @@ "lively.lang": "^1.0.5", "semver": "^5.3.0" }, - "systemjs": { - "map": { - "semver": "esm://cache/semver@7.3.7" - } - }, "devDependencies": { "mocha-es6": "*", "rollup": "^0.36.1", diff --git a/lively.server/index-base.js b/lively.server/index-base.js deleted file mode 100644 index d074fc74e0..0000000000 --- a/lively.server/index-base.js +++ /dev/null @@ -1,116 +0,0 @@ -/*global process,require,__dirname,module*/ -import "https://jspm.dev/npm:systemjs@0.21.6!cjs"; -import modules from "lively.modules"; -import { resource } from "lively.resources"; -import "https://jspm.dev/socket.io"; -import util from 'https://jspm.dev/util'; -import winston from "https://jspm.dev/winston"; - -const defaultServerDir = __dirname; -var livelySystem; -var config = { - serverDir: defaultServerDir, - port: 9011, - hostname: "localhost", - rootDirectory: null, - plugins: [] -}; - -export default function start(hostname, port, configFile, rootDirectory, serverDir) { - config.rootDirectory = rootDirectory || process.cwd(); - config.serverDir = serverDir || defaultServerDir; - setupLogger(); - var step = 1; - console.log(`[lively.server] system base directory: ${rootDirectory}`); - return setupSystem(config.rootDirectory) - .then(() => console.log(`[lively.server] ${step++}. preparing system`)) - .then(() => modules.registerPackage(config.serverDir)) - - // 1. This loads the lively system - .then(() => livelySystem.import("lively.resources")) - .then(resources => resources.ensureFetch()) - .then(() => livelySystem.import("lively.storage")) - .then(() => livelySystem.import("lively.vm")) - .then(vm => lively.vm = vm) - .then(() => livelySystem.import("lively.classes")) - .then(klass => lively.classes = klass) - .then(() => - silenceDuring( - // we use "GLOBAL" as normally declared var, nodejs doesn't seem to care... - data => !String(data).includes("DeprecationWarning: 'GLOBAL'"), - livelySystem.import("lively-system-interface"))) - .then(() => configFile ? console.log(`[lively.server] ${step++}. loading ${configFile}`) : null) - .then(() => configFile ? livelySystem.import(configFile) : null) - .then(configMod => { - if (!configMod || !configMod.default || !configMod.default.server) return; - Object.assign(config, configMod.default.server); - // passed in arguments take precedence - if (hostname) config.hostname = hostname; - if (port) config.port = port; - }) - - // 2. this loads and starts the server - .then(() => console.log(`[lively.server] ${step++}. starting server`)) - .then(() => livelySystem.import(config.serverDir + "/server.js")) - .then(serverMod => startServer(serverMod, config)) - .then(server => { - console.log(`[lively.server] ${step++}. server sucessfully started`); - return server; - }) - - .catch(err => { - console.error(`Error starting server: ${err.stack}`); - process.exit(1); - }); -}; - -async function silenceDuring(filter, promise) { - let {stdout, stderr} = process, - {write: stdoutWrite} = stdout, - {write: stderrWrite} = stderr; - stdout.write = d => filter(d) && stdoutWrite.call(stdout, d); - stderr.write = d => filter(d) && stderrWrite.call(stderr, d); - try { return await promise; } finally { - stdout.write = stdoutWrite; - stderr.write = stderrWrite; - } -} - -function formatArgs(args){ - return [util.format.apply(util.format, Array.prototype.slice.call(args))]; -} - -function setupLogger() { - let logger = new winston.Logger(); - logger.add(winston.transports.Console, {colorize: true, timestamp: true}); - console.livelyLogger = logger; - console.log = function() { logger.info.apply(logger, formatArgs(arguments)); }; - console.info = function() { logger.info.apply(logger, formatArgs(arguments)); }; - console.warn = function() { logger.warn.apply(logger, formatArgs(arguments)); }; - console.error = function() { logger.error.apply(logger, formatArgs(arguments)); }; - console.debug = function() { logger.debug.apply(logger, formatArgs(arguments)); }; - return logger; -} - -function setupSystem(rootDirectory) { - var baseURL = "file://" + rootDirectory; - livelySystem = modules.getSystem("lively", {baseURL}); - modules.changeSystem(livelySystem, true); - var registry = livelySystem["__lively.modules__packageRegistry"] = new modules.PackageRegistry(livelySystem); - registry.packageBaseDirs = process.env.FLATN_PACKAGE_COLLECTION_DIRS.split(":").map(ea => resource(`file://${ea}`)); - registry.devPackageDirs = process.env.FLATN_DEV_PACKAGE_DIRS.split(":").map(ea => resource(`file://${ea}`)); - registry.individualPackageDirs = process.env.FLATN_PACKAGE_DIRS.split(":").map(ea => resource(`file://${ea}`)); - return registry.update(); -} - -function startServer(serverMod, config) { - let {serverDir, port, hostname, rootDirectory, authServerURL, freezer} = config, - serverConfig = {port, hostname, plugins: [], jsdav: {rootDirectory}, authServerURL, freezer}; - return Promise.all( - config.plugins.map(path => - livelySystem.import(path) - .then(mod => serverConfig.plugins.push(new mod.default(serverConfig))) - .catch(err => { console.error(`Error loading plugin ${path}`); throw err; })) - ).then(() => serverMod.start(serverConfig)); -} - diff --git a/lively.server/index.js b/lively.server/index.js index 39d7ecccdf..6d2274ca22 100644 --- a/lively.server/index.js +++ b/lively.server/index.js @@ -7,6 +7,7 @@ import "socket.io"; import util from 'node:util'; import winston from "winston"; import { setupSystem } from "lively.installer"; +import { Generator } from "@jspm/generator"; const defaultServerDir = process.cwd(); var livelySystem; @@ -65,6 +66,7 @@ export default async function start(hostname, port, configFile, rootDirectory, s }) // 2. this loads and starts the server + .then(() => livelySystem.set('@jspm_generator', livelySystem.newModule({ default: Generator }))) .then(() => console.log(`[lively.server] ${step++}. starting server`)) .then(() => livelySystem.import(config.serverDir + "/server.js")) .then(serverMod => startServer(serverMod, config)) diff --git a/lively.server/package.json b/lively.server/package.json index 38e8090ec8..97adecb3d6 100644 --- a/lively.server/package.json +++ b/lively.server/package.json @@ -3,20 +3,22 @@ "version": "0.1.0", "type": "module", "dependencies": { + "@jspm/generator": "2.6.1", "http-proxy": "^1.16.2", "jsDAV": "mnutt/jsDAV", - "lively-system-interface": "https://github.com/LivelyKernel/lively-system-interface", - "lively.2lively": "https://github.com/LivelyKernel/lively.2lively.git", - "lively.modules": "*", - "lively.resources": "*", - "lively.shell": "https://github.com/LivelyKernel/lively.shell", "minimist": "^1.2.0", "socket.io": "^4.4.1", "socket.io-client": "^4.4.1", "systemjs": "^0.21.6", "winston": "^3.7.2", "formidable": "^1.2.2", - "@babel/core": "^7.12.0" + "@babel/core": "7.26.10", + "tar-fs": "^3.0.5", + "lively-system-interface": "https://github.com/LivelyKernel/lively-system-interface", + "lively.2lively": "https://github.com/LivelyKernel/lively.2lively.git", + "lively.modules": "*", + "lively.resources": "*", + "lively.shell": "https://github.com/LivelyKernel/lively.shell" }, "devDependencies": { "mocha-es6": "^0.5" diff --git a/lively.server/plugins/dav.js b/lively.server/plugins/dav.js index 800c746f18..f813e08684 100644 --- a/lively.server/plugins/dav.js +++ b/lively.server/plugins/dav.js @@ -111,7 +111,7 @@ export default class LivelyDAVPlugin { }); for (let file of filesToHash) { if (!file.isFile()) continue; - this.fileHashes[file.url.replace(System.baseURL, '/').replace('/esm_cache', 'esm://cache')] = string.hashCode(await file.read()); + this.fileHashes[file.url.replace(System.baseURL, '/')] = string.hashCode(await file.read()); } console.log('[lively.server] finished file hash map'); console.log('[lively.server] creating library snapshot...'); @@ -201,7 +201,7 @@ export default class LivelyDAVPlugin { if (req.method == 'PUT' || !this.fileHashes[req.url]) { this._skipHashUpdate = true; resource('file://' + this.options.rootDirectory).join(decodeURIComponent(req.url)).read().then(source => { - this.fileHashes[req.url.replace('/esm_cache', 'esm://cache')] = string.hashCode(source); + this.fileHashes[req.url] = string.hashCode(source); }).catch(err => { // ignore }); diff --git a/lively.server/plugins/lib-lookup.js b/lively.server/plugins/lib-lookup.js index 985229853c..803892b989 100644 --- a/lively.server/plugins/lib-lookup.js +++ b/lively.server/plugins/lib-lookup.js @@ -1,8 +1,51 @@ /*global System*/ -import LivelyServer from "../server.js"; import fs from "fs"; -import { basename, join } from "path"; +import { join } from "path"; import { resource } from "lively.resources"; +import { parseQuery } from "lively.resources"; +import { arr, obj } from "lively.lang"; +const Generator = System.get('@jspm_generator').default; + +async function installDeps(generator, deps, failed) { + for (let dep of deps) { + if (dep[0] == 'tar-fs' || !!generator.map.imports[dep[0]] || failed[dep[0]]) continue; + try { + await generator.install(dep.join('@')); + } catch (err) { + console.error('Failed to install ' + dep.join('@')); + failed[dep[0]] = true; + } + } + const toUninstall = arr.withoutAll(Object.keys(generator.map.imports), deps.map(d => d[0])); + await generator.uninstall(toUninstall); +} + +export async function generateImportMap (packageName) { + let inputMap = false; + const packageRegistry = System.get("@lively-env").packageRegistry; + const pkg = packageName && packageRegistry.lookup(packageName); + if (!pkg) return {}; + const cachedImportMap = resource(pkg.url).join('.cachedImportMap.json'); + if (await cachedImportMap.exists()) { + inputMap = JSON.parse((await cachedImportMap.read()).replace(/esm:\/\//g, 'https://')); // replace esm to make generator install again + } + const generator = new Generator({ + env: ["browser"], + defaultProvider: 'jspm.io', + inputMap + }); + const failed = inputMap?._failed || {}; // collect the packages where we fail to generate import maps, likely due to incompatibility with the browser + await installDeps( + generator, + Object.entries(pkg.config.dependencies || {}).filter(([dep]) => !dep.match(/lively(\.|-)/)), + failed + ); + const importMap = JSON.parse(JSON.stringify(generator.getMap()).replace(/https:\/\//g, 'esm://')) + if (!obj.isEmpty(failed)) importMap._failed = failed; + if (!obj.isEmpty(importMap)) await cachedImportMap.writeJson(importMap); + else if (inputMap) { await cachedImportMap.remove() } + return importMap; +} export default class LibLookupPlugin { @@ -40,10 +83,17 @@ export default class LibLookupPlugin { res.end(JSON.stringify(r.toJSON())); } + async sendImportmap (req, res) { + const { projectName } = parseQuery(req.url); + res.writeHead(200, {"Content-Type": "application/json"}); + res.end(JSON.stringify( await generateImportMap(projectName))); + } + async handleRequest(req, res, next) { let {libPath, fsRootDir} = this, {url: path} = req; if (path === "/package-registry.json") return this.sendPackageRegistry(req, res); + if (path.startsWith("/import-map.json")) return await this.sendImportmap(req, res); if (!path.startsWith(libPath) || path === libPath) return next(); if (fs.existsSync(join(fsRootDir, path))) return next(); diff --git a/lively.source-transform/babel/helpers.js b/lively.source-transform/babel/helpers.js index 290c8b8c38..0cb1279e76 100644 --- a/lively.source-transform/babel/helpers.js +++ b/lively.source-transform/babel/helpers.js @@ -1,6 +1,5 @@ import t from '@babel/types'; import { helpers } from 'lively.ast/lib/query.js'; -import { Path } from 'lively.lang'; export function getAncestryPath (path) { return path.getAncestry().map(m => m.inList ? [m.key, m.listKey] : m.key).flat().slice(0, -1).reverse(); @@ -259,7 +258,7 @@ export function additionalIgnoredDecls ({ varDecls, catches }) { }).flat()); } -export function additionalIgnoredRefs ({ varDecls, catches, importSpecifiers }, options) { +export function additionalIgnoredRefs ({ varDecls, catches, importSpecifiers = [] }, options) { const ignoreDecls = []; varDecls.forEach(pathToNode => { const decl = pathToNode.node; diff --git a/lively.source-transform/babel/plugin.js b/lively.source-transform/babel/plugin.js index b81b4cbbd8..dece790366 100644 --- a/lively.source-transform/babel/plugin.js +++ b/lively.source-transform/babel/plugin.js @@ -4,11 +4,15 @@ import babel from '@babel/core'; import systemjsTransform from '@babel/plugin-transform-modules-systemjs'; import dynamicImport from '@babel/plugin-proposal-dynamic-import'; import { arr, Path } from 'lively.lang'; -import { topLevelFuncDecls } from 'lively.ast/lib/visitors.js'; import { query } from 'lively.ast'; +import { topLevelFuncDecls } from 'lively.ast/lib/visitors.js'; +import { classToFunctionTransformBabel } from 'lively.classes'; import { getGlobal } from 'lively.vm/lib/util.js'; import { declarationWrapperCall, annotationSym, assignExpr, varDeclOrAssignment, transformPattern, generateUniqueName, varDeclAndImportCall, importCallStmt, shouldDeclBeCaptured, importCall, exportCallStmt, exportFromImport, additionalIgnoredDecls, additionalIgnoredRefs } from './helpers.js'; -import { classToFunctionTransformBabel } from 'lively.classes/class-to-function-transform.js'; + +export function babel_parse (source) { + return babel.parse(source).program.body; +} export const defaultDeclarationWrapperName = 'lively.capturing-declaration-wrapper'; export const defaultClassToFunctionConverter = t.Identifier('initializeES6ClassForLively'); @@ -22,7 +26,7 @@ function getVarDecls (scope) { return new Set(Object.values(scope.bindings).filter(decl => decl.kind !== 'module' && decl.kind !== 'hoisted').map(m => m.path.parentPath).filter(node => node.type === 'VariableDeclaration')); } -const babelNodes = { +export const babelNodes = { member: t.MemberExpression, property: t.ObjectProperty, property: (kind, key, val) => t.ObjectProperty(key, val), @@ -346,7 +350,7 @@ function ensureGlobalBinding (ref, options) { function replaceVarDeclsAndRefs (path, options) { const globalInitStmt = '_global = typeof globalThis !== "undefined" ? globalThis : typeof self !== "undefined" ? self : global'; - const refsToReplace = new Set(options.scope.refs.filter(ref => !options?.excludeRefs.includes(ref.name))); + const refsToReplace = new Set(options.scope.refs.filter(ref => !options?.excludeRefs.includes(ref.name) && path.scope.bindings[ref.name]?.kind !== 'module')); const varDeclsToReplace = getVarDecls(path.scope); const declaredNames = Object.keys(path.scope.bindings); const currentModuleAccessor = options.classToFunction?.currentModuleAccessor; @@ -580,14 +584,15 @@ function splitExportDeclarations (path) { } function insertCapturesForImportAndExportDeclarations (path, options) { + let i = 0; const declaredNames = new Set(Object.keys(path.scope.bindings)); function handleDeclarations (path) { const stmt = path.node; path.insertAfter(stmt.declaration.declarations.map(decl => { let assignVal = decl.id; if (options.declarationWrapper) { - const alreadyWrapped = decl.init.callee && - decl.init.callee.name === options.declarationWrapper.name; + const alreadyWrapped = decl.init?.callee && + decl.init?.callee.name === options.declarationWrapper.name; if (!alreadyWrapped) { assignVal = declarationWrapperCall( options.declarationWrapper, @@ -618,6 +623,44 @@ function insertCapturesForImportAndExportDeclarations (path, options) { ? null : varDeclOrAssignment(declaredNames, specifier.local, t.MemberExpression(options.captureObj, specifier.local))))); } + + if (stmt.specifiers.length && stmt.source) { + const specifiers = stmt.specifiers; + let paths = path.replaceWithMultiple([ + t.ImportDeclaration(specifiers.map(spec => { + if (spec.local.name === 'default' && spec.exported.name === 'default') { + return t.ImportSpecifier(t.Identifier(`__default${++i}__`), t.Identifier('default')); + } else if (spec.local.name !== spec.exported.name && spec.exported.name === 'default') { + spec.shadow = `__default${++i}__`; + return t.ImportSpecifier(t.Identifier(spec.shadow), t.Identifier(spec.local.name)); + } else if (spec.local.name !== spec.exported.name && spec.local.name === 'default') { + spec.shadow = spec.exported.name; + if (declaredNames.has(spec.exported.name)) return false; + return t.ImportSpecifier(t.Identifier(spec.exported.name), t.Identifier(spec.local.name)); + } else if (spec.local.name !== spec.exported.name) { + spec.shadow = spec.exported.name; + if (declaredNames.has(spec.exported.name)) return false; + return t.ImportSpecifier(t.Identifier(spec.exported.name), t.Identifier(spec.local.name)); + } else { + spec.shadow = `__${spec.local.name}__`; + return t.ImportSpecifier(t.Identifier(spec.shadow), t.Identifier(spec.local.name)); + } + }).filter(Boolean), t.StringLiteral(stmt.source.value)), + t.ExportNamedDeclaration(null, specifiers.map(spec => { + if (spec.local.name === 'default' && spec.exported.name === 'default') { + return t.ExportSpecifier(t.Identifier(`__default${i}__`), t.Identifier('default')); + } + declaredNames.add(spec.shadow); + return t.ExportSpecifier(t.Identifier(spec.shadow), t.Identifier(spec.exported.name)); + }).filter(Boolean)) + ]); + + for (let spec of specifiers) { + if (spec.local.name === 'default' && spec.exported.name === 'default') { paths[0].insertAfter(assignExpr(options.captureObj, t.Identifier('default'), t.Identifier(`__default${i}__`), false)); } else { paths[0].insertAfter(assignExpr(options.captureObj, t.Identifier(spec.exported.name), t.Identifier(spec.shadow), false)); } + } + + paths.forEach(path => path.skip()); + } }, ExportDefaultDeclaration (path) { const stmt = path.node; @@ -626,7 +669,7 @@ function insertCapturesForImportAndExportDeclarations (path, options) { // default export of an unnamed primitive value, i.e. // "export default "foo"", "export default 27;" const decl = stmt.declaration; - const refId = generateUniqueName(declaredNames, '$' + decl.extra.raw.split('"').join('')); + const refId = generateUniqueName(declaredNames, '$exportedLiteral'); path.insertBefore(assignExpr(options.captureObj, t.Identifier(refId), stmt.declaration, false)); path.get('declaration').replaceWith(t.Identifier(refId)); } else if (stmt.declaration.declarations) { @@ -685,7 +728,7 @@ function putFunctionDeclsInFront (path, options) { else declPath.replaceWith(funcId); } else if (parentPath.type === 'ExportNamedDeclaration') { // If the function is exported we change the export declaration into a reference - parentPath.replaceWith(t.ExportNamedDeclaration(null, [t.ExportSpecifier(funcId, funcId)], null)); + parentPath.replaceWith(t.ExportNamedDeclaration(null, [t.ExportSpecifier(funcId, funcId)], null)); } else if (parentPath.type === 'ExportDefaultDeclaration') { parentPath.replaceWith(t.ExportDefaultDeclaration(funcId)); } @@ -728,9 +771,11 @@ export function rewriteToCaptureTopLevelVariables (path, options) { path.unshiftContainer('body', header); path.pushContainer('body', footer); + + return options; } -function ensureComponentDescriptors (path, moduleId, options) { +export function ensureComponentDescriptors (path, moduleId, options) { // check first for top level decls const varDecls = getVarDecls(path.scope); let earlyReturn = false; @@ -776,10 +821,100 @@ function ensureComponentDescriptors (path, moduleId, options) { }); } +export function replaceExportedVarDeclarations (path, moduleId, options) { + path.traverse({ + ExportNamedDeclaration (path) { + const variableDeclaration = path.get('declaration'); + if (variableDeclaration.type !== 'VariableDeclaration') return; + const [exportedVariable] = variableDeclaration.get('declarations'); + if (!exportedVariable) return; + const exportExpression = babel.parse(`var ${exportedVariable.node.id.name}; export { ${exportedVariable.node.id.name} }`).program.body[1]; + path.replaceWithMultiple([variableDeclaration.node, exportExpression]); + } + }); + + if (moduleId.includes('lively.morphic/config.js')) { + for (let i = 0; i < path.node.body.length; i++) { + const stmt = path.get('body')[i]; + if (stmt.type === 'VariableDeclaration' && stmt.node.declarations?.[0].init?.type === 'ObjectExpression') { + const { id, init } = stmt.node.declarations[0]; + stmt.get('declarations.0.init').replaceWith(t.LogicalExpression('||', babel.parse(`${options.recorderName}.${id.name}`).program.body[0].expression, init)); + } + } + } +} + +export function replaceExportedNamespaces (path, moduleName, bundler, options) { + // namespace that are directly imported or getting re-exported need to be chanelled through the module recorder + const insertNodes = []; + let i = 0; + // such that the namespaces are getting correctly updated in case a module is getting revived + path.traverse({ + ExportAllDeclaration (path) { + let dep = bundler.resolveId(path.node.source.value, moduleName); + let name = path.node.exported?.name; + const isNamed = !!name; + if (isNamed) { + insertNodes.push( + babel.parse(`const ${name} = (lively.FreezerRuntime || lively.frozenModules).exportsOf("${bundler.normalizedId(dep)}") || ${name}_namespace;`).program.body[0], + babel.parse(`export { ${name} }`).program.body[0] + ); + path.replaceWith(babel.parse(`import * as ${name}_namespace from "${path.node.source.value}";`).program.body[0]); + options.exclude.push(`${name}_namespace`); + return; + } + insertNodes.push( + babel.parse(`import * as tmp_${++i} from "${path.node.source.value}";`).program.body[0], + babel.parse(`Object.assign((lively.FreezerRuntime || lively.frozenModules).recorderFor("${bundler.normalizedId(dep)}"), tmp_${i})`).program.body[0] + ); + options.exclude.push(`tmp_${i}`); + } + }); + + const insertFrom = path.get('body').find(n => n.type !== 'ImportDeclaration' && n.type !== 'ExportAllDeclaration'); + if (insertFrom) insertFrom.insertBefore(insertNodes); + else path.pushContainer('body', insertNodes); +} + +export function replaceImportedNamespaces (path, moduleName, bundler, options) { + // namespace that are directly imported or getting re-exported need to be chanelled through the module recorder + const namespaceVars = []; + // such that the namespaces are getting correctly updated in case a module is getting revived + + path.traverse({ + ImportDeclaration (path) { + if (path.node.specifiers[0]?.type !== 'ImportNamespaceSpecifier') return; + let dep = bundler.resolveId(path.node.source.value, moduleName); + let name = path.node.specifiers[0]?.local?.name; + if (name) { + namespaceVars.push([name, dep]); + path.get('specifiers.0.local').replaceWith(t.Identifier(name + '_namespace')); + options.exclude.push(`${name}_namespace`); + } + } + }); + + // now we have to insert the the assignments of the tmp namespace imports to the initial names, + // but filtered by the recorder object + const insertFrom = path.get('body').find(n => n.type !== 'ImportDeclaration' && n.type !== 'ExportAllDeclaration'); + for (let [namespaceVar, importedModule] of namespaceVars) { + insertFrom.insertBefore(babel.parse(`const ${namespaceVar} = (lively.FreezerRuntime || lively.frozenModules).exportsOf("${bundler.normalizedId(importedModule)}") || ${namespaceVar}_namespace;`).program.body[0]); + } +} + function getExportDecls (scope) { return [...new Set(Object.values(scope.bindings).map(m => m.referencePaths.filter(m => m.parentPath.parentPath?.type.match(/ExportNamedDeclaration|ExportDefaultDeclaration/))).flat().map(m => m.parent))]; } +export function getScopeFromPath (path) { + return { + classDecls: getClassDecls(path.scope), + funcDecls: getFuncDecls(path.scope), + refs: getRefs(path.scope), + varDecls: getVarDecls(path.scope) + }; +} + function evalCodeTransform (path, state, options) { // A: Rewrite the component definitions to create component descriptors. let { moduleName } = options; @@ -800,12 +935,7 @@ function evalCodeTransform (path, state, options) { // 2. Annotate definitions with code location. This is being used by the // function-wrapper-source transform. - options.scope = { - classDecls: getClassDecls(path.scope), - funcDecls: getFuncDecls(path.scope), - refs: getRefs(path.scope), - varDecls: getVarDecls(path.scope) - }; + options.scope = getScopeFromPath(path); if (options.hasOwnProperty('evalId')) annotation.evalId = options.evalId; if (options.sourceAccessorName) annotation.sourceAccessorName = options.sourceAccessorName; @@ -1040,8 +1170,8 @@ function rewriteToRegisterModuleToCaptureSetters (path, state, options) { if (registerCall.node.callee.property.name !== 'register') { throw new Error(`rewriteToRegisterModuleToCaptureSetters: input doesn't seem to be a System.register call: ${printAst()}...`); } - const registerBody = registerCall.get('arguments.1.body.body'); - const registerReturn = arr.last(registerBody); + const registerBody = registerCall.get('arguments.1.body'); + const registerReturn = arr.last(registerBody.get('body')); if (registerReturn.node.type !== 'ReturnStatement') { throw new Error(`rewriteToRegisterModuleToCaptureSetters: input doesn't seem to be a System.register call, at return statement: ${printAst()}...`); @@ -1055,6 +1185,7 @@ function rewriteToRegisterModuleToCaptureSetters (path, state, options) { throw new Error(`rewriteToRegisterModuleToCaptureSetters: input doesn't seem to be a System.register call, at finding execute: ${printAst()}...`); } + registerBody.get('directives').find(d => d.get('value.value').node === 'use strict')?.remove(); // remove the strict directive that systemjs appears to insert path.get('directives').find(d => d.get('value.value').node === 'format esm')?.remove(); // remove esm directive if still present // in each setter function: intercept the assignments to local vars and inject capture object @@ -1094,7 +1225,7 @@ function rewriteToRegisterModuleToCaptureSetters (path, state, options) { stmt.declarations[0].id.name === options.captureObj.name); if (captureInitialize) { - registerBody[0].insertAfter(captureInitialize.node); + registerBody.get('body.0').insertAfter(captureInitialize.node); captureInitialize.remove(); } @@ -1109,7 +1240,7 @@ function rewriteToRegisterModuleToCaptureSetters (path, state, options) { stmt.declarations[0].id && stmt.declarations[0].id.name === options.sourceAccessorName); if (origSourceInitialize) { - registerBody[0].insertAfter(origSourceInitialize.node); + registerBody.get('body.0').insertAfter(origSourceInitialize.node); origSourceInitialize.remove(); } } @@ -1150,7 +1281,8 @@ export function livelyModuleLoadTranspile (api, options) { return { visitor: { CallExpression (path) { - if (path.get('callee.property').node?.name === 'register') { + if (path.get('callee.property').node?.name === 'register' && + path.get('callee.object.name').node === 'System') { options.depNames.push(...path.node.arguments[0].elements.map(ea => ea.value)); const declareFuncNode = path.node.arguments[1]; const body = path.parentPath; @@ -1218,11 +1350,15 @@ export function setupBabelTranspiler (System) { // this also overrides native requires, which is not what we want really Module._load = (...args) => { let exports = origLoad(...args); - const isCoreModule = !!System.loads?.['@node/' + args[0]]; + const isCoreModule = Module.isBuiltin(args[0]); if (isCoreModule && !args[1].loaded && !exports.prototype) { - exports = Object.assign(Object.create(exports.prototype || {}), exports); - if (!exports.default) exports.default = exports; - exports.__esModule = true; + const newExports = Object.create(exports.prototype || {}); + for (const key in Object.getOwnPropertyDescriptors(exports)) { + newExports[key] = exports[key]; + } + if (!newExports.default) newExports.default = newExports; + newExports.__esModule = true; + return newExports; } return exports; }; @@ -1240,8 +1376,8 @@ export function setupBabelTranspiler (System) { transpiler: 'lively.transpiler.babel', babelOptions: { sourceMaps: true, - compact: false, - comments: true, + compact: false, // for some reason, the compact options messes up the source maps on a per module basis + comments: false, presets: [] } }); diff --git a/lively.source-transform/index.js b/lively.source-transform/index.js index cb885b5a0a..27a3c9aa8e 100644 --- a/lively.source-transform/index.js +++ b/lively.source-transform/index.js @@ -191,7 +191,7 @@ export async function replaceExportedNamespaces (translated, moduleName, bundler } insertNodes.push( parse(`import * as tmp_${i++} from "${exportAllDecl.source.value}";`).body[0], - (async () => parse(`Object.assign((lively.FreezerRuntime || lively.frozenModules).recorderFor("${bundler.normalizedId(await dep)}"), mp_${i++})`).body[0])() + (async () => parse(`Object.assign((lively.FreezerRuntime || lively.frozenModules).recorderFor("${bundler.normalizedId(await dep)}"), tmp_${i})`).body[0])() ); return exportAllDecl; }); diff --git a/lively.source-transform/package.json b/lively.source-transform/package.json index 9e15101ecf..6a14afae76 100644 --- a/lively.source-transform/package.json +++ b/lively.source-transform/package.json @@ -2,7 +2,6 @@ "name": "lively.source-transform", "version": "0.1.7", "description": "EcmaScript 6 classes for live development", - "main": "dist/lively.source-transform.js", "type": "module", "scripts": { "test": "mocha-es6 tests/*-test.js", @@ -27,11 +26,11 @@ "lively.ast": "https://github.com/LivelyKernel/lively.ast", "lively.lang": "https://github.com/LivelyKernel/lively.lang", "lively.classes": "https://github.com/LivelyKernel/lively.classes", - "@babel/plugin-proposal-optional-catch-binding": "7.16.7", "@babel/plugin-syntax-import-meta": "7.10.4", "@babel/plugin-proposal-dynamic-import": "7.18.6", + "@babel/plugin-transform-modules-systemjs": "7.25.9", "@babel/types": "7.26.0", - "@babel/core": "7.26.0" + "@babel/core": "7.26.10" }, "devDependencies": { "lively.vm": "*", @@ -45,76 +44,6 @@ "rollup-plugin-babel": "^2.6.1" }, "systemjs": { - "main": "index.js", - "importMap": { - "imports": { - "@babel/core": "esm://ga.jspm.io/npm:@babel/core@7.26.0/lib/dev.index.js", - "@babel/plugin-proposal-dynamic-import": "esm://ga.jspm.io/npm:@babel/plugin-proposal-dynamic-import@7.18.6/lib/index.js", - "@babel/plugin-transform-modules-systemjs": "esm://ga.jspm.io/npm:@babel/plugin-transform-modules-systemjs@7.25.9/lib/index.js", - "@babel/types": "esm://ga.jspm.io/npm:@babel/types@7.26.0/lib/index.js" - }, - "scopes": { - "esm://ga.jspm.io/": { - "#lib/config/files/index.js": "esm://ga.jspm.io/npm:@babel/core@7.26.0/lib/config/files/index-browser.js", - "#lib/config/resolve-targets.js": "esm://ga.jspm.io/npm:@babel/core@7.26.0/lib/config/resolve-targets-browser.js", - "#lib/transform-file.js": "esm://ga.jspm.io/npm:@babel/core@7.26.0/lib/transform-file-browser.js", - "#node.js": "esm://ga.jspm.io/npm:browserslist@4.24.2/browser.js", - "@ampproject/remapping": "esm://ga.jspm.io/npm:@ampproject/remapping@2.3.0/dist/remapping.umd.js", - "@babel/code-frame": "esm://ga.jspm.io/npm:@babel/code-frame@7.26.2/lib/index.js", - "@babel/compat-data/native-modules": "esm://ga.jspm.io/npm:@babel/compat-data@7.26.3/native-modules.js", - "@babel/compat-data/plugins": "esm://ga.jspm.io/npm:@babel/compat-data@7.26.3/plugins.js", - "@babel/generator": "esm://ga.jspm.io/npm:@babel/generator@7.26.3/lib/index.js", - "@babel/helper-compilation-targets": "esm://ga.jspm.io/npm:@babel/helper-compilation-targets@7.25.9/lib/index.js", - "@babel/helper-module-imports": "esm://ga.jspm.io/npm:@babel/helper-module-imports@7.25.9/lib/index.js", - "@babel/helper-module-transforms": "esm://ga.jspm.io/npm:@babel/helper-module-transforms@7.26.0/lib/index.js", - "@babel/helper-plugin-utils": "esm://ga.jspm.io/npm:@babel/helper-plugin-utils@7.25.9/lib/index.js", - "@babel/helper-string-parser": "esm://ga.jspm.io/npm:@babel/helper-string-parser@7.25.9/lib/index.js", - "@babel/helper-validator-identifier": "esm://ga.jspm.io/npm:@babel/helper-validator-identifier@7.25.9/lib/index.js", - "@babel/helper-validator-option": "esm://ga.jspm.io/npm:@babel/helper-validator-option@7.25.9/lib/index.js", - "@babel/plugin-syntax-dynamic-import": "esm://ga.jspm.io/npm:@babel/plugin-syntax-dynamic-import@7.8.3/lib/index.js", - "@babel/helpers": "esm://ga.jspm.io/npm:@babel/helpers@7.26.0/lib/index.js", - "@babel/parser": "esm://ga.jspm.io/npm:@babel/parser@7.26.3/lib/index.js", - "@babel/template": "esm://ga.jspm.io/npm:@babel/template@7.25.9/lib/index.js", - "@babel/traverse": "esm://ga.jspm.io/npm:@babel/traverse@7.26.4/lib/index.js", - "@babel/types": "esm://ga.jspm.io/npm:@babel/types@7.26.3/lib/index.js", - "@jridgewell/gen-mapping": "esm://ga.jspm.io/npm:@jridgewell/gen-mapping@0.3.5/dist/gen-mapping.umd.js", - "@jridgewell/resolve-uri": "esm://ga.jspm.io/npm:@jridgewell/resolve-uri@3.1.2/dist/resolve-uri.umd.js", - "@jridgewell/set-array": "esm://ga.jspm.io/npm:@jridgewell/set-array@1.2.1/dist/set-array.umd.js", - "@jridgewell/sourcemap-codec": "esm://ga.jspm.io/npm:@jridgewell/sourcemap-codec@1.5.0/dist/sourcemap-codec.umd.js", - "@jridgewell/trace-mapping": "esm://ga.jspm.io/npm:@jridgewell/trace-mapping@0.3.25/dist/trace-mapping.umd.js", - "assert": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/assert.js", - "browserslist": "esm://ga.jspm.io/npm:browserslist@4.24.2/index.js", - "buffer": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/buffer.js", - "caniuse-lite/dist/unpacker/agents": "esm://ga.jspm.io/npm:caniuse-lite@1.0.30001687/dist/unpacker/agents.js", - "convert-source-map": "esm://ga.jspm.io/npm:convert-source-map@2.0.0/index.js", - "debug": "esm://ga.jspm.io/npm:debug@4.3.7/src/browser.js", - "electron-to-chromium/versions": "esm://ga.jspm.io/npm:electron-to-chromium@1.5.71/versions.js", - "fs": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/fs.js", - "gensync": "esm://ga.jspm.io/npm:gensync@1.0.0-beta.2/index.js", - "globals": "esm://ga.jspm.io/npm:globals@11.12.0/index.js", - "js-tokens": "esm://ga.jspm.io/npm:js-tokens@4.0.0/index.js", - "jsesc": "esm://ga.jspm.io/npm:jsesc@3.0.2/jsesc.js", - "lru-cache": "esm://ga.jspm.io/npm:lru-cache@5.1.1/index.js", - "ms": "esm://ga.jspm.io/npm:ms@2.1.3/index.js", - "node-releases/data/processed/envs.json": "esm://ga.jspm.io/npm:node-releases@2.0.18/data/processed/envs.json.js", - "node-releases/data/release-schedule/release-schedule.json": "esm://ga.jspm.io/npm:node-releases@2.0.18/data/release-schedule/release-schedule.json.js", - "path": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/path.js", - "picocolors": "esm://ga.jspm.io/npm:picocolors@1.1.1/picocolors.browser.js", - "process": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/process.js", - "semver": "esm://ga.jspm.io/npm:semver@6.3.1/semver.js", - "yallist": "esm://ga.jspm.io/npm:yallist@3.1.1/yallist.js" - } - } - }, - "map": { - "@babel/plugin-proposal-optional-catch-binding": { - "~node": "esm://cache/@babel/plugin-proposal-optional-catch-binding", - "node": "@empty" - }, - "@babel/plugin-syntax-import-meta": { - "~node": "esm://cache/@babel/plugin-syntax-import-meta", - "node": "@empty" - } - } + "main": "index.js" } } diff --git a/lively.source-transform/tests/babel-test.js b/lively.source-transform/tests/babel-test.js index 2d2e6fdab8..6eb672ad47 100644 --- a/lively.source-transform/tests/babel-test.js +++ b/lively.source-transform/tests/babel-test.js @@ -85,7 +85,7 @@ function classTemplate (className, superClassName, methodString, classMethodStri return this[Symbol.for("lively-instance-initialize")].apply(this, arguments); } };${useClassHolder ? '' : '\nvar __lively_class__ = Foo;'} - if (Object.isFrozen(__lively_classholder__)) { + if (Object.isFrozen(__lively_classholder__) || Object.isFrozen(__lively_class__.prototype)) { return __lively_class__; } return _createOrExtendClass(__lively_class__, superclass, ${methodString}, ${classMethodString}, ${ useClassHolder ? '__lively_classholder__' : 'null'}, ${moduleMeta}${pos}); @@ -592,11 +592,17 @@ bar;`); testVarTfm('re-export named', 'export { name1, name2 } from "foo";', - 'export {\n name1,\n name2\n} from "foo";'); + `import { name1 as __name1__, name2 as __name2__ } from "foo"; +_rec.name2 = __name2__; +_rec.name1 = __name1__; +export { __name1__ as name1, __name2__ as name2 };`); testVarTfm('export from named', 'export { name1 as foo1, name2 as bar2 } from "foo";', - 'export {\n name1 as foo1,\n name2 as bar2\n} from "foo";'); + `import { name1 as foo1, name2 as bar2 } from "foo"; +_rec.bar2 = bar2; +_rec.foo1 = foo1; +export { foo1, bar2 };`); testVarTfm('export bug 1', 'foo();\nexport function a() {}\nexport function b() {}', @@ -610,7 +616,9 @@ bar;`); '_rec.b = b;\n' + 'function c() {\n}\n' + '_rec.c = c;\n' + - 'export {\n a\n} from "./package-commands.js";\n' + + 'import {\n a as __a__\n} from "./package-commands.js";\n' + + '_rec.a = __a__;\n' + + 'export {\n __a__ as a\n};\n' + 'export {\n b\n};\n' + 'export {\n c\n};'); }); @@ -761,7 +769,7 @@ describe('declarations', () => { 'wraps literals that are exported as defaults', opts, 'export default 32', - '_rec.$32 = 32;\nvar $32 = _rec.$32;\nexport default $32;'); + '_rec.$exportedLiteral = 32;\nvar $exportedLiteral = _rec.$exportedLiteral;\nexport default $exportedLiteral;'); testVarTfm( 'can be wrapped in define call', @@ -860,7 +868,6 @@ return { \"foo:a.js\", \"http://zork/b.js\" ], function (_export, _context) { - \"use strict\"; var x, y, z, _rec; _rec = System.get(\"@lively-env\").moduleEnv(\"c.js\").recorder; return { @@ -887,7 +894,6 @@ return { \"foo:a.js\", \"http://zork/b.js\" ], function (_export, _context) { - \"use strict\"; var x, y, z, _rec; _rec = System.get(\"@lively-env\").moduleEnv(\"c.js\").recorder; return { diff --git a/lively.source-transform/tests/capturing-test.js b/lively.source-transform/tests/capturing-test.js index 349f245f9c..2a1d7c8c41 100644 --- a/lively.source-transform/tests/capturing-test.js +++ b/lively.source-transform/tests/capturing-test.js @@ -61,7 +61,7 @@ function classTemplate (className, superClassName, methodString, classMethodStri return this[Symbol.for("lively-instance-initialize")].apply(this, arguments); } };${useClassHolder ? '' : '\nvar __lively_class__ = Foo;'} - if (Object.isFrozen(__lively_classholder__)) { + if (Object.isFrozen(__lively_classholder__) || Object.isFrozen(__lively_class__.prototype)) { return __lively_class__; } return _createOrExtendClass(__lively_class__, superclass, ${methodString}, ${classMethodString}, ${ useClassHolder ? '__lively_classholder__' : 'null'}, ${moduleMeta}${pos}); diff --git a/lively.storage/package.json b/lively.storage/package.json index 629ca201fc..481704cad9 100644 --- a/lively.storage/package.json +++ b/lively.storage/package.json @@ -20,8 +20,8 @@ "dependencies": { "lively.lang": "^1.0.0", "lively.resources": "^0.1.18", - "pouchdb": "7.3.0", - "pouchdb-adapter-memory": "7.3.0" + "pouchdb": "7.2.1", + "pouchdb-adapter-memory": "7.2.1" }, "devDependencies": { "babel-core": "^6.16.0", @@ -44,18 +44,5 @@ "test": "mocha-es6 tests/*-test.js", "build": "node ./tools/build.js" }, - "main": "index.js", - "systemjs": { - "main": "index.js", - "map": { - "pouchdb": { - "node": "pouchdb/lib/index.js", - "~node": "esm://cache/pouchdb@7.2.1" - }, - "pouchdb-adapter-memory": { - "node": "./dist/pouchdb-adapter-mem.js", - "~node": "esm://cache/pouchdb-adapter-memory@7.2.1" - } - } - } + "main": "index.js" } diff --git a/mocha-es6/index.js b/mocha-es6/index.js index bcfd156842..116a23e9fe 100644 --- a/mocha-es6/index.js +++ b/mocha-es6/index.js @@ -2,6 +2,7 @@ import * as modules from 'lively.modules'; import { withMozillaAstDo } from 'lively.ast'; +import { obj } from 'lively.lang'; import mocha from 'mocha'; import chai from 'chai'; import chaiSubset from 'chai-subset'; @@ -37,11 +38,6 @@ chai.Assertion.addChainableMethod('stringEquals', function (obj) { }); function lively_equals (_super) { - return function (other) { - if (this.__flags.deep) return _super.apply(this, arguments); - else if (Array.isArray(this._obj) && arrayEquals(this._obj, other)) { /* do nothin' */ } else if (this._obj && typeof this._obj.equals === 'function' && this._obj.equals(other)) { /* do nothin' */ } else _super.apply(this, arguments); - }; - function arrayEquals (array, otherArray) { let len = array.length; if (!otherArray || len !== otherArray.length) return false; @@ -61,6 +57,11 @@ function lively_equals (_super) { } return true; } + + return function (other) { + if (this.__flags.deep) return _super.apply(this, arguments); + else if (Array.isArray(this._obj) && arrayEquals(this._obj, other)) { /* do nothin' */ } else if (this._obj && typeof this._obj.equals === 'function' && this._obj.equals(other)) { /* do nothin' */ } else _super.apply(this, arguments); + }; } chai.Assertion.overwriteMethod('equal', lively_equals); @@ -71,8 +72,6 @@ chai.Assertion.overwriteMethod('equals', lively_equals); // default reporter, logs to console // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- -export { loadTestModuleAndExtractTestState, runTestFiles, chai, mocha, expect }; - function ConsoleReporter (runner) { let passes = 0; let failures = 0; @@ -189,6 +188,15 @@ async function test () { await runTestFiles([file], { package: 'http://localhost:9001/node_modules/lively.ast' }); } +function installMochaEs6ModuleExecute (load, executable, options = {}) { + // this is called from a System.instantiate hook to wrap the execution of the + // test module body. This is needed b/c mocha expects globals to be present. + // We can't just simply install those globally b/c the test context needs to be + // bound into those functions and it is individual for each test module + let origExecute = executable.execute; + executable.execute = () => recordTestsWhile(load.name, origExecute); +} + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- // System loader extension // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- @@ -232,15 +240,6 @@ export async function isMochaTestLoad (load, executable) { return isTest; } -function installMochaEs6ModuleExecute (load, executable, options = {}) { - // this is called from a System.instantiate hook to wrap the execution of the - // test module body. This is needed b/c mocha expects globals to be present. - // We can't just simply install those globally b/c the test context needs to be - // bound into those functions and it is individual for each test module - let origExecute = executable.execute; - executable.execute = () => recordTestsWhile(load.name, origExecute); -} - function recordTestsWhile (file, whileFn, options = {}) { let module = modules.module(file); var options = { @@ -249,8 +248,7 @@ function recordTestsWhile (file, whileFn, options = {}) { ...options }; let logger = options.logger || console; - let _mocha = mocha || global.mocha; - let _Mocha = _mocha.constructor || global.Mocha; + let _Mocha = (obj.isFunction(mocha.constructor) ? mocha : mocha.constructor) || global.Mocha; let m = options.mocha || (options.mocha = new _Mocha({ reporter: options.reporter || ConsoleReporter })); module.define('mocha', m); @@ -287,3 +285,5 @@ function recordTestsWhile (file, whileFn, options = {}) { // System.instantiate = System.instantiate.originalFunction.originalFunction installSystemInstantiateHook(); + +export { loadTestModuleAndExtractTestState, runTestFiles, chai, mocha, expect }; diff --git a/mocha-es6/package.json b/mocha-es6/package.json index 90ac42938d..502c5c5660 100644 --- a/mocha-es6/package.json +++ b/mocha-es6/package.json @@ -3,14 +3,15 @@ "main": "index.js", "version": "0.5.6", "dependencies": { - "flatn": "^1.3.3", + "flatn": "*", + "lively.modules": "*", + "lively.ast": "*", + "lively.lang": "*", "glob": "^7.0.3", - "lively.modules": "^0.8.38", - "lively.ast": "^0.11.1", "minimist": "^1.2.0", "systemjs": "^0.21.6", - "mocha": "^10.0.0", - "chai": "^4.3.6", + "mocha": "10.0.0", + "chai": "4.3.6", "chai-subset": "1.6.0" }, "devDependencies": { @@ -31,77 +32,9 @@ "scripts": { "build": "node tools/build.js" }, - "license": "MIT", - "author": "Robert Krahn", "systemjs": { - "main": "index.js", - "importMap": { - "imports": { - "chai": "esm://ga.jspm.io/npm:chai@4.3.6/index.mjs", - "mocha": "esm://ga.jspm.io/npm:mocha@10.0.0/dev.browser-entry.js" - }, - "scopes": { - "esm://ga.jspm.io/": { - "#lib/nodejs/esm-utils.js": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/@empty.js", - "#lib/nodejs/file-unloader.js": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/@empty.js", - "#lib/nodejs/parallel-buffered-runner.js": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/@empty.js", - "assertion-error": "esm://ga.jspm.io/npm:assertion-error@1.1.0/index.js", - "browser-stdout": "esm://ga.jspm.io/npm:browser-stdout@1.3.1/index.js", - "buffer": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/buffer.js", - "check-error": "esm://ga.jspm.io/npm:check-error@1.0.3/index.js", - "debug": "esm://ga.jspm.io/npm:debug@4.3.4/src/browser.js", - "deep-eql": "esm://ga.jspm.io/npm:deep-eql@3.0.1/index.js", - "diff": "esm://ga.jspm.io/npm:diff@5.0.0/lib/index.js", - "escape-string-regexp": "esm://ga.jspm.io/npm:escape-string-regexp@4.0.0/index.js", - "events": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/events.js", - "fs": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/fs.js", - "get-func-name": "esm://ga.jspm.io/npm:get-func-name@2.0.2/index.js", - "he": "esm://ga.jspm.io/npm:he@1.2.0/he.js", - "log-symbols": "esm://ga.jspm.io/npm:log-symbols@4.1.0/browser.js", - "loupe": "esm://ga.jspm.io/npm:loupe@2.3.7/loupe.js", - "ms": "esm://ga.jspm.io/npm:ms@2.1.3/index.js", - "nanoid/non-secure": "esm://ga.jspm.io/npm:nanoid@3.3.3/non-secure/index.cjs", - "path": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/path.js", - "pathval": "esm://ga.jspm.io/npm:pathval@1.1.1/index.js", - "process": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/process.js", - "stream": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/stream.js", - "supports-color": "esm://ga.jspm.io/npm:supports-color@8.1.1/browser.js", - "type-detect": "esm://ga.jspm.io/npm:type-detect@4.1.0/type-detect.js", - "util": "esm://ga.jspm.io/npm:@jspm/core@2.1.0/nodelibs/browser/util.js" - }, - "esm://ga.jspm.io/npm:debug@4.3.4/": { - "ms": "esm://ga.jspm.io/npm:ms@2.1.2/index.js" - } - } - }, - "meta": { - "dist/mocha.js": { - "format": "global", - "exports": "mocha" - }, - "dist/chai.js": { - "format": "global", - "exports": "chai" - } - }, - "map": { - "chai-subset": { - "~node": "esm://ga.jspm.io/npm:chai-subset@1.6.0/lib/chai-subset.js" - }, - "chai": { - "node": "./dist/chai.js" - }, - "mocha": { - "node": "./dist/mocha.js" - }, - "fs": { - "node": "@node/fs", - "~node": "@empty" - }, - "path": { - "node": "@node/path", - "~node": "@empty" - } - } - } -} \ No newline at end of file + "nodeRequirePackages": ["chai", "mocha"] + }, + "license": "MIT", + "author": "Robert Krahn" +} diff --git a/scripts/check-boot.sh b/scripts/check-boot.sh index 2a9dfea1dd..5868951e01 100755 --- a/scripts/check-boot.sh +++ b/scripts/check-boot.sh @@ -3,4 +3,4 @@ lv_next_dir=$PWD . $lv_next_dir/scripts/lively-next-env.sh lively_next_env $lv_next_dir -node --no-experimental-fetch --no-warnings --experimental-import-meta-resolve --experimental-loader ./flatn/resolver.mjs ./scripts/check_boot.js +node --no-warnings --experimental-import-meta-resolve --experimental-loader ./flatn/resolver.mjs ./scripts/check_boot.js diff --git a/scripts/check-build-status.py b/scripts/check-build-status.py index 6f1d92a045..ca8ff2f783 100755 --- a/scripts/check-build-status.py +++ b/scripts/check-build-status.py @@ -8,6 +8,9 @@ import json from sultan.api import Sultan import os +import difflib +import tempfile +import subprocess # branch against which to diff changes target_branch_name = "main" @@ -16,6 +19,54 @@ artifacts_to_check = ["flatn/flatn-cjs.js"] # flag used to check if the script should fail (i.e. at least one dependent needs to be rebuild) fail = False + +def show_file_diff(artifact_path): + """Show the differences between committed and newly built version of a file.""" + try: + # Get the committed version using git show + result = subprocess.run( + ['git', 'show', f'HEAD:{artifact_path}'], + capture_output=True, + text=True, + check=True + ) + committed_content = result.stdout.splitlines(keepends=True) + + # Read the newly built version + with open(artifact_path, 'r', encoding='utf-8', errors='replace') as f: + new_content = f.readlines() + + # Generate unified diff + diff = difflib.unified_diff( + committed_content, + new_content, + fromfile=f'{artifact_path} (committed)', + tofile=f'{artifact_path} (new build)', + lineterm='' + ) + + print("\n📝 Differences found:") + print("=" * 80) + + # Show diff with line limit to avoid overwhelming output + diff_lines = list(diff) + max_lines = 100 + + for i, line in enumerate(diff_lines): + if i >= max_lines: + remaining = len(diff_lines) - max_lines + print(f"\n... ({remaining} more lines of differences omitted)") + break + print(line) + + print("=" * 80) + + except subprocess.CalledProcessError as e: + print(f"⚠️ Could not get committed version: {e}") + except FileNotFoundError: + print(f"⚠️ Could not read file: {artifact_path}") + except Exception as e: + print(f"⚠️ Error showing diff: {e}") for artifact in artifacts_to_check: # flag to determine if this specific dependent is ok, just for nicer output single_fail = False @@ -26,6 +77,7 @@ new_checksum = s.md5sum(f"{artifact}").run().stdout if new_checksum != committed_checksum: print(f"❌ {artifact} needs to be rebuild!") + show_file_diff(artifact) fail = True single_fail = True break diff --git a/scripts/lively-next-env.sh b/scripts/lively-next-env.sh index 6a9355d548..2bf387d074 100644 --- a/scripts/lively-next-env.sh +++ b/scripts/lively-next-env.sh @@ -2,7 +2,7 @@ function lively_next_env { echo "Setting V8 memory size limit to allow space for builds" - export NODE_OPTIONS=--max_old_space_size=4096 + export NODE_OPTIONS=--max_old_space_size=8192 lv_next_dir=$1 export PUPPETEER_CACHE_DIR=$lv_next_dir/.puppeteer-browser-cache echo "Setting env vars for FLATN_PACKAGE_DIRS, FLATN_PACKAGE_COLLECTION_DIRS, FLATN_DEV_PACKAGE_DIRS for lively.next" diff --git a/scripts/lively-next-flatn-env.sh b/scripts/lively-next-flatn-env.sh index 554f7441d4..e8eff72246 100644 --- a/scripts/lively-next-flatn-env.sh +++ b/scripts/lively-next-flatn-env.sh @@ -4,7 +4,7 @@ # DO NOT USE IT FOR NEWER CODE. USE `lively-next-env.sh` INSTEAD. function lively_next_flatn_env { echo "Setting V8 memory size limit to allow space for builds" - export NODE_OPTIONS=--max_old_space_size=4096 + export NODE_OPTIONS=--max_old_space_size=8192 lv_next_dir=$1 export PUPPETEER_CACHE_DIR=$lv_next_dir/.puppeteer-browser-cache echo "Setting env vars for FLATN_PACKAGE_DIRS, FLATN_PACKAGE_COLLECTION_DIRS, FLATN_DEV_PACKAGE_DIRS for lively.next" diff --git a/scripts/node_version_checker.sh b/scripts/node_version_checker.sh index 2510c6582b..2d2a2d2b27 100755 --- a/scripts/node_version_checker.sh +++ b/scripts/node_version_checker.sh @@ -3,7 +3,7 @@ NODE_VERSION=$(node -v) NODE_VERSION=$(echo "$NODE_VERSION" | sed -En 's/v([0-9]+)\..*/\1/p') -if [[ $NODE_VERSION -lt 18 ]]; then - echo -n "Your node version is not supported. Please use node 18.X or higher."; echo; +if [[ $NODE_VERSION -lt 24 ]]; then + echo -n "Your node version is not supported. Please use node 24.X or higher."; echo; exit 1; fi diff --git a/scripts/test.sh b/scripts/test.sh index 5994b22512..e73fc1c43b 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -19,6 +19,19 @@ SKIPPED_TESTS=0 STARTED_SERVER=0 FAILURE=0 +# Clean up any lingering headless Chrome processes and lock files +# These can remain from previous test runs and block new browser instances +echo "Cleaning up any lingering Chrome processes..." +pkill -f "chrome.*--headless" 2>/dev/null || true +pkill -f "chromium.*--headless" 2>/dev/null || true + +# Remove Chrome singleton lock file if it exists +CHROME_LOCK_FILE="lively.headless/chrome-data-dir/SingletonLock" +if [ -f "$CHROME_LOCK_FILE" ]; then + echo "Removing stale Chrome lock file: $CHROME_LOCK_FILE" + rm -f "$CHROME_LOCK_FILE" +fi + testfiles=( "lively.lang" "lively.resources" @@ -78,15 +91,15 @@ fi if [ ! "$CI" ]; then if uname | grep 'Linux' > /dev/null; then - ACTIVE_PORTS=$(ss -lt) + ACTIVE_PORTS=$(ss -lt 2>/dev/null) elif uname | grep 'Darwin' > /dev/null; then - ACTIVE_PORTS=$(netstat -tunlp tcp) - else + ACTIVE_PORTS=$(netstat -tunlp tcp 2>/dev/null) + else cat "Only MacOS and Linux are supported at the moment." exit 1 fi - if grep -E '(:9011|.9011)' > /dev/null <<< "$ACTIVE_PORTS"; then + if grep -E '(:9011|\.9011)' > /dev/null <<< "$ACTIVE_PORTS"; then echo "Found a running lively server on port 9011 that will be used for testing." else STARTED_SERVER=1 @@ -145,6 +158,11 @@ then pkill -f -n lively.*start fi +# Clean up any headless Chrome processes that may have been started during tests +echo "Cleaning up Chrome processes after tests..." +pkill -f "chrome.*--headless" 2>/dev/null || true +pkill -f "chromium.*--headless" 2>/dev/null || true + ((ALL_TESTS=GREEN_TESTS + RED_TESTS + SKIPPED_TESTS)) if [ ! "$1" ]; then