diff --git a/.env.sample b/.env.sample index 05699699..de4d9555 100644 --- a/.env.sample +++ b/.env.sample @@ -79,6 +79,7 @@ POOL_DESTROY_TIMEOUT = 5000 POOL_IDLE_TIMEOUT = 30000 POOL_CREATE_RETRY_INTERVAL = 200 POOL_REAPER_INTERVAL = 1000 +POOL_RESOURCES_INTERVAL = 30000 POOL_BENCHMARKING = false # LOGGING CONFIG @@ -98,6 +99,7 @@ OTHER_LISTEN_TO_PROCESS_EXITS = true OTHER_NO_LOGO = false OTHER_HARD_RESET_PAGE = false OTHER_BROWSER_SHELL_MODE = true +OTHER_CONNECTION_OVER_PIPE = false OTHER_VALIDATION = true # DEBUG CONFIG diff --git a/CHANGELOG.md b/CHANGELOG.md index 61c9a0d5..72ee4332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ _New Features:_ - Added the `validateOption` function for validating a single option. It is used in the code to validate individual options (`svg`, `instr`, `resources`, `customCode`, `callback`, `globalOptions`, and `themeOptions`) loaded from a file. - Added the `validateOptions` function for validating the full set of options. It is used in the code to validate options coming from functions that update global options, CLI arguments, configurations loaded via `--loadConfig`, and configurations created using the prompts functionality. - Introduced redefined `getOptions` and `updateOptions` functions to retrieve and update the original global options or a copy of global options, allowing flexibility in export scenarios. +- Added a toggleable mechanism to ensure the correct number of resources exist at any given time on an interval, creating them if necessary. +- Added the `OTHER_CONNECTION_OVER_PIPE` option for a more stable and optimized connection to the browser using a `Pipe` instead of `WebSockets`. +- Added an optional mechanism to reconnect the browser in case of an unexpected disconnect (only applicable for `WebSocket` connections). +- Added an optional health-checking mechanism for resource pages, with automatic recreation if needed (only applicable for `WebSocket` connections). - Introduced the ability to enable a customizable `WebSocket` connection between the Export Server instance and any server or service that supports such connections to collect chart options usage data. This is useful for gathering telemetry data. - Added a simple filtering mechanism (based on the `./lib/schemas/telemetry.json` file) to control the data being sent. - Added a new option called `uploadLimit` to control the maximum size of a request's payload body. @@ -56,6 +60,7 @@ _Enhancements:_ - Created `_checkDataSize` for handling the data size validation. - Optimized `initExport`, utilizing `updateOptions` for global option updates. - The `initExport` now have its options parameters defaulted to an empty object, using global option values if none are provided. +- Various optimizations and performance improvements in functions like `createBrowser`, `killPool`, and `shutdownCleanUp` to enhance performance and address memory issues. - Updated exported API functions for module usage. - Adjusted imports to get functions from corresponding modules rather than `index.js`. - Server functions are now directly exported (rather than within a `server` object) as API functions. @@ -75,6 +80,7 @@ _Enhancements:_ - Refactored and split the `_updateCache` function into smaller, more manageable parts. - The `updateHcVersion` function now correctly updates the global options. - The new `_configureRequest` function is responsible for setting the proxy agent. +- Improved logging with more detailed information about rejected requests at an early stage (in the `validation` middleware). - Passing `version` as the first argument to `_saveConfigToManifest`. - Removed the `fetchAndProcessScript` and `fetchScripts` functions, replacing them with a single function, `_fetchScript`, for the same purpose. - Renamed the `checkAndUpdateCache` function to `checkCache`, the `updateVersion` function to `updateHcVersion`, the `version` function to `getHcVersion`, the `saveConfigToManifest` function to `_saveConfigToManifest`, the `extractModuleName` function to `_extractModuleName`, the `extractVersion` function to `extractHcVersion`, and the `cdnURL` property to `cdnUrl` in the `cache.js` module. diff --git a/lib/browser.js b/lib/browser.js index 1c0dd88d..6d520de7 100644 --- a/lib/browser.js +++ b/lib/browser.js @@ -29,7 +29,8 @@ import { getCachePath } from './cache.js'; import { getOptions } from './config.js'; import { setupHighcharts } from './highcharts.js'; import { log, logWithStack } from './logger.js'; -import { __dirname, getAbsolutePath } from './utils.js'; +import { __dirname, expBackoff, getAbsolutePath } from './utils.js'; +import { envs } from './validation.js'; import ExportError from './errors/ExportError.js'; @@ -42,6 +43,9 @@ const pageTemplate = readFileSync( // To save the browser let browser = null; +// To save the WebSocket endpoint in case of a sudden disconnect +let wsEndpoint; + /** * Retrieves the existing Puppeteer browser instance. * @@ -87,6 +91,8 @@ export async function createBrowser(puppeteerArgs) { headless: other.browserShellMode ? 'shell' : true, userDataDir: 'tmp', args: puppeteerArgs || [], + // Must be disabled for debugging to work + pipe: envs.OTHER_CONNECTION_OVER_PIPE, handleSIGINT: false, handleSIGTERM: false, handleSIGHUP: false, @@ -108,6 +114,23 @@ export async function createBrowser(puppeteerArgs) { // Launch the browser browser = await puppeteer.launch(launchOptions); + + // Close the initial pages if any found + const pages = await browser.pages(); + if (pages) { + for (const page of pages) { + await page.close(); + } + } + + // Only for the WebSocket connection + if (!launchOptions.pipe) { + // Save WebSocket endpoint + wsEndpoint = browser.wsEndpoint(); + + // Attach the disconnected event + browser.on('disconnected', _reconnect); + } } catch (error) { logWithStack( 1, @@ -203,7 +226,7 @@ export async function newPage(poolResource) { await _setPageContent(poolResource.page); // Set page events - _setPageEvents(poolResource.page); + _setPageEvents(poolResource); // Check if the page is correctly created if (!poolResource.page || poolResource.page.isClosed()) { @@ -431,6 +454,61 @@ export async function clearPageResources(page, injectedResources) { } } +/** + * Reconnects to the browser instance when it is disconnected. If the current + * browser connection is lost, it attempts to reconnect using the previous + * WebSocket endpoint. If the reconnection fails, it will try to close + * the browser and relaunch a new instance. + * + * @async + * @function _reconnect + */ +async function _reconnect() { + try { + // Start the reconnecting + log(3, `[browser] Restarting the browser connection.`); + + // Try to reconnect the browser + if (browser && !browser.connected) { + browser = await puppeteer.connect({ + browserWSEndpoint: wsEndpoint + }); + } + + // Save a new WebSocket endpoint + wsEndpoint = browser.wsEndpoint(); + + // Add the reconnect event again + browser.on('disconnected', _reconnect); + + // Log the success message + log(3, `[browser] Browser reconnected successfully.`); + } catch (error) { + logWithStack( + 1, + error, + '[browser] Could not restore the browser connection, attempting to relaunch.' + ); + + // Try to close the browser before relaunching + try { + await close(); + } catch (error) { + logWithStack( + 1, + error, + '[browser] Could not close the browser before relaunching (probably is already closed).' + ); + } + + // Try to relaunch the browser + await createBrowser(getOptions().puppeteer.args || []); + + // Log the success message + log(3, `[browser] Browser relaunched successfully.`); + } +} + /** * Sets the content for a Puppeteer page using a predefined template * and additional scripts. @@ -458,28 +536,81 @@ async function _setPageContent(page) { * * @function _setPageEvents * - * @param {Object} page - The Puppeteer page object to which the listeners - * are being set. + * @param {Object} poolResource - The pool resource that contains `id`, + * `workCount`, and `page`. */ -function _setPageEvents(page) { +function _setPageEvents(poolResource) { // Get `debug` options - const { debug } = getOptions(); + const { debug, pool } = getOptions(); // Set the `pageerror` listener - page.on('pageerror', async () => { + poolResource.page.on('pageerror', async () => { // It would seem like this may fire at the same time or shortly before // a page is closed. - if (page.isClosed()) { + if (poolResource.page.isClosed()) { return; } }); // Set the `console` listener, if needed if (debug.enable && debug.listenToConsole) { - page.on('console', (message) => { + poolResource.page.on('console', (message) => { console.log(`[debug] ${message.text()}`); }); } + + // Add the framedetached event if the connection is over WebSocket + if (envs.OTHER_CONNECTION_OVER_PIPE === false) { + poolResource.page.on('framedetached', async (frame) => { + // Get the main frame + const mainFrame = poolResource.page.mainFrame(); + + // Check if a page's frame is detached and requires to be recreated + if ( + frame === mainFrame && + mainFrame.detached && + poolResource.workCount <= pool.workLimit + ) { + log( + 3, + `[browser] Pool resource [${poolResource.id}] - Page's frame detached.` + ); + try { + // Try to connect to a new page using exponential backoff strategy + expBackoff( + async (poolResourceId, poolResource) => { + try { + // Try to close the page with a detached frame + if (!poolResource.page.isClosed()) { + await poolResource.page.close(); + } + } catch (error) { + log( + 3, + `[browser] Pool resource [${poolResourceId}] - Could not close the page with a detached frame.` + ); + } + + // Trigger a page creation + await newPage(poolResource); + }, + 0, + poolResource.id, + poolResource + ); + } catch (error) { + logWithStack( + 3, + error, + `[browser] Pool resource [${poolResource.id}] - Could not create a new page.` + ); + + // Set the `workLimit` to exceeded in order to recreate the resource + poolResource.workCount = pool.workLimit + 1; + } + } + }); + } } export default { diff --git a/lib/pool.js b/lib/pool.js index 2c8fd9e7..8ae6857c 100644 --- a/lib/pool.js +++ b/lib/pool.js @@ -26,7 +26,9 @@ import { v4 as uuid } from 'uuid'; import { clearPage, createBrowser, closeBrowser, newPage } from './browser.js'; import { puppeteerExport } from './export.js'; import { log, logWithStack } from './logger.js'; +import { addTimer } from './timer.js'; import { getNewDateTime, measureTime } from './utils.js'; +import { envs } from './validation.js'; import ExportError from './errors/ExportError.js'; @@ -136,6 +138,12 @@ export async function initPool(poolOptions, puppeteerArgs) { pool.release(resource); }); + // Init the interval for checking if the minimum number of resources exist + if (envs.POOL_RESOURCES_INTERVAL) { + // Register interval for the later clearing + addTimer(_checkingResourcesInterval(envs.POOL_RESOURCES_INTERVAL)); + } + log( 3, `[pool] The pool is ready${initialResources.length ? ` with ${initialResources.length} initial resources waiting.` : '.'}` @@ -168,6 +176,11 @@ export async function killPool() { pool.release(worker.resource); } + // Remove all attached event listeners from the pool + pool.removeAllListeners('release'); + pool.removeAllListeners('destroySuccess'); + pool.removeAllListeners('destroyFail'); + // Destroy the pool if it is still available if (!pool.destroyed) { await pool.destroy(); @@ -610,6 +623,47 @@ function _factory(poolOptions) { }; } +/** + * Periodically checks and ensures the minimum number of resources in the pool. + * If the total number of used, free and about to be created resources falls + * below the minimum set with the `pool.min`, it creates additional resources to + * meet the minimum requirement. + * + * @function _checkingResourcesInterval + * + * @param {number} resourceCheckInterval - The interval, in milliseconds, at + * which the pool resources are checked. + * + * @returns {NodeJS.Timeout} - Returns a timer ID that can be used to clear the + * interval later. + */ +function _checkingResourcesInterval(resourceCheckInterval) { + // Set the interval for checking the number of pool resources + return setInterval(async () => { + try { + // Get the current number of resources + let currentNumber = + pool.numUsed() + pool.numFree() + pool.numPendingCreates(); + + // Create missing resources + while (currentNumber++ < pool.min) { + try { + // Explicitly creating a resource + await pool._doCreate(); + } catch (error) { + logWithStack(2, error, '[pool] Could not create a missing resource.'); + } + } + } catch (error) { + logWithStack( + 1, + error, + `[pool] Something went wrong when trying to create missing resources.` + ); + } + }, resourceCheckInterval); +} + export default { initPool, killPool, diff --git a/lib/resourceRelease.js b/lib/resourceRelease.js index db901662..c043d988 100644 --- a/lib/resourceRelease.js +++ b/lib/resourceRelease.js @@ -17,6 +17,7 @@ See LICENSE file in root for details. * proper cleanup of resources such as browser, pages, servers, and timers. */ +import { getBrowser } from './browser.js'; import { killPool } from './pool.js'; import { clearAllTimers } from './timer.js'; @@ -35,6 +36,9 @@ import { terminateClients } from './server/webSocket.js'; * The default value is `0`. */ export async function shutdownCleanUp(exitCode = 0) { + // Remove all attached event listeners from the browser + getBrowser().removeAllListeners('disconnected'); + // Await freeing all resources await Promise.allSettled([ // Clear all ongoing intervals diff --git a/lib/server/middlewares/validation.js b/lib/server/middlewares/validation.js index 2ea2ca32..1c374486 100644 --- a/lib/server/middlewares/validation.js +++ b/lib/server/middlewares/validation.js @@ -127,9 +127,18 @@ function requestBodyMiddleware(request, response, next) { if (instr === null && !body.svg) { log( 2, - `[validation] Request [${requestId}] - The request from ${ + `[validation] Request [${requestId}] - The request with ID ${requestId} from ${ request.headers['x-forwarded-for'] || request.connection.remoteAddress - } was incorrect. Received payload is missing correct chart data for export: ${JSON.stringify(body)}.` + } was incorrect: + Content-Type: ${request.headers['content-type']}. + Chart constructor: ${body.constr}. + Dimensions: ${body.width}x${body.height} @ ${body.scale} scale. + Type: ${body.type}. + Is SVG set? ${typeof body.svg !== 'undefined'}. + B64? ${typeof body.b64 !== 'undefined'}. + No download? ${typeof body.noDownload !== 'undefined'}. + Received payload is missing correct chart data for export: ${JSON.stringify(body)}. + ` ); throw new ExportError( diff --git a/lib/validation.js b/lib/validation.js index b3b1c5ed..d93b14e4 100644 --- a/lib/validation.js +++ b/lib/validation.js @@ -1899,40 +1899,40 @@ export const validators = { }, /** - * The `poolBenchmarking` validator that returns a Zod schema with an optional - * stricter check based on the `strictCheck` parameter. + * The `resourcesInterval` validator that returns a Zod schema with + * an optional stricter check based on the `strictCheck` parameter. * - * The validation schema ensures the same work as the `boolean` validator. + * The validation schema ensures the same work as the `nonNegativeNum` + * validator. * - * @function poolBenchmarking + * @function resourcesInterval * * @param {boolean} strictCheck - Determines if stricter validation should * be applied. * * @returns {z.ZodSchema} A Zod schema object for validating - * the `poolBenchmarking` option. + * the `resourcesInterval` option. */ - poolBenchmarking(strictCheck) { - return v.boolean(strictCheck); + resourcesInterval(strictCheck) { + return v.nonNegativeNum(strictCheck); }, /** - * The `resourcesInterval` validator that returns a Zod schema with - * an optional stricter check based on the `strictCheck` parameter. + * The `poolBenchmarking` validator that returns a Zod schema with an optional + * stricter check based on the `strictCheck` parameter. * - * The validation schema ensures the same work as the `nonNegativeNum` - * validator. + * The validation schema ensures the same work as the `boolean` validator. * - * @function resourcesInterval + * @function poolBenchmarking * * @param {boolean} strictCheck - Determines if stricter validation should * be applied. * * @returns {z.ZodSchema} A Zod schema object for validating - * the `resourcesInterval` option. + * the `poolBenchmarking` option. */ - resourcesInterval(strictCheck) { - return v.nonNegativeNum(strictCheck); + poolBenchmarking(strictCheck) { + return v.boolean(strictCheck); }, /** @@ -2198,6 +2198,21 @@ export const validators = { return v.boolean(strictCheck); }, + /** + * The `connectionOverPipe` validator that returns a Zod schema with + * an optional stricter check based on the `strictCheck` parameter. + * + * The validation schema ensures the same work as the `boolean` validator. + * + * @function connectionOverPipe + * + * @returns {z.ZodSchema} A Zod schema object for validating + * the `connectionOverPipe` option. + */ + connectionOverPipe(strictCheck) { + return v.boolean(strictCheck); + }, + /** * The `validation` validator that returns a Zod schema with an optional * stricter check based on the `strictCheck` parameter. @@ -2858,6 +2873,7 @@ export const EnvSchema = z.object({ POOL_IDLE_TIMEOUT: validators.idleTimeout(false), POOL_CREATE_RETRY_INTERVAL: validators.createRetryInterval(false), POOL_REAPER_INTERVAL: validators.reaperInterval(false), + POOL_RESOURCES_INTERVAL: validators.resourcesInterval(false), POOL_BENCHMARKING: validators.poolBenchmarking(false), // logging @@ -2877,6 +2893,7 @@ export const EnvSchema = z.object({ OTHER_NO_LOGO: validators.noLogo(false), OTHER_HARD_RESET_PAGE: validators.hardResetPage(false), OTHER_BROWSER_SHELL_MODE: validators.browserShellMode(false), + OTHER_CONNECTION_OVER_PIPE: validators.connectionOverPipe(false), OTHER_VALIDATION: validators.validation(false), // debugger