From 088692529549d3c39c826fefa16ad3b6aee42cdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Wed, 9 Oct 2024 13:19:21 +0100 Subject: [PATCH 1/8] feat: write metadata file --- .../zip-it-and-ship-it/src/feature_flags.ts | 4 +- .../src/runtimes/node/index.ts | 2 + .../src/runtimes/node/utils/entry_file.ts | 1 + .../src/runtimes/node/utils/metadata_file.ts | 11 ++++ .../src/runtimes/node/utils/zip.ts | 12 +++-- .../src/runtimes/runtime.ts | 1 + packages/zip-it-and-ship-it/src/zip.ts | 3 ++ .../zip-it-and-ship-it/tests/v2api.test.ts | 54 ++++++++++++++----- 8 files changed, 67 insertions(+), 21 deletions(-) create mode 100644 packages/zip-it-and-ship-it/src/runtimes/node/utils/metadata_file.ts diff --git a/packages/zip-it-and-ship-it/src/feature_flags.ts b/packages/zip-it-and-ship-it/src/feature_flags.ts index c8198d07e1..9f1a39ab25 100644 --- a/packages/zip-it-and-ship-it/src/feature_flags.ts +++ b/packages/zip-it-and-ship-it/src/feature_flags.ts @@ -33,8 +33,8 @@ export const defaultFlags = { // Adds the `___netlify-telemetry.mjs` file to the function bundle. zisi_add_instrumentation_loader: true, - // Adds a `___netlify-bootstrap-version` file to the function bundle. - zisi_add_version_file: false, + // Adds a `___netlify-metadata.json` file to the function bundle. + zisi_add_metadata_file: false, } as const export type FeatureFlags = Partial> diff --git a/packages/zip-it-and-ship-it/src/runtimes/node/index.ts b/packages/zip-it-and-ship-it/src/runtimes/node/index.ts index 3ed6e10eb8..aedf137231 100644 --- a/packages/zip-it-and-ship-it/src/runtimes/node/index.ts +++ b/packages/zip-it-and-ship-it/src/runtimes/node/index.ts @@ -37,6 +37,7 @@ const getSrcFilesWithBundler: GetSrcFilesFunction = async (parameters) => { const zipFunction: ZipFunction = async function ({ archiveFormat, basePath, + branch, cache, config = {}, destFolder, @@ -113,6 +114,7 @@ const zipFunction: ZipFunction = async function ({ aliases, archiveFormat, basePath: finalBasePath, + branch, cache, destFolder, extension, diff --git a/packages/zip-it-and-ship-it/src/runtimes/node/utils/entry_file.ts b/packages/zip-it-and-ship-it/src/runtimes/node/utils/entry_file.ts index 5e1cf7ab17..837cbe8def 100644 --- a/packages/zip-it-and-ship-it/src/runtimes/node/utils/entry_file.ts +++ b/packages/zip-it-and-ship-it/src/runtimes/node/utils/entry_file.ts @@ -18,6 +18,7 @@ import { normalizeFilePath } from './normalize_path.js' export const ENTRY_FILE_NAME = '___netlify-entry-point' export const BOOTSTRAP_FILE_NAME = '___netlify-bootstrap.mjs' export const BOOTSTRAP_VERSION_FILE_NAME = '___netlify-bootstrap-version' +export const METADATA_FILE_NAME = '___netlify-metadata.json' export const TELEMETRY_FILE_NAME = '___netlify-telemetry.mjs' const require = createRequire(import.meta.url) diff --git a/packages/zip-it-and-ship-it/src/runtimes/node/utils/metadata_file.ts b/packages/zip-it-and-ship-it/src/runtimes/node/utils/metadata_file.ts new file mode 100644 index 0000000000..ad4edf327c --- /dev/null +++ b/packages/zip-it-and-ship-it/src/runtimes/node/utils/metadata_file.ts @@ -0,0 +1,11 @@ +export interface MetadataFile { + bootstrap_version?: string + branch?: string + version: number +} + +export const getMetadataFile = (bootstrapVersion?: string, branch?: string): MetadataFile => ({ + bootstrap_version: bootstrapVersion, + branch, + version: 1, +}) diff --git a/packages/zip-it-and-ship-it/src/runtimes/node/utils/zip.ts b/packages/zip-it-and-ship-it/src/runtimes/node/utils/zip.ts index 657e0583e5..5712460712 100644 --- a/packages/zip-it-and-ship-it/src/runtimes/node/utils/zip.ts +++ b/packages/zip-it-and-ship-it/src/runtimes/node/utils/zip.ts @@ -23,13 +23,14 @@ import { cachedLstat, mkdirAndWriteFile } from '../../../utils/fs.js' import { BOOTSTRAP_FILE_NAME, - BOOTSTRAP_VERSION_FILE_NAME, + METADATA_FILE_NAME, conflictsWithEntryFile, EntryFile, getEntryFile, getTelemetryFile, isNamedLikeEntryFile, } from './entry_file.js' +import { getMetadataFile } from './metadata_file.js' import { ModuleFormat } from './module_format.js' import { normalizeFilePath } from './normalize_path.js' import { getPackageJsonIfAvailable } from './package_json.js' @@ -44,6 +45,7 @@ const DEFAULT_USER_SUBDIRECTORY = 'src' interface ZipNodeParameters { aliases?: Map basePath: string + branch?: string cache: RuntimeCache destFolder: string extension: string @@ -186,6 +188,7 @@ const createDirectory = async function ({ const createZipArchive = async function ({ aliases = new Map(), basePath, + branch, cache, destFolder, extension, @@ -251,12 +254,11 @@ const createZipArchive = async function ({ if (runtimeAPIVersion === 2) { const bootstrapPath = addBootstrapFile(srcFiles, aliases) - if (featureFlags.zisi_add_version_file === true) { + if (featureFlags.zisi_add_metadata_file === true) { const { version: bootstrapVersion } = await getPackageJsonIfAvailable(bootstrapPath) + const payload = JSON.stringify(getMetadataFile(bootstrapVersion, branch)) - if (bootstrapVersion) { - addZipContent(archive, bootstrapVersion, BOOTSTRAP_VERSION_FILE_NAME) - } + addZipContent(archive, payload, METADATA_FILE_NAME) } } diff --git a/packages/zip-it-and-ship-it/src/runtimes/runtime.ts b/packages/zip-it-and-ship-it/src/runtimes/runtime.ts index e527c97adc..2361eb5b55 100644 --- a/packages/zip-it-and-ship-it/src/runtimes/runtime.ts +++ b/packages/zip-it-and-ship-it/src/runtimes/runtime.ts @@ -66,6 +66,7 @@ export type ZipFunction = ( args: { archiveFormat: ArchiveFormat basePath?: string + branch?: string cache: RuntimeCache config: FunctionConfig destFolder: string diff --git a/packages/zip-it-and-ship-it/src/zip.ts b/packages/zip-it-and-ship-it/src/zip.ts index c805e2a934..6abd52302c 100644 --- a/packages/zip-it-and-ship-it/src/zip.ts +++ b/packages/zip-it-and-ship-it/src/zip.ts @@ -21,6 +21,7 @@ import { nonNullable } from './utils/non_nullable.js' export interface ZipFunctionOptions { archiveFormat?: ArchiveFormat basePath?: string + branch?: string config?: Config featureFlags?: FeatureFlags repositoryRoot?: string @@ -53,6 +54,7 @@ export const zipFunctions = async function ( { archiveFormat = ARCHIVE_FORMAT.ZIP, basePath, + branch, config = {}, configFileDirectories, featureFlags: inputFeatureFlags, @@ -94,6 +96,7 @@ export const zipFunctions = async function ( const zipResult = await func.runtime.zipFunction({ archiveFormat, basePath, + branch, cache, config: func.config, destFolder, diff --git a/packages/zip-it-and-ship-it/tests/v2api.test.ts b/packages/zip-it-and-ship-it/tests/v2api.test.ts index 1cddf65d41..ff20297f21 100644 --- a/packages/zip-it-and-ship-it/tests/v2api.test.ts +++ b/packages/zip-it-and-ship-it/tests/v2api.test.ts @@ -709,22 +709,48 @@ describe.runIf(semver.gte(nodeVersion, '18.13.0'))('V2 functions API', () => { expect(files[0].runtimeVersion).toBe('nodejs20.x') }) - test('Adds a file with the bootstrap version to the ZIP archive', async () => { - const fixtureName = 'v2-api' - const { files } = await zipFixture(fixtureName, { - fixtureDir: FIXTURES_ESM_DIR, - opts: { - featureFlags: { - zisi_add_version_file: true, + describe('Adds a file with metadata', () => { + test('Without a branch', async () => { + const fixtureName = 'v2-api' + const { files } = await zipFixture(fixtureName, { + fixtureDir: FIXTURES_ESM_DIR, + opts: { + featureFlags: { + zisi_add_metadata_file: true, + }, }, - }, + }) + const [unzippedFunction] = await unzipFiles(files) + const bootstrapPath = getBootstrapPath() + const bootstrapPackageJson = await readFile(resolve(bootstrapPath, '..', '..', 'package.json'), 'utf8') + const { version: bootstrapVersion } = JSON.parse(bootstrapPackageJson) + const versionFileContents = await readFile(join(unzippedFunction.unzipPath, '___netlify-metadata.json'), 'utf8') + + expect(JSON.parse(versionFileContents)).toEqual({ bootstrap_version: bootstrapVersion, version: 1 }) }) - const [unzippedFunction] = await unzipFiles(files) - const bootstrapPath = getBootstrapPath() - const bootstrapPackageJson = await readFile(resolve(bootstrapPath, '..', '..', 'package.json'), 'utf8') - const { version: bootstrapVersion } = JSON.parse(bootstrapPackageJson) - const versionFileContents = await readFile(join(unzippedFunction.unzipPath, '___netlify-bootstrap-version'), 'utf8') - expect(versionFileContents).toBe(bootstrapVersion) + test('With a branch', async () => { + const fixtureName = 'v2-api' + const { files } = await zipFixture(fixtureName, { + fixtureDir: FIXTURES_ESM_DIR, + opts: { + branch: 'main', + featureFlags: { + zisi_add_metadata_file: true, + }, + }, + }) + const [unzippedFunction] = await unzipFiles(files) + const bootstrapPath = getBootstrapPath() + const bootstrapPackageJson = await readFile(resolve(bootstrapPath, '..', '..', 'package.json'), 'utf8') + const { version: bootstrapVersion } = JSON.parse(bootstrapPackageJson) + const versionFileContents = await readFile(join(unzippedFunction.unzipPath, '___netlify-metadata.json'), 'utf8') + + expect(JSON.parse(versionFileContents)).toEqual({ + bootstrap_version: bootstrapVersion, + branch: 'main', + version: 1, + }) + }) }) }) From 49fcc669cb39aa5b2f9b7f07aa9e7b253246178a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Wed, 9 Oct 2024 16:01:57 +0100 Subject: [PATCH 2/8] refactor: use exported types --- packages/build/src/plugins_core/functions/index.ts | 6 +++--- packages/zip-it-and-ship-it/src/main.ts | 9 ++++++++- packages/zip-it-and-ship-it/src/zip.ts | 4 ++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/build/src/plugins_core/functions/index.ts b/packages/build/src/plugins_core/functions/index.ts index 29beaec90f..7cbb1901c5 100644 --- a/packages/build/src/plugins_core/functions/index.ts +++ b/packages/build/src/plugins_core/functions/index.ts @@ -1,6 +1,6 @@ import { resolve } from 'path' -import { NodeBundlerName, RUNTIME, zipFunctions } from '@netlify/zip-it-and-ship-it' +import { NodeBundlerName, RUNTIME, zipFunctions, ZippedFunctions } from '@netlify/zip-it-and-ship-it' import { pathExists } from 'path-exists' import { addErrorInfo } from '../../error/info.js' @@ -13,7 +13,7 @@ import { getUserAndInternalFunctions, validateFunctionsSrc } from './utils.js' import { getZisiParameters } from './zisi.js' // Get a list of all unique bundlers in this run -const getBundlers = (results: Awaited> = []) => +const getBundlers = (results: ZippedFunctions = []) => // using a Set to filter duplicates new Set( results @@ -38,7 +38,7 @@ const eventTriggeredFunctions = new Set([ 'identity-login', ]) -const validateCustomRoutes = function (functions: Awaited>) { +const validateCustomRoutes = function (functions: ZippedFunctions) { for (const { routes, name, schedule } of functions) { if (!routes || routes.length === 0) continue diff --git a/packages/zip-it-and-ship-it/src/main.ts b/packages/zip-it-and-ship-it/src/main.ts index 9a81132629..c6d8d0c99a 100644 --- a/packages/zip-it-and-ship-it/src/main.ts +++ b/packages/zip-it-and-ship-it/src/main.ts @@ -11,7 +11,14 @@ import { RuntimeCache } from './utils/cache.js' import { listFunctionsDirectories, resolveFunctionsDirectories } from './utils/fs.js' export { Config, FunctionConfig } from './config.js' -export { zipFunction, zipFunctions, ZipFunctionOptions, ZipFunctionsOptions } from './zip.js' +export { + zipFunction, + zipFunctions, + ZipFunctionOptions, + ZipFunctionsOptions, + ZippedFunction, + ZippedFunctions, +} from './zip.js' export { ArchiveFormat, ARCHIVE_FORMAT } from './archive.js' export type { TrafficRules } from './rate_limit.js' diff --git a/packages/zip-it-and-ship-it/src/zip.ts b/packages/zip-it-and-ship-it/src/zip.ts index 6abd52302c..183783fb65 100644 --- a/packages/zip-it-and-ship-it/src/zip.ts +++ b/packages/zip-it-and-ship-it/src/zip.ts @@ -135,6 +135,8 @@ export const zipFunctions = async function ( return formattedResults } +export type ZippedFunctions = Awaited> + export const zipFunction = async function ( relativeSrcPath: string, destFolder: string, @@ -204,3 +206,5 @@ export const zipFunction = async function ( return formatZipResult({ ...zipResult, mainFile, name, runtime }) } + +export type ZippedFunction = Awaited> From 78066cddde9c85af2dad67a17d616f36394e9f83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Wed, 9 Oct 2024 16:37:25 +0100 Subject: [PATCH 3/8] refactor: use `FunctionResult` type --- packages/build/src/plugins_core/functions/index.ts | 8 ++++---- packages/zip-it-and-ship-it/src/main.ts | 9 +-------- packages/zip-it-and-ship-it/src/zip.ts | 10 +++------- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/packages/build/src/plugins_core/functions/index.ts b/packages/build/src/plugins_core/functions/index.ts index 7cbb1901c5..8f853e095e 100644 --- a/packages/build/src/plugins_core/functions/index.ts +++ b/packages/build/src/plugins_core/functions/index.ts @@ -1,6 +1,6 @@ import { resolve } from 'path' -import { NodeBundlerName, RUNTIME, zipFunctions, ZippedFunctions } from '@netlify/zip-it-and-ship-it' +import { NodeBundlerName, RUNTIME, zipFunctions, FunctionResult } from '@netlify/zip-it-and-ship-it' import { pathExists } from 'path-exists' import { addErrorInfo } from '../../error/info.js' @@ -13,7 +13,7 @@ import { getUserAndInternalFunctions, validateFunctionsSrc } from './utils.js' import { getZisiParameters } from './zisi.js' // Get a list of all unique bundlers in this run -const getBundlers = (results: ZippedFunctions = []) => +const getBundlers = (results: FunctionResult[] = []) => // using a Set to filter duplicates new Set( results @@ -38,7 +38,7 @@ const eventTriggeredFunctions = new Set([ 'identity-login', ]) -const validateCustomRoutes = function (functions: ZippedFunctions) { +const validateCustomRoutes = function (functions: FunctionResult[]) { for (const { routes, name, schedule } of functions) { if (!routes || routes.length === 0) continue @@ -237,7 +237,7 @@ export const bundleFunctions = { // `zip-it-and-ship-it` methods. Therefore, we need to use an intermediary // function and export them so tests can use it. export const zipItAndShipIt = { - async zipFunctions(...args: Parameters) { + async zipFunctions(...args: Parameters): Promise { return await zipFunctions(...args) }, } diff --git a/packages/zip-it-and-ship-it/src/main.ts b/packages/zip-it-and-ship-it/src/main.ts index c6d8d0c99a..9a81132629 100644 --- a/packages/zip-it-and-ship-it/src/main.ts +++ b/packages/zip-it-and-ship-it/src/main.ts @@ -11,14 +11,7 @@ import { RuntimeCache } from './utils/cache.js' import { listFunctionsDirectories, resolveFunctionsDirectories } from './utils/fs.js' export { Config, FunctionConfig } from './config.js' -export { - zipFunction, - zipFunctions, - ZipFunctionOptions, - ZipFunctionsOptions, - ZippedFunction, - ZippedFunctions, -} from './zip.js' +export { zipFunction, zipFunctions, ZipFunctionOptions, ZipFunctionsOptions } from './zip.js' export { ArchiveFormat, ARCHIVE_FORMAT } from './archive.js' export type { TrafficRules } from './rate_limit.js' diff --git a/packages/zip-it-and-ship-it/src/zip.ts b/packages/zip-it-and-ship-it/src/zip.ts index 183783fb65..995ccaa2e0 100644 --- a/packages/zip-it-and-ship-it/src/zip.ts +++ b/packages/zip-it-and-ship-it/src/zip.ts @@ -13,7 +13,7 @@ import { getFunctionsFromPaths } from './runtimes/index.js' import { MODULE_FORMAT } from './runtimes/node/utils/module_format.js' import { addArchiveSize } from './utils/archive_size.js' import { RuntimeCache } from './utils/cache.js' -import { formatZipResult } from './utils/format_result.js' +import { formatZipResult, FunctionResult } from './utils/format_result.js' import { listFunctionsDirectories, resolveFunctionsDirectories } from './utils/fs.js' import { getLogger, LogFunction } from './utils/logger.js' import { nonNullable } from './utils/non_nullable.js' @@ -65,7 +65,7 @@ export const zipFunctions = async function ( debug, internalSrcFolder, }: ZipFunctionsOptions = {}, -) { +): Promise { validateArchiveFormat(archiveFormat) const logger = getLogger(systemLog, debug) @@ -135,8 +135,6 @@ export const zipFunctions = async function ( return formattedResults } -export type ZippedFunctions = Awaited> - export const zipFunction = async function ( relativeSrcPath: string, destFolder: string, @@ -150,7 +148,7 @@ export const zipFunction = async function ( debug, internalSrcFolder, }: ZipFunctionOptions = {}, -) { +): Promise { validateArchiveFormat(archiveFormat) const logger = getLogger(systemLog, debug) @@ -206,5 +204,3 @@ export const zipFunction = async function ( return formatZipResult({ ...zipResult, mainFile, name, runtime }) } - -export type ZippedFunction = Awaited> From d4af2d2695d9dc556369b557136b9a1d105b5d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Wed, 9 Oct 2024 16:58:19 +0100 Subject: [PATCH 4/8] chore: add JSDoc --- packages/build/src/log/messages/core_steps.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/build/src/log/messages/core_steps.js b/packages/build/src/log/messages/core_steps.js index 6fce440850..e5587e288a 100644 --- a/packages/build/src/log/messages/core_steps.js +++ b/packages/build/src/log/messages/core_steps.js @@ -15,6 +15,13 @@ const logBundleResultFunctions = ({ functions, headerMessage, logs, error }) => logArray(logs, functionNames) } +/** + * Logs the result of bundling functions + * + * @param {object} options + * @param {any} options.logs + * @param {import("@netlify/zip-it-and-ship-it").FunctionResult[]} options.results + */ export const logBundleResults = ({ logs, results = [] }) => { const resultsWithErrors = results.filter(({ bundlerErrors }) => bundlerErrors && bundlerErrors.length !== 0) const resultsWithWarnings = results.filter( From 616511d8b0c64a8c0b2010bb29cedc5060580d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 10 Oct 2024 10:38:24 +0100 Subject: [PATCH 5/8] feat: pass branch to zip-it-and-ship-it --- .../build/src/plugins_core/functions/index.ts | 4 ++ .../build/src/plugins_core/functions/zisi.ts | 3 ++ .../fixtures/v2/netlify/functions/test.mjs | 3 ++ packages/build/tests/functions/tests.js | 40 ++++++++++++++++++- packages/testing/src/fs.ts | 14 +++++++ packages/testing/src/index.ts | 1 + packages/zip-it-and-ship-it/package.json | 1 + .../zip-it-and-ship-it/tests/helpers/main.ts | 16 ++------ 8 files changed, 67 insertions(+), 15 deletions(-) create mode 100644 packages/build/tests/functions/fixtures/v2/netlify/functions/test.mjs create mode 100644 packages/testing/src/fs.ts diff --git a/packages/build/src/plugins_core/functions/index.ts b/packages/build/src/plugins_core/functions/index.ts index 8f853e095e..a5819f8179 100644 --- a/packages/build/src/plugins_core/functions/index.ts +++ b/packages/build/src/plugins_core/functions/index.ts @@ -61,6 +61,7 @@ const validateCustomRoutes = function (functions: FunctionResult[]) { } const zipFunctionsAndLogResults = async ({ + branch, buildDir, childEnv, featureFlags, @@ -76,6 +77,7 @@ const zipFunctionsAndLogResults = async ({ systemLog, }) => { const zisiParameters = getZisiParameters({ + branch, buildDir, childEnv, featureFlags, @@ -118,6 +120,7 @@ const coreStep = async function ({ FUNCTIONS_DIST: relativeFunctionsDist, }, buildDir, + branch, packagePath, logs, netlifyConfig, @@ -166,6 +169,7 @@ const coreStep = async function ({ } const { bundlers } = await zipFunctionsAndLogResults({ + branch, buildDir, childEnv, featureFlags, diff --git a/packages/build/src/plugins_core/functions/zisi.ts b/packages/build/src/plugins_core/functions/zisi.ts index 41651e3cd3..d6204b3094 100644 --- a/packages/build/src/plugins_core/functions/zisi.ts +++ b/packages/build/src/plugins_core/functions/zisi.ts @@ -9,6 +9,7 @@ import type { FeatureFlags } from '../../core/feature_flags.js' import { getZisiFeatureFlags } from './feature_flags.js' type GetZisiParametersType = { + branch?: string buildDir: string childEnv: Record featureFlags: FeatureFlags @@ -40,6 +41,7 @@ const getLambdaNodeVersion = (childEnv: Record, userNodeVersion: } export const getZisiParameters = ({ + branch, buildDir, childEnv, featureFlags, @@ -65,6 +67,7 @@ export const getZisiParameters = ({ return { basePath: buildDir, + branch, config, manifest, featureFlags: zisiFeatureFlags, diff --git a/packages/build/tests/functions/fixtures/v2/netlify/functions/test.mjs b/packages/build/tests/functions/fixtures/v2/netlify/functions/test.mjs new file mode 100644 index 0000000000..11d0241321 --- /dev/null +++ b/packages/build/tests/functions/fixtures/v2/netlify/functions/test.mjs @@ -0,0 +1,3 @@ +export default async () => new Response("Hello") + +export const config = { path: "/hello" } diff --git a/packages/build/tests/functions/tests.js b/packages/build/tests/functions/tests.js index 3d7d505bf9..a23893c647 100644 --- a/packages/build/tests/functions/tests.js +++ b/packages/build/tests/functions/tests.js @@ -1,9 +1,9 @@ import { readdir, readFile, rm, stat, writeFile } from 'fs/promises' -import { resolve } from 'path' +import { join, resolve } from 'path' import { version as nodeVersion } from 'process' import { fileURLToPath } from 'url' -import { Fixture, normalizeOutput, removeDir, getTempName } from '@netlify/testing' +import { Fixture, normalizeOutput, removeDir, getTempName, unzipFile } from '@netlify/testing' import test from 'ava' import { pathExists } from 'path-exists' import semver from 'semver' @@ -204,3 +204,39 @@ if (semver.gte(nodeVersion, '16.9.0')) { t.true(app2FunctionsDist.includes('worker.zip')) }) } + +test('Functions: creates metadata file', async (t) => { + const fixture = await new Fixture('./fixtures/v2').withCopyRoot({ git: false }) + const build = await fixture + .withFlags({ + branch: 'my-branch', + cwd: fixture.repositoryRoot, + featureFlags: { zisi_add_metadata_file: true }, + }) + .runWithBuildAndIntrospect() + + t.true(build.success) + + const functionsDistPath = resolve(fixture.repositoryRoot, '.netlify/functions') + const functionsDistFiles = await readdir(functionsDistPath) + + t.true(functionsDistFiles.includes('manifest.json')) + t.true(functionsDistFiles.includes('test.zip')) + + const unzipPath = join(functionsDistPath, `.netlify-test-${Date.now()}`) + + await unzipFile(join(functionsDistPath, 'test.zip'), unzipPath) + + const functionFiles = await readdir(unzipPath) + + t.true(functionFiles.includes('___netlify-bootstrap.mjs')) + t.true(functionFiles.includes('___netlify-entry-point.mjs')) + t.true(functionFiles.includes('___netlify-metadata.json')) + t.true(functionFiles.includes('test.mjs')) + + const metadata = JSON.parse(await readFile(join(unzipPath, '___netlify-metadata.json'), 'utf8')) + + t.is(semver.valid(metadata.bootstrap_version), metadata.bootstrap_version) + t.is(metadata.branch, 'my-branch') + t.is(metadata.version, 1) +}) diff --git a/packages/testing/src/fs.ts b/packages/testing/src/fs.ts new file mode 100644 index 0000000000..93d06f1838 --- /dev/null +++ b/packages/testing/src/fs.ts @@ -0,0 +1,14 @@ +import { mkdir } from 'fs/promises' +import { platform } from 'process' + +import { execa } from 'execa' + +export const unzipFile = async function (path: string, dest: string): Promise { + await mkdir(dest, { recursive: true }) + + if (platform === 'win32') { + await execa('tar', ['-xf', path, '-C', dest]) + } else { + await execa('unzip', ['-o', path, '-d', dest]) + } +} diff --git a/packages/testing/src/index.ts b/packages/testing/src/index.ts index ebb6511823..4b25185540 100644 --- a/packages/testing/src/index.ts +++ b/packages/testing/src/index.ts @@ -1,5 +1,6 @@ export * from './dir.js' export * from './fixture.js' +export * from './fs.js' export * from './normalize.js' export * from './server.js' export * from './tcp_server.js' diff --git a/packages/zip-it-and-ship-it/package.json b/packages/zip-it-and-ship-it/package.json index bbca2ec5df..e0ab1e0564 100644 --- a/packages/zip-it-and-ship-it/package.json +++ b/packages/zip-it-and-ship-it/package.json @@ -77,6 +77,7 @@ "zod": "^3.23.8" }, "devDependencies": { + "@netlify/testing": "*", "@types/archiver": "5.3.4", "@types/glob": "8.1.0", "@types/is-ci": "3.0.4", diff --git a/packages/zip-it-and-ship-it/tests/helpers/main.ts b/packages/zip-it-and-ship-it/tests/helpers/main.ts index 77a7042aec..5ade39fe60 100644 --- a/packages/zip-it-and-ship-it/tests/helpers/main.ts +++ b/packages/zip-it-and-ship-it/tests/helpers/main.ts @@ -1,9 +1,9 @@ -import { mkdir, rm } from 'fs/promises' +import { rm } from 'fs/promises' import { dirname, resolve, join, basename, relative } from 'path' -import { env, platform } from 'process' +import { env } from 'process' import { fileURLToPath } from 'url' -import { execa } from 'execa' +import { unzipFile } from '@netlify/testing' import isCI from 'is-ci' import { dir as getTmpDir } from 'tmp-promise' import { afterAll, expect } from 'vitest' @@ -138,16 +138,6 @@ export const unzipFiles = async function (files: FunctionResult[]): Promise { - await mkdir(dest, { recursive: true }) - - if (platform === 'win32') { - await execa('tar', ['-xf', path, '-C', dest]) - } else { - await execa('unzip', ['-o', path, '-d', dest]) - } -} - const replaceUnzipPath = function ({ path }: { path: string }): string { return join(path.replace(/.zip$/, ''), basename(path).replace(/.zip$/, '.js')) } From 1cdda62791fa4a1ec9d3e5510adee5ca414e06f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 10 Oct 2024 10:42:59 +0100 Subject: [PATCH 6/8] chore: remove dep --- packages/zip-it-and-ship-it/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/zip-it-and-ship-it/package.json b/packages/zip-it-and-ship-it/package.json index e0ab1e0564..bbca2ec5df 100644 --- a/packages/zip-it-and-ship-it/package.json +++ b/packages/zip-it-and-ship-it/package.json @@ -77,7 +77,6 @@ "zod": "^3.23.8" }, "devDependencies": { - "@netlify/testing": "*", "@types/archiver": "5.3.4", "@types/glob": "8.1.0", "@types/is-ci": "3.0.4", From ad595f8413b453646c1a56838ee311ee90738c12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 10 Oct 2024 10:55:21 +0100 Subject: [PATCH 7/8] chore: add util back --- .../zip-it-and-ship-it/tests/helpers/main.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/zip-it-and-ship-it/tests/helpers/main.ts b/packages/zip-it-and-ship-it/tests/helpers/main.ts index 5ade39fe60..77a7042aec 100644 --- a/packages/zip-it-and-ship-it/tests/helpers/main.ts +++ b/packages/zip-it-and-ship-it/tests/helpers/main.ts @@ -1,9 +1,9 @@ -import { rm } from 'fs/promises' +import { mkdir, rm } from 'fs/promises' import { dirname, resolve, join, basename, relative } from 'path' -import { env } from 'process' +import { env, platform } from 'process' import { fileURLToPath } from 'url' -import { unzipFile } from '@netlify/testing' +import { execa } from 'execa' import isCI from 'is-ci' import { dir as getTmpDir } from 'tmp-promise' import { afterAll, expect } from 'vitest' @@ -138,6 +138,16 @@ export const unzipFiles = async function (files: FunctionResult[]): Promise { + await mkdir(dest, { recursive: true }) + + if (platform === 'win32') { + await execa('tar', ['-xf', path, '-C', dest]) + } else { + await execa('unzip', ['-o', path, '-d', dest]) + } +} + const replaceUnzipPath = function ({ path }: { path: string }): string { return join(path.replace(/.zip$/, ''), basename(path).replace(/.zip$/, '.js')) } From 2ed85d41454387dcd91470aac5e07c1648ca1d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 10 Apr 2025 09:53:02 +0100 Subject: [PATCH 8/8] feat: add API cache --- .../src/plugins_core/pre_dev_cleanup/index.ts | 5 +- packages/config/src/api/client.js | 9 +- packages/config/src/api/site_info.ts | 47 ++--- packages/config/src/main.ts | 3 +- packages/js-client/src/buffered_response.ts | 33 +++ packages/js-client/src/cache.test.ts | 198 ++++++++++++++++++ packages/js-client/src/cache.ts | 155 ++++++++++++++ packages/js-client/src/headers.ts | 14 ++ packages/js-client/src/index.ts | 8 + packages/js-client/src/methods/index.js | 25 ++- .../src/methods/{response.js => response.ts} | 22 +- .../js-client/src/methods/response_type.ts | 11 + 12 files changed, 478 insertions(+), 52 deletions(-) create mode 100644 packages/js-client/src/buffered_response.ts create mode 100644 packages/js-client/src/cache.test.ts create mode 100644 packages/js-client/src/cache.ts create mode 100644 packages/js-client/src/headers.ts rename packages/js-client/src/methods/{response.js => response.ts} (74%) create mode 100644 packages/js-client/src/methods/response_type.ts diff --git a/packages/build/src/plugins_core/pre_dev_cleanup/index.ts b/packages/build/src/plugins_core/pre_dev_cleanup/index.ts index c4a41a9c1c..c90e9b434c 100644 --- a/packages/build/src/plugins_core/pre_dev_cleanup/index.ts +++ b/packages/build/src/plugins_core/pre_dev_cleanup/index.ts @@ -3,14 +3,13 @@ import { resolve } from 'node:path' import { listFrameworks } from '@netlify/framework-info' -import { log } from '../../log/logger.js' import { CoreStep, CoreStepCondition, CoreStepFunction, CoreStepFunctionArgs } from '../types.js' const dirExists = async (path: string): Promise => { try { await stat(path) return true - } catch (error) { + } catch { return false } } @@ -37,7 +36,7 @@ const coreStep: CoreStepFunction = async (input) => { for (const dir of dirs) { await rm(resolve(input.buildDir, dir), { recursive: true, force: true }) } - log(input.logs, `Cleaned up ${dirs.join(', ')}.`) + input.systemLog(`Cleaned up ${dirs.join(', ')}.`) return {} } diff --git a/packages/config/src/api/client.js b/packages/config/src/api/client.js index d5d1fe4926..e19e4ff945 100644 --- a/packages/config/src/api/client.js +++ b/packages/config/src/api/client.js @@ -3,13 +3,18 @@ import { NetlifyAPI } from 'netlify' import { removeUndefined } from '../utils/remove_falsy.js' // Retrieve Netlify API client, if an access token was passed -export const getApiClient = function ({ token, offline, testOpts = {}, host, scheme, pathPrefix }) { +export const getApiClient = function ({ cache, token, offline, testOpts = {}, host, scheme, pathPrefix }) { if (!token || offline) { return } // TODO: find less intrusive way to mock HTTP requests - const parameters = removeUndefined({ scheme: testOpts.scheme || scheme, host: testOpts.host || host, pathPrefix }) + const parameters = removeUndefined({ + cache, + scheme: testOpts.scheme || scheme, + host: testOpts.host || host, + pathPrefix, + }) const api = new NetlifyAPI(token, parameters) return api } diff --git a/packages/config/src/api/site_info.ts b/packages/config/src/api/site_info.ts index 699219768d..600c8e5779 100644 --- a/packages/config/src/api/site_info.ts +++ b/packages/config/src/api/site_info.ts @@ -39,7 +39,6 @@ export const getSiteInfo = async function ({ siteFeatureFlagPrefix, }: GetSiteInfoOpts) { const { env: testEnv = false } = testOpts - const useV2Endpoint = !!accountId && featureFlags.cli_integration_installations_meta if (useV2Endpoint) { @@ -51,7 +50,7 @@ export const getSiteInfo = async function ({ const integrations = mode === 'buildbot' && !offline - ? await getIntegrations({ siteId, testOpts, offline, useV2Endpoint, accountId }) + ? await getIntegrations({ api, siteId, testOpts, offline, useV2Endpoint, accountId }) : [] return { siteInfo, accounts: [], addons: [], integrations } @@ -60,11 +59,10 @@ export const getSiteInfo = async function ({ const promises = [ getSite(api, siteId, siteFeatureFlagPrefix), getAccounts(api), - getAddons(api, siteId), - getIntegrations({ siteId, testOpts, offline, useV2Endpoint, accountId }), + getIntegrations({ api, siteId, testOpts, offline, useV2Endpoint, accountId }), ] - const [siteInfo, accounts, addons, integrations] = await Promise.all(promises) + const [siteInfo, accounts, integrations] = await Promise.all(promises) if (siteInfo.use_envelope) { const envelope = await getEnvelope({ api, accountId: siteInfo.account_slug, siteId, context }) @@ -72,7 +70,7 @@ export const getSiteInfo = async function ({ siteInfo.build_settings.env = envelope } - return { siteInfo, accounts, addons, integrations } + return { siteInfo, accounts, addons: [], integrations } } if (api === undefined || mode === 'buildbot' || testEnv) { @@ -81,7 +79,8 @@ export const getSiteInfo = async function ({ if (siteId !== undefined) siteInfo.id = siteId if (accountId !== undefined) siteInfo.account_id = accountId - const integrations = mode === 'buildbot' && !offline ? await getIntegrations({ siteId, testOpts, offline }) : [] + const integrations = + mode === 'buildbot' && !offline ? await getIntegrations({ api, siteId, testOpts, offline }) : [] return { siteInfo, accounts: [], addons: [], integrations } } @@ -89,11 +88,10 @@ export const getSiteInfo = async function ({ const promises = [ getSite(api, siteId, siteFeatureFlagPrefix), getAccounts(api), - getAddons(api, siteId), - getIntegrations({ siteId, testOpts, offline }), + getIntegrations({ api, siteId, testOpts, offline }), ] - const [siteInfo, accounts, addons, integrations] = await Promise.all(promises) + const [siteInfo, accounts, integrations] = await Promise.all(promises) if (siteInfo.use_envelope) { const envelope = await getEnvelope({ api, accountId: siteInfo.account_slug, siteId, context }) @@ -101,7 +99,7 @@ export const getSiteInfo = async function ({ siteInfo.build_settings.env = envelope } - return { siteInfo, accounts, addons, integrations } + return { siteInfo, accounts, addons: [], integrations } } const getSite = async function (api: NetlifyAPI, siteId: string, siteFeatureFlagPrefix: string) { @@ -126,20 +124,8 @@ const getAccounts = async function (api: NetlifyAPI) { } } -const getAddons = async function (api: NetlifyAPI, siteId: string) { - if (siteId === undefined) { - return [] - } - - try { - const addons = await (api as any).listServiceInstancesForSite({ siteId }) - return Array.isArray(addons) ? addons : [] - } catch (error) { - throwUserError(`Failed retrieving addons for site ${siteId}: ${error.message}. ${ERROR_CALL_TO_ACTION}`) - } -} - type GetIntegrationsOpts = { + api?: NetlifyAPI siteId?: string accountId?: string testOpts: TestOptions @@ -148,6 +134,7 @@ type GetIntegrationsOpts = { } const getIntegrations = async function ({ + api, siteId, accountId, testOpts, @@ -168,11 +155,21 @@ const getIntegrations = async function ({ : `${baseUrl}site/${siteId}/integrations/safe` try { + // Even though integrations don't come through the Netlify API, we can + // still leverage the API cache if one is being used. + if (api?.cache) { + const response = await api.cache.get(url, 'get', {}) + + if (response !== null && Array.isArray(response.body)) { + return response.body + } + } + const response = await fetch(url) const integrations = await response.json() return Array.isArray(integrations) ? integrations : [] - } catch (error) { + } catch { // Integrations should not block the build if they fail to load // TODO: We should consider blocking the build as integrations are a critical part of the build process // https://linear.app/netlify/issue/CT-1214/implement-strategy-in-builds-to-deal-with-integrations-that-we-fail-to diff --git a/packages/config/src/main.ts b/packages/config/src/main.ts index a42dd9e7a8..e6a5e9b064 100644 --- a/packages/config/src/main.ts +++ b/packages/config/src/main.ts @@ -26,6 +26,7 @@ import { getRedirectsPath, addRedirects } from './redirects.js' */ export const resolveConfig = async function (opts) { const { + apiCache, cachedConfig, cachedConfigPath, host, @@ -39,7 +40,7 @@ export const resolveConfig = async function (opts) { ...optsA } = addDefaultOpts(opts) as $TSFixMe // `api` is not JSON-serializable, so we cannot cache it inside `cachedConfig` - const api = getApiClient({ token, offline, host, scheme, pathPrefix, testOpts }) + const api = getApiClient({ token, offline, host, scheme, pathPrefix, testOpts, cache: apiCache }) const parsedCachedConfig = await getCachedConfig({ cachedConfig, cachedConfigPath, token, api }) // If there is a cached config, use it. The exception is when a default config, diff --git a/packages/js-client/src/buffered_response.ts b/packages/js-client/src/buffered_response.ts new file mode 100644 index 0000000000..78b20cfd9f --- /dev/null +++ b/packages/js-client/src/buffered_response.ts @@ -0,0 +1,33 @@ +import { getHeadersObject, HeadersObject } from './headers.js' +import { getResponseType } from './methods/response_type.js' + +type JSONResponse = { type: 'json'; body: any } +type TextResponse = { type: 'text'; body: string } + +/** + * An HTTP response that has been fully read. The body has been buffered and so + * it can be read multiple times and serialized to disk. + */ +export type BufferedResponse = { headers: HeadersObject; timestamp: number } & (JSONResponse | TextResponse) + +/** + * Consumes an HTTP response and returns a `BufferedResponse` object. + */ +export const getBufferedResponse = async (res: Response): Promise => { + const headers = getHeadersObject(res.headers) + const data = { + headers, + timestamp: Date.now(), + } + const type = getResponseType(res) + + if (type === 'json') { + return { + ...data, + type: 'json', + body: await res.json(), + } + } + + return { ...data, type: 'text', body: await res.text() } +} diff --git a/packages/js-client/src/cache.test.ts b/packages/js-client/src/cache.test.ts new file mode 100644 index 0000000000..21ad08ffdd --- /dev/null +++ b/packages/js-client/src/cache.test.ts @@ -0,0 +1,198 @@ +import { promises as fs } from 'fs' +import path from 'path' + +import test, { ExecutionContext } from 'ava' + +import { APICache } from '../lib/index.js' + +const dateNow = Date.now +const globalFetch = globalThis.fetch + +test.afterEach(() => { + Date.now = dateNow + globalThis.fetch = globalFetch +}) + +const getMockFetch = (t: ExecutionContext, mocks: Record Response>) => { + const calls: Record = {} + + const mockFetch = async (input: URL | RequestInfo) => { + for (const url in mocks) { + if (input.toString() === url) { + calls[url] = calls[url] ?? 0 + calls[url]++ + + return mocks[url]() + } + } + + t.fail(`Unexpected fetch call: ${input}`) + + return new Response(null, { status: 400 }) + } + + return { calls, mockFetch } +} + +const sleep = (ms: number) => + new Promise((resolve) => { + setTimeout(resolve, ms) + }) + +test.serial('Returns response from cache if within TTL', async (t) => { + const mockEndpoint = 'https://api.netlify/endpoint' + const mockResponse = { messages: ['Hello', 'Goodbye'] } + const { calls, mockFetch } = getMockFetch(t, { + [mockEndpoint]: () => Response.json(mockResponse), + }) + + globalThis.fetch = mockFetch + + const cache = new APICache({ + ttl: 30, + swr: 30, + }) + + const res1 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res1?.body, mockResponse) + t.deepEqual(res1?.headers, { 'content-type': 'application/json' }) + + const now = Date.now() + + const future = now + 10 + Date.now = () => future + + const res2 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res2?.body, mockResponse) + t.deepEqual(res2?.headers, { 'content-type': 'application/json' }) + + t.is(calls[mockEndpoint], 1) +}) + +test.serial('Returns response from cache if outside of TTL but within SWR', async (t) => { + const mockEndpoint = 'https://api.netlify/endpoint' + const mockResponse = { messages: ['Hello', 'Goodbye'] } + const { calls, mockFetch } = getMockFetch(t, { + [mockEndpoint]: () => Response.json(mockResponse), + }) + + globalThis.fetch = mockFetch + + const cache = new APICache({ + ttl: 30, + swr: 60, + }) + + const res1 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res1?.body, mockResponse) + t.deepEqual(res1?.headers, { 'content-type': 'application/json' }) + + const now = Date.now() + + const future = now + 45 + Date.now = () => future + + const res2 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res2?.body, mockResponse) + t.deepEqual(res2?.headers, { 'content-type': 'application/json' }) + + t.is(calls[mockEndpoint], 1) + + await sleep(10) + + t.is(calls[mockEndpoint], 2) + + const cacheKey = cache.getCacheKey(mockEndpoint, 'get', {}) + t.is(cache.entries[cacheKey].timestamp, future) +}) + +test.serial('Returns fresh response if outside of TTL and SWR', async (t) => { + const mockEndpoint = 'https://api.netlify/endpoint' + const mockResponse = { messages: ['Hello', 'Goodbye'] } + const { calls, mockFetch } = getMockFetch(t, { + [mockEndpoint]: () => Response.json(mockResponse), + }) + + globalThis.fetch = mockFetch + + const cache = new APICache({ + ttl: 30, + swr: 60, + }) + + const res1 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res1?.body, mockResponse) + t.deepEqual(res1?.headers, { 'content-type': 'application/json' }) + + const now = Date.now() + + const future = now + 90 + Date.now = () => future + + const res2 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res2?.body, mockResponse) + t.deepEqual(res2?.headers, { 'content-type': 'application/json' }) + + t.is(calls[mockEndpoint], 2) +}) + +test.serial('Uses disk fallback', async (t) => { + const mockEndpoint = 'https://api.netlify/endpoint' + const mockResponse = { messages: ['Hello', 'Goodbye'] } + const { calls, mockFetch } = getMockFetch(t, { + [mockEndpoint]: () => Response.json(mockResponse), + }) + + globalThis.fetch = mockFetch + + const fsPath = await fs.mkdtemp('netlify-js-client-test') + const cache = new APICache({ + fsPath, + ttl: 30, + swr: 60, + }) + + t.teardown(async () => { + await fs.rm(fsPath, { recursive: true }) + }) + + const now = Date.now() + const cacheKey = cache.getCacheKey(mockEndpoint, 'get', {}) + const filePath = path.join(fsPath, cacheKey) + const file = { + body: mockResponse, + headers: { + 'content-type': 'application/json', + }, + timestamp: now - 20, + type: 'json', + } + + await fs.writeFile(filePath, JSON.stringify(file)) + + const res1 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res1?.body, mockResponse) + t.deepEqual(res1?.headers, { 'content-type': 'application/json' }) + + t.falsy(calls[mockEndpoint]) + + const future = now + 20 + Date.now = () => future + + const res2 = await cache.get(mockEndpoint, 'get', {}) + t.deepEqual(res2?.body, mockResponse) + t.deepEqual(res2?.headers, { 'content-type': 'application/json' }) + + t.falsy(calls[mockEndpoint]) + + await sleep(10) + + t.is(calls[mockEndpoint], 1) + + const newFile = await fs.readFile(filePath, 'utf8') + const data = JSON.parse(newFile) + + t.deepEqual(data.body, mockResponse) + t.deepEqual(data.headers, { 'content-type': 'application/json' }) + t.is(data.timestamp, future) +}) diff --git a/packages/js-client/src/cache.ts b/packages/js-client/src/cache.ts new file mode 100644 index 0000000000..d51207fe50 --- /dev/null +++ b/packages/js-client/src/cache.ts @@ -0,0 +1,155 @@ +import { createHash } from 'crypto' +import { promises as fs } from 'fs' +import { join } from 'path' + +import { BufferedResponse, getBufferedResponse } from './buffered_response.js' +import { HeadersObject } from './headers.js' + +const DEFAULT_TTL = 30_000 +const DEFAULT_SWR = 120_000 + +interface APIOptions { + fsPath?: string + ttl?: number + swr?: number +} + +export class APICache { + entries: Record + fetches: Record> + fsPath?: string + fsPathSetup?: Promise + ttl: number + swr: number + + constructor({ fsPath, ttl = DEFAULT_TTL, swr = DEFAULT_SWR }: APIOptions) { + this.entries = {} + this.fetches = {} + this.fsPath = fsPath + this.fsPathSetup = fsPath ? fs.mkdir(fsPath, { recursive: true }) : Promise.resolve() + this.ttl = ttl + this.swr = swr + } + + private async addToCache(key: string, res: BufferedResponse) { + this.entries[key] = res + + await this.saveToDisk(key, res) + } + + private async getCached(key: string) { + if (this.entries[key]) { + return this.entries[key] + } + + const fromDisk = await this.loadFromDisk(key) + + if (fromDisk) { + this.entries[key] = fromDisk + + return this.entries[key] + } + } + + private fetchAndAddToCache(url: string, method: string, headers: HeadersObject) { + const key = this.getCacheKey(url, method, headers) + + if (!this.fetches[key]) { + this.fetches[key] = fetch(url, { + headers, + method, + }) + .then(async (res) => { + delete this.fetches[key] + + const bufferedRes = await getBufferedResponse(res) + + if (res.status === 200) { + this.addToCache(key, bufferedRes) + } + + return bufferedRes + }) + .catch(() => { + delete this.fetches[key] + + return null + }) + } + + return this.fetches[key] + } + + private async loadFromDisk(key: string) { + if (!this.fsPath) { + return + } + + const filePath = join(this.fsPath, key) + + try { + const file = await fs.readFile(filePath, 'utf8') + const data = JSON.parse(file) as BufferedResponse + + if (data.type !== 'json' && data.type !== 'text') { + throw new Error('Unsupported response type') + } + + return data + } catch { + // no-op + } + } + + private async saveToDisk(key: string, res: BufferedResponse) { + if (!this.fsPath) { + return + } + + const data = { + ...res, + timestamp: Date.now(), + } + + try { + await this.fsPathSetup + await fs.writeFile(join(this.fsPath, key), JSON.stringify(data)) + } catch { + // no-op + } + } + + async get(url: string, method: string, headers: HeadersObject) { + const key = this.getCacheKey(url, method, headers) + const cached = await this.getCached(key) + + if (cached) { + const currentTimestamp = Date.now() + + if (this.ttl > 0 && currentTimestamp - cached.timestamp <= this.ttl) { + return cached + } + + if (this.swr > 0 && currentTimestamp - cached.timestamp <= this.swr) { + setTimeout(() => { + this.fetchAndAddToCache(url, method, headers) + }, 0) + + return cached + } + } + + return this.fetchAndAddToCache(url, method, headers) + } + + getCacheKey(url: string, method: string, headers: HeadersObject) { + const headersInKey = { + authorization: headers.Authorization, + } + const hash = createHash('md5') + .update(JSON.stringify({ url, method, headers: headersInKey })) + .digest('hex') + + return hash + } +} diff --git a/packages/js-client/src/headers.ts b/packages/js-client/src/headers.ts new file mode 100644 index 0000000000..1b9c6547f9 --- /dev/null +++ b/packages/js-client/src/headers.ts @@ -0,0 +1,14 @@ +export type HeadersObject = Record + +/** + * Serializes a `Headers` object into a plain object. + */ +export const getHeadersObject = (headers: Headers) => { + const obj: HeadersObject = {} + + for (const [key, value] of headers.entries()) { + obj[key] = value + } + + return obj +} diff --git a/packages/js-client/src/index.ts b/packages/js-client/src/index.ts index c4a44d410e..3cb698eec1 100644 --- a/packages/js-client/src/index.ts +++ b/packages/js-client/src/index.ts @@ -1,5 +1,6 @@ import pWaitFor from 'p-wait-for' +import { APICache } from './cache.js' import { getMethods } from './methods/index.js' import { openApiSpec } from './open_api.js' import { getOperations } from './operations.js' @@ -26,11 +27,13 @@ type APIOptions = { * Global params are only sent of the OpenAPI spec specifies the provided params. */ globalParams?: Record + cache?: APICache } export class NetlifyAPI { #accessToken: string | null = null + cache?: APICache defaultHeaders: Record = { accept: 'application/json', } @@ -56,7 +59,10 @@ export class NetlifyAPI { this.accessToken = options.accessToken || accessTokenInput || null this.defaultHeaders['User-agent'] = options.userAgent || 'netlify/js-client' + this.cache = options.cache + const methods = getMethods({ + cache: this.cache, basePath: this.basePath, defaultHeaders: this.defaultHeaders, agent: this.agent, @@ -123,3 +129,5 @@ export class NetlifyAPI { } export const methods = getOperations() + +export { APICache } diff --git a/packages/js-client/src/methods/index.js b/packages/js-client/src/methods/index.js index d78df9a09a..67d03a12a1 100644 --- a/packages/js-client/src/methods/index.js +++ b/packages/js-client/src/methods/index.js @@ -10,24 +10,26 @@ import { getUrl } from './url.js' // For each OpenAPI operation, add a corresponding method. // The `operationId` is the method name. -export const getMethods = function ({ basePath, defaultHeaders, agent, globalParams }) { +export const getMethods = function ({ basePath, cache, defaultHeaders, agent, globalParams }) { const operations = getOperations() - const methods = operations.map((method) => getMethod({ method, basePath, defaultHeaders, agent, globalParams })) + const methods = operations.map((method) => + getMethod({ method, basePath, defaultHeaders, agent, globalParams, cache }), + ) return Object.assign({}, ...methods) } -const getMethod = function ({ method, basePath, defaultHeaders, agent, globalParams }) { +const getMethod = function ({ method, basePath, defaultHeaders, agent, globalParams, cache }) { return { [method.operationId](params, opts) { - return callMethod({ method, basePath, defaultHeaders, agent, globalParams, params, opts }) + return callMethod({ method, basePath, defaultHeaders, agent, globalParams, params, opts, cache }) }, } } -const callMethod = async function ({ method, basePath, defaultHeaders, agent, globalParams, params, opts }) { +const callMethod = async function ({ method, basePath, defaultHeaders, agent, globalParams, params, opts, cache }) { const requestParams = { ...globalParams, ...params } const url = getUrl(method, basePath, requestParams) - const response = await makeRequestOrRetry({ url, method, defaultHeaders, agent, requestParams, opts }) + const response = await makeRequestOrRetry({ url, method, defaultHeaders, agent, requestParams, opts, cache }) const parsedResponse = await parseResponse(response) return parsedResponse @@ -70,11 +72,11 @@ const addAgent = function (agent, opts) { return opts } -const makeRequestOrRetry = async function ({ url, method, defaultHeaders, agent, requestParams, opts }) { +const makeRequestOrRetry = async function ({ url, method, defaultHeaders, agent, requestParams, opts, cache }) { // Using a loop is simpler here for (let index = 0; index <= MAX_RETRY; index++) { const optsA = getOpts({ method, defaultHeaders, agent, requestParams, opts }) - const { response, error } = await makeRequest(url, optsA) + const { response, error } = await makeRequest(url, optsA, cache) if (shouldRetry({ response, error, method }) && index !== MAX_RETRY) { await waitForRetry(response) @@ -89,8 +91,13 @@ const makeRequestOrRetry = async function ({ url, method, defaultHeaders, agent, } } -const makeRequest = async function (url, opts) { +const makeRequest = async function (url, opts, cache) { try { + if (cache) { + const response = await cache.get(url, opts.method, opts.headers) + return { response } + } + const response = await fetch(url, opts) return { response } } catch (error) { diff --git a/packages/js-client/src/methods/response.js b/packages/js-client/src/methods/response.ts similarity index 74% rename from packages/js-client/src/methods/response.js rename to packages/js-client/src/methods/response.ts index 548a153b99..5cb9dd39f3 100644 --- a/packages/js-client/src/methods/response.js +++ b/packages/js-client/src/methods/response.ts @@ -1,8 +1,16 @@ import { JSONHTTPError, TextHTTPError } from 'micro-api-client' import omit from 'omit.js' +import { BufferedResponse } from '../buffered_response.js' + +import { getResponseType, ResponseType } from './response_type.js' + // Read and parse the HTTP response -export const parseResponse = async function (response) { +export const parseResponse = async function (response: BufferedResponse | Response) { + if (!(response instanceof Response)) { + return response.body + } + const responseType = getResponseType(response) const textResponse = await response.text() @@ -16,17 +24,7 @@ export const parseResponse = async function (response) { return parsedResponse } -const getResponseType = function ({ headers }) { - const contentType = headers.get('Content-Type') - - if (contentType != null && contentType.includes('json')) { - return 'json' - } - - return 'text' -} - -const parseJsonResponse = function (response, textResponse, responseType) { +const parseJsonResponse = function (response: Response, textResponse: string, responseType: ResponseType) { if (responseType === 'text') { return textResponse } diff --git a/packages/js-client/src/methods/response_type.ts b/packages/js-client/src/methods/response_type.ts new file mode 100644 index 0000000000..1064309b67 --- /dev/null +++ b/packages/js-client/src/methods/response_type.ts @@ -0,0 +1,11 @@ +export type ResponseType = 'json' | 'text' + +export const getResponseType = function ({ headers }): ResponseType { + const contentType = headers.get('Content-Type') + + if (contentType != null && contentType.includes('json')) { + return 'json' + } + + return 'text' +}