perf: memoize blobs requests in the request scope

Author: pieh

package-lock.json

@@ -76,6 +76,7 @@
     "fs-monkey": "^1.0.6",
     "get-port": "^7.1.0",
     "lambda-local": "^2.2.0",
+    "lru-cache": "^10.4.3",
     "memfs": "^4.9.2",
     "mock-require": "^3.0.3",
     "msw": "^2.0.7",

package.json

@@ -5,6 +5,7 @@ import { join, resolve } from 'node:path'
 import type { NextConfigComplete } from 'next/dist/server/config-shared.js'
 
 import { PLUGIN_DIR, RUN_CONFIG } from './constants.js'
+import { setInMemoryCacheMaxSizeFromNextConfig } from './regional-blob-store.cjs'
 
 /**
  * Get Next.js config from the build output
@@ -13,10 +14,27 @@ export const getRunConfig = async () => {
   return JSON.parse(await readFile(resolve(PLUGIN_DIR, RUN_CONFIG), 'utf-8'))
 }
 
+type NextConfigForMultipleVersions = NextConfigComplete & {
+  experimental: NextConfigComplete['experimental'] & {
+    // those are pre 14.1.0 options that were moved out of experimental in // https://github.com/vercel/next.js/pull/57953/files#diff-c49c4767e6ed8627e6e1b8f96b141ee13246153f5e9142e1da03450c8e81e96fL311
+
+    // https://github.com/vercel/next.js/blob/v14.0.4/packages/next/src/server/config-shared.ts#L182-L183
+    // custom path to a cache handler to use
+    incrementalCacheHandlerPath?: string
+    // https://github.com/vercel/next.js/blob/v14.0.4/packages/next/src/server/config-shared.ts#L207-L212
+    /**
+     * In-memory cache size in bytes.
+     *
+     * If `isrMemoryCacheSize: 0` disables in-memory caching.
+     */
+    isrMemoryCacheSize?: number
+  }
+}
+
 /**
  * Configure the custom cache handler at request time
  */
-export const setRunConfig = (config: NextConfigComplete) => {
+export const setRunConfig = (config: NextConfigForMultipleVersions) => {
   const cacheHandler = join(PLUGIN_DIR, '.netlify/dist/run/handlers/cache.cjs')
   if (!existsSync(cacheHandler)) {
     throw new Error(`Cache handler not found at ${cacheHandler}`)
@@ -25,15 +43,17 @@ export const setRunConfig = (config: NextConfigComplete) => {
   // set the path to the cache handler
   config.experimental = {
     ...config.experimental,
-    // @ts-expect-error incrementalCacheHandlerPath was removed from config type
-    // but we still need to set it for older Next.js versions
+    // Before Next.js 14.1.0 path to the cache handler was in experimental section, see NextConfigForMultipleVersions type
     incrementalCacheHandlerPath: cacheHandler,
   }
 
-  // Next.js 14.1.0 moved the cache handler from experimental to stable
-  // https://github.com/vercel/next.js/pull/57953/files#diff-c49c4767e6ed8627e6e1b8f96b141ee13246153f5e9142e1da03450c8e81e96fL311
+  // Next.js 14.1.0 moved the cache handler from experimental to stable, see NextConfigForMultipleVersions type
   config.cacheHandler = cacheHandler
-  config.cacheMaxMemorySize = 0
+
+  // honor the in-memory cache size from next.config (either one set by user or Next.js default)
+  setInMemoryCacheMaxSizeFromNextConfig(
+    config.cacheMaxMemorySize ?? config.experimental?.isrMemoryCacheSize,
+  )
 
   // set config
   process.env.__NEXT_PRIVATE_STANDALONE_CONFIG = JSON.stringify(config)

src/run/config.ts

@@ -30,22 +30,18 @@ import {
 import { getLogger, getRequestContext } from './request-context.cjs'
 import { getTracer, recordWarning } from './tracer.cjs'
 
-type TagManifestBlobCache = Record<string, Promise<TagManifest | null>>
-
 const purgeCacheUserAgent = `${nextRuntimePkgName}@${nextRuntimePkgVersion}`
 
 export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
   options: CacheHandlerContext
   revalidatedTags: string[]
   cacheStore: MemoizedKeyValueStoreBackedByRegionalBlobStore
   tracer = getTracer()
-  tagManifestsFetchedFromBlobStoreInCurrentRequest: TagManifestBlobCache
 
   constructor(options: CacheHandlerContext) {
     this.options = options
     this.revalidatedTags = options.revalidatedTags
     this.cacheStore = getMemoizedKeyValueStoreBackedByRegionalBlobStore({ consistency: 'strong' })
-    this.tagManifestsFetchedFromBlobStoreInCurrentRequest = {}
   }
 
   private getTTL(blob: NetlifyCacheHandlerValue) {
@@ -469,7 +465,8 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
   }
 
   resetRequestCache() {
-    this.tagManifestsFetchedFromBlobStoreInCurrentRequest = {}
+    // no-op because in-memory cache is scoped to requests and not global
+    // see getRequestSpecificInMemoryCache
   }
 
   /**
@@ -508,10 +505,9 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
     }
 
     // 2. If any in-memory tags don't indicate that any of tags was invalidated
-    //    we will check blob store, but memoize results for duration of current request
-    //    so that we only check blob store once per tag within a single request
-    //    full-route cache and fetch caches share a lot of tags so this might save
-    //    some roundtrips to the blob store.
+    //    we will check blob store. Full-route cache and fetch caches share a lot of tags
+    //    but we will only do actual blob read once withing a single request due to cacheStore
+    //    memoization.
     //    Additionally, we will resolve the promise as soon as we find first
     //    stale tag, so that we don't wait for all of them to resolve (but keep all
     //    running in case future `CacheHandler.get` calls would be able to use results).
@@ -521,14 +517,10 @@ export class NetlifyCacheHandler implements CacheHandlerForMultipleVersions {
       const tagManifestPromises: Promise<boolean>[] = []
 
       for (const tag of cacheTags) {
-        let tagManifestPromise: Promise<TagManifest | null> =
-          this.tagManifestsFetchedFromBlobStoreInCurrentRequest[tag]
-
-        if (!tagManifestPromise) {
-          tagManifestPromise = this.cacheStore.get<TagManifest>(tag, 'tagManifest.get')
-
-          this.tagManifestsFetchedFromBlobStoreInCurrentRequest[tag] = tagManifestPromise
-        }
+        const tagManifestPromise: Promise<TagManifest | null> = this.cacheStore.get<TagManifest>(
+          tag,
+          'tagManifest.get',
+        )
 
         tagManifestPromises.push(
           tagManifestPromise.then((tagManifest) => {

src/run/handlers/cache.cts

@@ -39,9 +39,22 @@ export type RequestContext = {
    */
   backgroundWorkPromise: Promise<unknown>
   logger: SystemLogger
+  requestID: string
 }
 
 type RequestContextAsyncLocalStorage = AsyncLocalStorage<RequestContext>
+const REQUEST_CONTEXT_GLOBAL_KEY = Symbol.for('nf-request-context-async-local-storage')
+const REQUEST_COUNTER_KEY = Symbol.for('nf-request-counter')
+const extendedGlobalThis = globalThis as typeof globalThis & {
+  [REQUEST_CONTEXT_GLOBAL_KEY]?: RequestContextAsyncLocalStorage
+  [REQUEST_COUNTER_KEY]?: number
+}
+
+function getFallbackRequestID() {
+  const requestNumber = extendedGlobalThis[REQUEST_COUNTER_KEY] ?? 0
+  extendedGlobalThis[REQUEST_COUNTER_KEY] = requestNumber + 1
+  return `#${requestNumber}`
+}
 
 export function createRequestContext(request?: Request, context?: FutureContext): RequestContext {
   const backgroundWorkPromises: Promise<unknown>[] = []
@@ -72,10 +85,10 @@ export function createRequestContext(request?: Request, context?: FutureContext)
       return Promise.allSettled(backgroundWorkPromises)
     },
     logger,
+    requestID: request?.headers.get('x-nf-request-id') ?? getFallbackRequestID(),
   }
 }
 
-const REQUEST_CONTEXT_GLOBAL_KEY = Symbol.for('nf-request-context-async-local-storage')
 let requestContextAsyncLocalStorage: RequestContextAsyncLocalStorage | undefined
 function getRequestContextAsyncLocalStorage() {
   if (requestContextAsyncLocalStorage) {
@@ -85,9 +98,6 @@ function getRequestContextAsyncLocalStorage() {
   // AsyncLocalStorage in the module scope, because it will be different for each
   // copy - so first time an instance of this module is used, we store AsyncLocalStorage
   // in global scope and reuse it for all subsequent calls
-  const extendedGlobalThis = globalThis as typeof globalThis & {
-    [REQUEST_CONTEXT_GLOBAL_KEY]?: RequestContextAsyncLocalStorage
-  }
   if (extendedGlobalThis[REQUEST_CONTEXT_GLOBAL_KEY]) {
     return extendedGlobalThis[REQUEST_CONTEXT_GLOBAL_KEY]
   }