tag cache refactored to use tagKey

conico974 · conico974 · commit 732282aa54ed · 2025-07-19T17:53:01.000+02:00
diff --git a/packages/open-next/src/adapters/cache.ts b/packages/open-next/src/adapters/cache.ts
@@ -6,6 +6,7 @@ import type {
 import type { CacheKey } from "types/overrides";
 import {
   createCacheKey,
+  createTagKey,
   getTagsFromValue,
   hasBeenRevalidated,
   writeTags,
@@ -71,7 +72,7 @@ export default class Cache {
       const _tags = [...(tags ?? []), ...(softTags ?? [])];
       const _lastModified = cachedEntry.lastModified ?? Date.now();
       const _hasBeenRevalidated = await hasBeenRevalidated(
-        baseKey,
+        key,
         _tags,
         cachedEntry,
       );
@@ -90,7 +91,7 @@ export default class Cache {
         );
         if (path) {
           const hasPathBeenUpdated = await hasBeenRevalidated(
-            path.replace("_N_T_/", ""),
+            createCacheKey({key: path.replace("_N_T_/", ""), type: "cache"}),
             [],
             cachedEntry,
           );
@@ -132,7 +133,7 @@ export default class Cache {
       const tags = getTagsFromValue(cacheData);
       const _lastModified = cachedEntry.lastModified ?? Date.now();
       const _hasBeenRevalidated = await hasBeenRevalidated(
-        baseKey,
+        key,
         tags,
         cachedEntry,
       );
@@ -323,7 +324,7 @@ export default class Cache {
 
     try {
       if (globalThis.tagCache.mode === "nextMode") {
-        const paths = (await globalThis.tagCache.getPathsByTags?.(_tags)) ?? [];
+        const paths = (await globalThis.tagCache.getPathsByTags?.(_tags.map(createTagKey))) ?? [];
 
         await writeTags(_tags);
         if (paths.length > 0) {
@@ -349,7 +350,7 @@ export default class Cache {
       for (const tag of _tags) {
         debug("revalidateTag", tag);
         // Find all keys with the given tag
-        const paths = await globalThis.tagCache.getByTag(tag);
+        const paths = await globalThis.tagCache.getByTag(createTagKey(tag));
         debug("Items", paths);
         const toInsert = paths.map((path) => ({
           path,
@@ -360,11 +361,11 @@ export default class Cache {
         if (tag.startsWith("_N_T_/")) {
           for (const path of paths) {
             // We need to find all hard tags for a given path
-            const _tags = await globalThis.tagCache.getByPath(path);
+            const _tags = await globalThis.tagCache.getByPath(createTagKey(path));
             const hardTags = _tags.filter((t) => !t.startsWith("_N_T_/"));
             // For every hard tag, we need to find all paths and revalidate them
             for (const hardTag of hardTags) {
-              const _paths = await globalThis.tagCache.getByTag(hardTag);
+              const _paths = await globalThis.tagCache.getByTag(createTagKey(hardTag));
               debug({ hardTag, _paths });
               toInsert.push(
                 ..._paths.map((path) => ({
@@ -377,7 +378,10 @@ export default class Cache {
         }
 
         // Update all keys with the given tag with revalidatedAt set to now
-        await writeTags(toInsert);
+        await writeTags(toInsert.map((t) => ({
+          path: createTagKey(t.path),
+          tag: createTagKey(t.tag),
+        })));
 
         // We can now invalidate all paths in the CDN
         // This only applies to `revalidateTag`, not to `res.revalidate()`
@@ -438,13 +442,13 @@ export default class Cache {
 
     // Get all tags stored in dynamodb for the given key
     // If any of the derived tags are not stored in dynamodb for the given key, write them
-    const storedTags = await globalThis.tagCache.getByPath(key);
+    const storedTags = await globalThis.tagCache.getByPath(createTagKey(key));
     const tagsToWrite = derivedTags.filter((tag) => !storedTags.includes(tag));
     if (tagsToWrite.length > 0) {
       await writeTags(
         tagsToWrite.map((tag) => ({
-          path: key,
-          tag: tag,
+          path: createTagKey(key),
+          tag: createTagKey(tag),
           // In case the tags are not there we just need to create them
           // but we don't want them to return from `getLastModified` as they are not stale
           revalidatedAt: 1,
diff --git a/packages/open-next/src/adapters/composable-cache.ts b/packages/open-next/src/adapters/composable-cache.ts
@@ -1,6 +1,6 @@
 import type { ComposableCacheEntry, ComposableCacheHandler } from "types/cache";
-import type { CacheKey } from "types/overrides";
-import { createCacheKey, writeTags } from "utils/cache";
+import type { CacheKey, TagKey } from "types/overrides";
+import { createCacheKey, createTagKey, writeTags } from "utils/cache";
 import { fromReadableStream, toReadableStream } from "utils/stream";
 import { debug, warn } from "./logger";
 
@@ -28,7 +28,7 @@ export default {
         result.value.tags.length > 0
       ) {
         const hasBeenRevalidated = await globalThis.tagCache.hasBeenRevalidated(
-          result.value.tags,
+          result.value.tags.map(createTagKey),
           result.lastModified,
         );
         if (hasBeenRevalidated) return undefined;
@@ -38,7 +38,7 @@ export default {
       ) {
         const hasBeenRevalidated =
           (await globalThis.tagCache.getLastModified(
-            cacheKey.baseKey,
+            cacheKey,
             result.lastModified,
           )) === -1;
         if (hasBeenRevalidated) return undefined;
@@ -66,11 +66,11 @@ export default {
       value: valueToStore,
     });
     if (globalThis.tagCache.mode === "original") {
-      const storedTags = await globalThis.tagCache.getByPath(cacheKey.baseKey);
+      const storedTags = await globalThis.tagCache.getByPath(cacheKey);
       const tagsToWrite = entry.tags.filter((tag) => !storedTags.includes(tag));
       if (tagsToWrite.length > 0) {
         await writeTags(
-          tagsToWrite.map((tag) => ({ tag, path: cacheKey.baseKey })),
+          tagsToWrite.map((tag) => ({ tag: createTagKey(tag), path: createTagKey(cacheKey.baseKey) })),
         );
       }
     }
@@ -82,7 +82,7 @@ export default {
   },
   async getExpiration(...tags: string[]) {
     if (globalThis.tagCache.mode === "nextMode") {
-      return globalThis.tagCache.getLastRevalidated(tags);
+      return globalThis.tagCache.getLastRevalidated(tags.map(createTagKey));
     }
     // We always return 0 here, original tag cache are handled directly in the get part
     // TODO: We need to test this more, i'm not entirely sure that this is working as expected
@@ -98,16 +98,16 @@ export default {
     // We need to find all paths linked to to these tags
     const pathsToUpdate = await Promise.all(
       tags.map(async (tag) => {
-        const paths = await tagCache.getByTag(tag);
+        const paths = await tagCache.getByTag(createTagKey(tag));
         return paths.map((path) => ({
-          path,
-          tag,
+          path: createTagKey(path),
+          tag: createTagKey(tag),
           revalidatedAt,
         }));
       }),
     );
     // We need to deduplicate paths, we use a set for that
-    const setToWrite = new Set<{ path: string; tag: string }>();
+    const setToWrite = new Set<{ path: TagKey; tag: TagKey; revalidatedAt: number }>();
     for (const entry of pathsToUpdate.flat()) {
       setToWrite.add(entry);
     }
diff --git a/packages/open-next/src/adapters/dynamo-provider.ts b/packages/open-next/src/adapters/dynamo-provider.ts
@@ -2,6 +2,7 @@ import { readFileSync } from "node:fs";
 
 import { createGenericHandler } from "../core/createGenericHandler.js";
 import { resolveTagCache } from "../core/resolve.js";
+import { createTagKey } from "utils/cache.js";
 
 const PHYSICAL_RESOURCE_ID = "dynamodb-cache" as const;
 
@@ -60,8 +61,8 @@ async function insert(
   const data: DataType[] = JSON.parse(file);
 
   const parsedData = data.map((item) => ({
-    tag: item.tag.S,
-    path: item.path.S,
+    tag: createTagKey(item.tag.S),
+    path: createTagKey(item.path.S),
     revalidatedAt: Number.parseInt(item.revalidatedAt.N),
   }));
 
diff --git a/packages/open-next/src/core/routing/cacheInterceptor.ts b/packages/open-next/src/core/routing/cacheInterceptor.ts
@@ -226,7 +226,7 @@ export async function cacheInterceptor(
       if (cachedData.value?.type === "app") {
         const tags = getTagsFromValue(cachedData.value);
         const _hasBeenRevalidated = await hasBeenRevalidated(
-          localizedPath,
+          createCacheKey({ key: localizedPath, type: "cache" }),
           tags,
           cachedData,
         );
diff --git a/packages/open-next/src/overrides/tagCache/dummy.ts b/packages/open-next/src/overrides/tagCache/dummy.ts
@@ -1,4 +1,4 @@
-import type { TagCache } from "types/overrides";
+import type { TagCache, TagKey } from "types/overrides";
 
 // We don't want to throw error on this one because we might use it when we don't need tag cache
 const dummyTagCache: TagCache = {
@@ -10,7 +10,7 @@ const dummyTagCache: TagCache = {
   getByTag: async () => {
     return [];
   },
-  getLastModified: async (_: string, lastModified) => {
+  getLastModified: async (_: TagKey, lastModified) => {
     return lastModified ?? Date.now();
   },
   writeTags: async () => {
diff --git a/packages/open-next/src/overrides/tagCache/dynamodb-lite.ts b/packages/open-next/src/overrides/tagCache/dynamodb-lite.ts
@@ -2,7 +2,7 @@
 import path from "node:path";
 
 import { AwsClient } from "aws4fetch";
-import type { TagCache } from "types/overrides";
+import type { TagCache, TagKey } from "types/overrides";
 import { RecoverableError } from "utils/error";
 import { customFetchClient } from "utils/fetch";
 
@@ -50,17 +50,14 @@ const awsFetch = (
   );
 };
 
-function buildDynamoKey(key: string) {
-  const { NEXT_BUILD_ID } = process.env;
-  // FIXME: We should probably use something else than path.join here
-  // this could transform some fetch cache key into a valid path
-  return path.posix.join(NEXT_BUILD_ID ?? "", key);
+function buildDynamoKey(key: TagKey) {
+  return `${key.buildId ?? ""}_${key.baseKey}`;
 }
 
-function buildDynamoObject(path: string, tags: string, revalidatedAt?: number) {
+function buildDynamoObject(path: TagKey, tag: TagKey, revalidatedAt?: number) {
   return {
     path: { S: buildDynamoKey(path) },
-    tag: { S: buildDynamoKey(tags) },
+    tag: { S: buildDynamoKey(tag) },
     revalidatedAt: { N: `${revalidatedAt ?? Date.now()}` },
   };
 }
diff --git a/packages/open-next/src/overrides/tagCache/dynamodb-nextMode.ts b/packages/open-next/src/overrides/tagCache/dynamodb-nextMode.ts
@@ -1,4 +1,4 @@
-import type { NextModeTagCache } from "types/overrides";
+import type { NextModeTagCache, TagKey } from "types/overrides";
 
 import { AwsClient } from "aws4fetch";
 import { RecoverableError } from "utils/error";
@@ -49,17 +49,14 @@ const awsFetch = (
   );
 };
 
-function buildDynamoKey(key: string) {
-  const { NEXT_BUILD_ID } = process.env;
-  // FIXME: We should probably use something else than path.join here
-  // this could transform some fetch cache key into a valid path
-  return path.posix.join(NEXT_BUILD_ID ?? "", "_tag", key);
+function buildDynamoKey(key: TagKey) {
+  return `${key.buildId ?? ""}_${key.baseKey}`;
 }
 
 // We use the same key for both path and tag
 // That's mostly for compatibility reason so that it's easier to use this with existing infra
 // FIXME: Allow a simpler object without an unnecessary path key
-function buildDynamoObject(tag: string, revalidatedAt?: number) {
+function buildDynamoObject(tag: TagKey, revalidatedAt?: number) {
   return {
     path: { S: buildDynamoKey(tag) },
     tag: { S: buildDynamoKey(tag) },
@@ -71,11 +68,11 @@ function buildDynamoObject(tag: string, revalidatedAt?: number) {
 export default {
   name: "ddb-nextMode",
   mode: "nextMode",
-  getLastRevalidated: async (tags: string[]) => {
+  getLastRevalidated: async (tags: TagKey[]) => {
     // Not supported for now
     return 0;
   },
-  hasBeenRevalidated: async (tags: string[], lastModified?: number) => {
+  hasBeenRevalidated: async (tags: TagKey[], lastModified?: number) => {
     if (globalThis.openNextConfig.dangerous?.disableTagCache) {
       return false;
     }
@@ -117,7 +114,7 @@ export default {
     debug("retrieved tags", revalidatedTags);
     return revalidatedTags.length > 0;
   },
-  writeTags: async (tags: string[]) => {
+  writeTags: async (tags: TagKey[]) => {
     try {
       const { CACHE_DYNAMO_TABLE } = process.env;
       if (globalThis.openNextConfig.dangerous?.disableTagCache) {
diff --git a/packages/open-next/src/overrides/tagCache/dynamodb.ts b/packages/open-next/src/overrides/tagCache/dynamodb.ts
@@ -6,7 +6,7 @@ import {
   DynamoDBClient,
   QueryCommand,
 } from "@aws-sdk/client-dynamodb";
-import type { TagCache } from "types/overrides";
+import type { TagCache, TagKey } from "types/overrides";
 
 import { awsLogger, debug, error } from "../../adapters/logger";
 import { chunk, parseNumberFromEnv } from "../../adapters/util";
@@ -27,16 +27,14 @@ function parseDynamoClientConfigFromEnv(): DynamoDBClientConfig {
 
 const dynamoClient = new DynamoDBClient(parseDynamoClientConfigFromEnv());
 
-function buildDynamoKey(key: string) {
-  // FIXME: We should probably use something else than path.join here
-  // this could transform some fetch cache key into a valid path
-  return path.posix.join(NEXT_BUILD_ID ?? "", key);
+function buildDynamoKey(key: TagKey) {
+  return `${key.buildId ?? ""}_${key.baseKey}`;
 }
 
-function buildDynamoObject(path: string, tags: string, revalidatedAt?: number) {
+function buildDynamoObject(path: TagKey, tag: TagKey, revalidatedAt?: number) {
   return {
     path: { S: buildDynamoKey(path) },
-    tag: { S: buildDynamoKey(tags) },
+    tag: { S: buildDynamoKey(tag) },
     revalidatedAt: { N: `${revalidatedAt ?? Date.now()}` },
   };
 }
diff --git a/packages/open-next/src/overrides/tagCache/fs-dev.ts b/packages/open-next/src/overrides/tagCache/fs-dev.ts
@@ -1,7 +1,7 @@
 import fs from "node:fs";
 import path from "node:path";
 
-import type { TagCache } from "types/overrides";
+import type { TagCache, TagKey } from "types/overrides";
 import { getMonorepoRelativePath } from "utils/normalize-path";
 
 const tagFile = path.join(
@@ -19,20 +19,20 @@ let tags = JSON.parse(tagContent) as {
 const tagCache: TagCache = {
   name: "fs-dev",
   mode: "original",
-  getByPath: async (path: string) => {
+  getByPath: async (path: TagKey) => {
     return tags
-      .filter((tagPathMapping) => tagPathMapping.path.S === path)
+      .filter((tagPathMapping) => tagPathMapping.path.S === path.baseKey)
       .map((tag) => tag.tag.S);
   },
-  getByTag: async (tag: string) => {
+  getByTag: async (tag: TagKey) => {
     return tags
-      .filter((tagPathMapping) => tagPathMapping.tag.S === tag)
+      .filter((tagPathMapping) => tagPathMapping.tag.S === tag.baseKey)
       .map((tag) => tag.path.S);
   },
-  getLastModified: async (path: string, lastModified?: number) => {
+  getLastModified: async (path: TagKey, lastModified?: number) => {
     const revalidatedTags = tags.filter(
       (tagPathMapping) =>
-        tagPathMapping.path.S === path &&
+        tagPathMapping.path.S === path.baseKey &&
         Number.parseInt(tagPathMapping.revalidatedAt.N) > (lastModified ?? 0),
     );
     return revalidatedTags.length > 0 ? -1 : (lastModified ?? Date.now());
@@ -46,8 +46,8 @@ const tagCache: TagCache = {
     );
     tags = unchangedTags.concat(
       newTags.map((tag) => ({
-        tag: { S: tag.tag },
-        path: { S: tag.path },
+        tag: { S: tag.tag.baseKey },
+        path: { S: tag.path.baseKey },
         revalidatedAt: { N: String(tag.revalidatedAt ?? 1) },
       })),
     );
diff --git a/packages/open-next/src/types/overrides.ts b/packages/open-next/src/types/overrides.ts
diff --git a/packages/open-next/src/utils/cache.ts b/packages/open-next/src/utils/cache.ts
