From 4623ebbb549972ea6b68a1168195f82019673807 Mon Sep 17 00:00:00 2001 From: Zach Leatherman Date: Thu, 19 Dec 2024 13:38:07 -0600 Subject: [PATCH] Keep cached contents in memory for AssetCache instance. Renames `wasLastFetchHit` to `wasLastFetchCacheHit`. Cache fetch promise. --- src/AssetCache.js | 48 ++++++++++++------- src/RemoteAssetCache.js | 18 ++++++- test/QueueTest.js | 101 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 18 deletions(-) diff --git a/src/AssetCache.js b/src/AssetCache.js index 9ced8c5..001d8e4 100644 --- a/src/AssetCache.js +++ b/src/AssetCache.js @@ -5,7 +5,9 @@ const { createHash } = require("crypto"); const Sources = require("./Sources.js"); -const debug = require("debug")("Eleventy:Fetch"); +const debugUtil = require("debug"); +const debug = debugUtil("Eleventy:Fetch"); +const debugAssets = debugUtil("Eleventy:Assets"); class AssetCache { #source; @@ -15,6 +17,7 @@ class AssetCache { #cacheDirectory; #cacheLocationDirty = false; #dirEnsured = false; + #rawContents = {} constructor(source, cacheDirectory, options = {}) { if(!Sources.isValidSource(source)) { @@ -232,7 +235,20 @@ class AssetCache { throw new Error("save(contents) expects contents (was falsy)"); } + this.cache.set(this.hash, { + cachedAt: Date.now(), + type: type, + metadata, + }); + let contentPath = this.getCachedContentsPath(type); + + if (type === "json" || type === "parsed-xml") { + contents = JSON.stringify(contents); + } + + this.#rawContents[type] = contents; + if(this.options.dryRun) { debug(`Dry run writing ${contentPath}`); return; @@ -240,24 +256,15 @@ class AssetCache { this.ensureDir(); - if (type === "json" || type === "parsed-xml") { - contents = JSON.stringify(contents); - } - + debugAssets("Writing cache file to disk for %o", this.source) // the contents must exist before the cache metadata are saved below fs.writeFileSync(contentPath, contents); debug(`Writing ${contentPath}`); - this.cache.set(this.hash, { - cachedAt: Date.now(), - type: type, - metadata, - }); - this.cache.save(); } - async getCachedContents(type) { + async #getCachedContents(type) { let contentPath = this.getCachedContentsPath(type); debug(`Fetching from cache ${contentPath}`); @@ -268,6 +275,15 @@ class AssetCache { return fs.readFileSync(contentPath, type !== "buffer" ? "utf8" : null); } + getCachedContents(type) { + if(!this.#rawContents[type]) { + this.#rawContents[type] = this.#getCachedContents(type); + } + + // already saved on this instance in-memory + return this.#rawContents[type]; + } + _backwardsCompatibilityGetCachedValue(type) { if (type === "json") { return this.cachedObject.contents; @@ -341,16 +357,16 @@ class AssetCache { return !this.isCacheValid(duration); } - async fetch(options) { - if (this.isCacheValid(options.duration)) { + // This is only included for completenes—not on the docs. + async fetch(optionsOverride = {}) { + if (this.isCacheValid(optionsOverride.duration)) { // promise this.log(`Using cached version of: ${this.uniqueKey}`); return this.getCachedValue(); } this.log(`Saving ${this.uniqueKey} to ${this.cacheFilename}`); - - await this.save(this.source, options.type); + await this.save(this.source, optionsOverride.type); return this.source; } diff --git a/src/RemoteAssetCache.js b/src/RemoteAssetCache.js index 785b543..6122b7d 100644 --- a/src/RemoteAssetCache.js +++ b/src/RemoteAssetCache.js @@ -7,6 +7,7 @@ const assetDebug = require("debug")("Eleventy:Assets"); class RemoteAssetCache extends AssetCache { #queue; #queuePromise; + #fetchPromise; #lastFetchType; constructor(source, cacheDirectory, options = {}) { @@ -126,11 +127,11 @@ class RemoteAssetCache extends AssetCache { // if last fetch was a cache hit (no fetch occurred) or a cache miss (fetch did occur) // used by Eleventy Image in disk cache checks. - wasLastFetchHit() { + wasLastFetchCacheHit() { return this.#lastFetchType === "hit"; } - async fetch(optionsOverride = {}) { + async #fetch(optionsOverride = {}) { // Important: no disk writes when dryRun // As of Fetch v4, reads are now allowed! if (this.isCacheValid(optionsOverride.duration)) { @@ -203,5 +204,18 @@ class RemoteAssetCache extends AssetCache { } } } + + // async but not explicitly declared for promise equality checks + // returns a Promise + fetch(optionsOverride = {}) { + if(!this.#fetchPromise) { + // one at a time. clear when finished + this.#fetchPromise = this.#fetch(optionsOverride).finally(() => { + this.#fetchPromise = undefined; + }); + } + + return this.#fetchPromise; + } } module.exports = RemoteAssetCache; diff --git a/test/QueueTest.js b/test/QueueTest.js index eddd318..30fe04d 100644 --- a/test/QueueTest.js +++ b/test/QueueTest.js @@ -117,3 +117,104 @@ test("Docs example https://www.11ty.dev/docs/plugins/fetch/#manually-store-your- followerCount: 1000 }); }); + +test("Raw Fetch using queue method", async (t) => { + let pngUrl = "https://www.zachleat.com/img/avatar-2017.png?q=1"; + let ac1 = Fetch(pngUrl); + let ac2 = Fetch(pngUrl); + + // Destroy to clear any existing cache + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} + + // Make sure the instance is the same + t.is(ac1, ac2); + + let result1 = await ac1.queue(); + t.false(ac1.wasLastFetchCacheHit()) + + let result2 = await ac1.queue(); + // reuses the same fetch + t.false(ac1.wasLastFetchCacheHit()) + + t.is(result1, result2); + + // file is now accessible + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} +}); + + +test("Raw Fetch using fetch method", async (t) => { + let pngUrl = "https://www.zachleat.com/img/avatar-2017.png?q=2"; + let ac1 = Fetch(pngUrl); + let ac2 = Fetch(pngUrl); + + // Destroy to clear any existing cache + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} + + // Make sure the instance is the same + t.is(ac1, ac2); + + let result1 = await ac1.fetch(); + t.false(ac1.wasLastFetchCacheHit()) + + let result2 = await ac1.fetch(); + t.true(ac1.wasLastFetchCacheHit()) + + t.is(result1, result2); + + // file is now accessible + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} +}); + +test("Raw Fetch using fetch method (check parallel fetch promise reuse)", async (t) => { + let pngUrl = "https://www.zachleat.com/img/avatar-2017.png?q=3"; + let ac1 = Fetch(pngUrl); + let ac2 = Fetch(pngUrl); + + // Destroy to clear any existing cache + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} + + // Make sure the instance is the same + t.is(ac1, ac2); + + let fetch1 = ac1.fetch(); + let fetch2 = ac1.fetch(); + t.is(fetch1, fetch2); + + t.is(await fetch1, await fetch2); + + t.false(ac1.wasLastFetchCacheHit()) + + // file is now accessible + try { + await ac1.destroy(); + } catch (e) {} + try { + await ac2.destroy(); + } catch (e) {} +});