diff --git a/src/index.ts b/src/index.ts index 99cfef927..23686955d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -7,6 +7,7 @@ import * as sumchecker from 'sumchecker'; import { getArtifactFileName, getArtifactRemoteURL, getArtifactVersion } from './artifact-utils'; import { ElectronArtifactDetails, + ElectronDownloadCacheMode, ElectronDownloadRequestOptions, ElectronGenericArtifactDetails, ElectronPlatformArtifactDetails, @@ -21,6 +22,11 @@ import { getNodeArch, ensureIsTruthyString, isOfficialLinuxIA32Download, + mkdtemp, + doesCallerOwnTemporaryOutput, + effectiveCacheMode, + shouldTryReadCache, + TempDirCleanUpMode, } from './utils'; export { getHostArch } from './utils'; @@ -42,58 +48,76 @@ async function validateArtifact( downloadedAssetPath: string, _downloadArtifact: ArtifactDownloader, ): Promise { - return await withTempDirectoryIn(artifactDetails.tempDirectory, async tempFolder => { - // Don't try to verify the hash of the hash file itself - // and for older versions that don't have a SHASUMS256.txt - if ( - !artifactDetails.artifactName.startsWith('SHASUMS256') && - !artifactDetails.unsafelyDisableChecksums && - semver.gte(artifactDetails.version, '1.3.2') - ) { - let shasumPath: string; - const checksums = artifactDetails.checksums; - if (checksums) { - shasumPath = path.resolve(tempFolder, 'SHASUMS256.txt'); - const fileNames: string[] = Object.keys(checksums); - if (fileNames.length === 0) { - throw new Error( - 'Provided "checksums" object is empty, cannot generate a valid SHASUMS256.txt', - ); + return await withTempDirectoryIn( + artifactDetails.tempDirectory, + async tempFolder => { + // Don't try to verify the hash of the hash file itself + // and for older versions that don't have a SHASUMS256.txt + if ( + !artifactDetails.artifactName.startsWith('SHASUMS256') && + !artifactDetails.unsafelyDisableChecksums && + semver.gte(artifactDetails.version, '1.3.2') + ) { + let shasumPath: string; + const checksums = artifactDetails.checksums; + if (checksums) { + shasumPath = path.resolve(tempFolder, 'SHASUMS256.txt'); + const fileNames: string[] = Object.keys(checksums); + if (fileNames.length === 0) { + throw new Error( + 'Provided "checksums" object is empty, cannot generate a valid SHASUMS256.txt', + ); + } + const generatedChecksums = fileNames + .map(fileName => `${checksums[fileName]} *${fileName}`) + .join('\n'); + await fs.writeFile(shasumPath, generatedChecksums); + } else { + shasumPath = await _downloadArtifact({ + isGeneric: true, + version: artifactDetails.version, + artifactName: 'SHASUMS256.txt', + force: false, + downloadOptions: artifactDetails.downloadOptions, + cacheRoot: artifactDetails.cacheRoot, + downloader: artifactDetails.downloader, + mirrorOptions: artifactDetails.mirrorOptions, + // Never use the cache for loading checksums, load + // them fresh every time + cacheMode: ElectronDownloadCacheMode.Bypass, + }); } - const generatedChecksums = fileNames - .map(fileName => `${checksums[fileName]} *${fileName}`) - .join('\n'); - await fs.writeFile(shasumPath, generatedChecksums); - } else { - shasumPath = await _downloadArtifact({ - isGeneric: true, - version: artifactDetails.version, - artifactName: 'SHASUMS256.txt', - force: artifactDetails.force, - downloadOptions: artifactDetails.downloadOptions, - cacheRoot: artifactDetails.cacheRoot, - downloader: artifactDetails.downloader, - mirrorOptions: artifactDetails.mirrorOptions, - }); - } - // For versions 1.3.2 - 1.3.4, need to overwrite the `defaultTextEncoding` option: - // https://github.com/electron/electron/pull/6676#discussion_r75332120 - if (semver.satisfies(artifactDetails.version, '1.3.2 - 1.3.4')) { - const validatorOptions: sumchecker.ChecksumOptions = {}; - validatorOptions.defaultTextEncoding = 'binary'; - const checker = new sumchecker.ChecksumValidator('sha256', shasumPath, validatorOptions); - await checker.validate( - path.dirname(downloadedAssetPath), - path.basename(downloadedAssetPath), - ); - } else { - await sumchecker('sha256', shasumPath, path.dirname(downloadedAssetPath), [ - path.basename(downloadedAssetPath), - ]); + try { + // For versions 1.3.2 - 1.3.4, need to overwrite the `defaultTextEncoding` option: + // https://github.com/electron/electron/pull/6676#discussion_r75332120 + if (semver.satisfies(artifactDetails.version, '1.3.2 - 1.3.4')) { + const validatorOptions: sumchecker.ChecksumOptions = {}; + validatorOptions.defaultTextEncoding = 'binary'; + const checker = new sumchecker.ChecksumValidator( + 'sha256', + shasumPath, + validatorOptions, + ); + await checker.validate( + path.dirname(downloadedAssetPath), + path.basename(downloadedAssetPath), + ); + } else { + await sumchecker('sha256', shasumPath, path.dirname(downloadedAssetPath), [ + path.basename(downloadedAssetPath), + ]); + } + } finally { + // Once we're done make sure we clean up the shasum temp dir + await fs.remove(path.dirname(shasumPath)); + } } - } - }); + }, + doesCallerOwnTemporaryOutput(effectiveCacheMode(artifactDetails)) + ? TempDirCleanUpMode.ORPHAN + : TempDirCleanUpMode.CLEAN, + ); } /** @@ -133,9 +157,10 @@ export async function downloadArtifact( const fileName = getArtifactFileName(details); const url = await getArtifactRemoteURL(details); const cache = new Cache(details.cacheRoot); + const cacheMode = effectiveCacheMode(details); // Do not check if the file exists in the cache when force === true - if (!details.force) { + if (shouldTryReadCache(cacheMode)) { d(`Checking the cache (${details.cacheRoot}) for ${fileName} (${url})`); const cachedPath = await cache.getPathForFileInCache(url, fileName); @@ -143,11 +168,22 @@ export async function downloadArtifact( d('Cache miss'); } else { d('Cache hit'); + let artifactPath = cachedPath; + if (doesCallerOwnTemporaryOutput(cacheMode)) { + // Copy out of cache into temporary directory if readOnly cache so + // that the caller can take ownership of the returned file + const tempDir = await mkdtemp(artifactDetails.tempDirectory); + artifactPath = path.resolve(tempDir, fileName); + await fs.copyFile(cachedPath, artifactPath); + } try { - await validateArtifact(details, cachedPath, downloadArtifact); + await validateArtifact(details, artifactPath, downloadArtifact); - return cachedPath; + return artifactPath; } catch (err) { + if (doesCallerOwnTemporaryOutput(cacheMode)) { + await fs.remove(path.dirname(artifactPath)); + } d("Artifact in cache didn't match checksums", err); d('falling back to re-download'); } @@ -167,21 +203,29 @@ export async function downloadArtifact( console.warn('For more info: https://electronjs.org/blog/linux-32bit-support'); } - return await withTempDirectoryIn(details.tempDirectory, async tempFolder => { - const tempDownloadPath = path.resolve(tempFolder, getArtifactFileName(details)); + return await withTempDirectoryIn( + details.tempDirectory, + async tempFolder => { + const tempDownloadPath = path.resolve(tempFolder, getArtifactFileName(details)); - const downloader = details.downloader || (await getDownloaderForSystem()); - d( - `Downloading ${url} to ${tempDownloadPath} with options: ${JSON.stringify( - details.downloadOptions, - )}`, - ); - await downloader.download(url, tempDownloadPath, details.downloadOptions); + const downloader = details.downloader || (await getDownloaderForSystem()); + d( + `Downloading ${url} to ${tempDownloadPath} with options: ${JSON.stringify( + details.downloadOptions, + )}`, + ); + await downloader.download(url, tempDownloadPath, details.downloadOptions); - await validateArtifact(details, tempDownloadPath, downloadArtifact); + await validateArtifact(details, tempDownloadPath, downloadArtifact); - return await cache.putFileInCache(url, tempDownloadPath, fileName); - }); + if (doesCallerOwnTemporaryOutput(cacheMode)) { + return tempDownloadPath; + } else { + return await cache.putFileInCache(url, tempDownloadPath, fileName); + } + }, + doesCallerOwnTemporaryOutput(cacheMode) ? TempDirCleanUpMode.ORPHAN : TempDirCleanUpMode.CLEAN, + ); } /** diff --git a/src/types.ts b/src/types.ts index 08b4f5378..c753e1885 100644 --- a/src/types.ts +++ b/src/types.ts @@ -82,6 +82,28 @@ export interface ElectronDownloadRequest { artifactName: string; } +export enum ElectronDownloadCacheMode { + /** + * Reads from the cache if present + * Writes to the cache after fetch if not present + */ + ReadWrite, + /** + * Reads from the cache if present + * Will **not** write back to the cache after fetching missing artifact + */ + ReadOnly, + /** + * Skips reading from the cache + * Will write back into the cache, overwriting anything currently in the cache after fetch + */ + WriteOnly, + /** + * Bypasses the cache completely, neither reads from nor writes to the cache + */ + Bypass, +} + /** * @category Download Electron */ @@ -90,6 +112,7 @@ export interface ElectronDownloadRequestOptions { * Whether to download an artifact regardless of whether it's in the cache directory. * * @defaultValue `false` + * @deprecated This option is deprecated and directly maps to {@link cacheMode | `cacheMode: ElectronDownloadCacheMode.WriteOnly`} */ force?: boolean; /** @@ -148,6 +171,24 @@ export interface ElectronDownloadRequestOptions { * @defaultValue the OS default temporary directory via [`os.tmpdir()`](https://nodejs.org/api/os.html#ostmpdir) */ tempDirectory?: string; + /** + * Controls the cache read and write behavior. + * + * When set to either {@link ElectronDownloadCacheMode.ReadOnly | ReadOnly} or + * {@link ElectronDownloadCacheMode.Bypass | Bypass}, the caller is responsible + * for cleaning up the returned file path once they are done using it + * (e.g. via `fs.remove(path.dirname(pathFromElectronGet))`). + * + * When set to either {@link ElectronDownloadCacheMode.WriteOnly | WriteOnly} or + * {@link ElectronDownloadCacheMode.ReadWrite | ReadWrite} (the default), the caller + * should not move or delete the file path that is returned as the path + * points directly to the disk cache. + * + * This option cannot be used in conjunction with {@link ElectronDownloadRequestOptions.force}. + * + * @defaultValue {@link ElectronDownloadCacheMode.ReadWrite} + */ + cacheMode?: ElectronDownloadCacheMode; } /** diff --git a/src/utils.ts b/src/utils.ts index f9c8f1cf0..284b6c958 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,6 +2,11 @@ import * as childProcess from 'child_process'; import * as fs from 'fs-extra'; import * as os from 'os'; import * as path from 'path'; +import { + ElectronDownloadCacheMode, + ElectronGenericArtifactDetails, + ElectronPlatformArtifactDetailsWithDefaults, +} from './types'; async function useAndRemoveDirectory( directory: string, @@ -16,17 +21,34 @@ async function useAndRemoveDirectory( return result; } +export async function mkdtemp(parentDirectory: string = os.tmpdir()): Promise { + const tempDirectoryPrefix = 'electron-download-'; + return await fs.mkdtemp(path.resolve(parentDirectory, tempDirectoryPrefix)); +} + +export enum TempDirCleanUpMode { + CLEAN, + ORPHAN, +} + export async function withTempDirectoryIn( parentDirectory: string = os.tmpdir(), fn: (directory: string) => Promise, + cleanUp: TempDirCleanUpMode, ): Promise { - const tempDirectoryPrefix = 'electron-download-'; - const tempDirectory = await fs.mkdtemp(path.resolve(parentDirectory, tempDirectoryPrefix)); - return useAndRemoveDirectory(tempDirectory, fn); + const tempDirectory = await mkdtemp(parentDirectory); + if (cleanUp === TempDirCleanUpMode.CLEAN) { + return useAndRemoveDirectory(tempDirectory, fn); + } else { + return fn(tempDirectory); + } } -export async function withTempDirectory(fn: (directory: string) => Promise): Promise { - return withTempDirectoryIn(undefined, fn); +export async function withTempDirectory( + fn: (directory: string) => Promise, + cleanUp: TempDirCleanUpMode, +): Promise { + return withTempDirectoryIn(undefined, fn, cleanUp); } export function normalizeVersion(version: string): string { @@ -122,3 +144,39 @@ export function setEnv(key: string, value: string | undefined): void { process.env[key] = value; } } + +export function effectiveCacheMode( + artifactDetails: ElectronPlatformArtifactDetailsWithDefaults | ElectronGenericArtifactDetails, +): ElectronDownloadCacheMode { + if (artifactDetails.force) { + if (artifactDetails.cacheMode) { + throw new Error( + 'Setting both "force" and "cacheMode" is not supported, please exclusively use "cacheMode"', + ); + } + return ElectronDownloadCacheMode.WriteOnly; + } + + return artifactDetails.cacheMode || ElectronDownloadCacheMode.ReadWrite; +} + +export function shouldTryReadCache(cacheMode: ElectronDownloadCacheMode): boolean { + return ( + cacheMode === ElectronDownloadCacheMode.ReadOnly || + cacheMode === ElectronDownloadCacheMode.ReadWrite + ); +} + +export function shouldWriteCache(cacheMode: ElectronDownloadCacheMode): boolean { + return ( + cacheMode === ElectronDownloadCacheMode.WriteOnly || + cacheMode === ElectronDownloadCacheMode.ReadWrite + ); +} + +export function doesCallerOwnTemporaryOutput(cacheMode: ElectronDownloadCacheMode): boolean { + return ( + cacheMode === ElectronDownloadCacheMode.Bypass || + cacheMode === ElectronDownloadCacheMode.ReadOnly + ); +} diff --git a/test/GotDownloader.network.spec.ts b/test/GotDownloader.network.spec.ts index 62add393e..e2a7e3dd3 100644 --- a/test/GotDownloader.network.spec.ts +++ b/test/GotDownloader.network.spec.ts @@ -2,7 +2,7 @@ import * as fs from 'fs-extra'; import * as path from 'path'; import { GotDownloader } from '../src/GotDownloader'; -import { withTempDirectory } from '../src/utils'; +import { TempDirCleanUpMode, withTempDirectory } from '../src/utils'; import { EventEmitter } from 'events'; describe('GotDownloader', () => { @@ -26,7 +26,7 @@ describe('GotDownloader', () => { expect(await fs.pathExists(testFile)).toEqual(true); expect(await fs.readFile(testFile, 'utf8')).toMatchSnapshot(); expect(progressCallbackCalled).toEqual(true); - }); + }, TempDirCleanUpMode.CLEAN); }); it('should throw an error if the file does not exist', async function() { @@ -36,7 +36,7 @@ describe('GotDownloader', () => { const url = 'https://github.com/electron/electron/releases/download/v2.0.18/bad.file'; const snapshot = `[HTTPError: Response code 404 (Not Found) for ${url}]`; await expect(downloader.download(url, testFile)).rejects.toMatchInlineSnapshot(snapshot); - }); + }, TempDirCleanUpMode.CLEAN); }); it('should throw an error if the file write stream fails', async () => { @@ -55,7 +55,7 @@ describe('GotDownloader', () => { testFile, ), ).rejects.toMatchInlineSnapshot(`"bad write error thing"`); - }); + }, TempDirCleanUpMode.CLEAN); }); it('should download to a deep uncreated path', async () => { @@ -71,7 +71,7 @@ describe('GotDownloader', () => { ).resolves.toBeUndefined(); expect(await fs.pathExists(testFile)).toEqual(true); expect(await fs.readFile(testFile, 'utf8')).toMatchSnapshot(); - }); + }, TempDirCleanUpMode.CLEAN); }); }); }); diff --git a/test/index.spec.ts b/test/index.spec.ts index 58ed8c534..732756d7c 100644 --- a/test/index.spec.ts +++ b/test/index.spec.ts @@ -5,7 +5,7 @@ import * as path from 'path'; import { FixtureDownloader } from './FixtureDownloader'; import { download, downloadArtifact } from '../src'; -import { DownloadOptions } from '../src/types'; +import { DownloadOptions, ElectronDownloadCacheMode } from '../src/types'; import * as sumchecker from 'sumchecker'; jest.mock('sumchecker'); @@ -31,6 +31,8 @@ describe('Public API', () => { expect(typeof zipPath).toEqual('string'); expect(await fs.pathExists(zipPath)).toEqual(true); expect(path.extname(zipPath)).toEqual('.zip'); + expect(path.dirname(zipPath).startsWith(cacheRoot)).toEqual(true); + expect(fs.readdirSync(cacheRoot).length).toBeGreaterThan(0); }); it('should return a valid path to a downloaded zip file for nightly releases', async () => { @@ -129,6 +131,35 @@ describe('Public API', () => { expect(path.extname(zipPath)).toEqual('.zip'); process.env.ELECTRON_CUSTOM_VERSION = ''; }); + + it('should not put artifact in cache when cacheMode=ReadOnly', async () => { + const zipPath = await download('2.0.10', { + cacheRoot, + downloader, + cacheMode: ElectronDownloadCacheMode.ReadOnly, + }); + expect(typeof zipPath).toEqual('string'); + expect(await fs.pathExists(zipPath)).toEqual(true); + expect(path.extname(zipPath)).toEqual('.zip'); + expect(path.dirname(zipPath).startsWith(cacheRoot)).toEqual(false); + expect(fs.readdirSync(cacheRoot).length).toEqual(0); + }); + + it('should use cache hits when cacheMode=ReadOnly', async () => { + const zipPath = await download('2.0.9', { + cacheRoot, + downloader, + }); + await fs.writeFile(zipPath, 'cached content'); + const zipPath2 = await download('2.0.9', { + cacheRoot, + downloader, + cacheMode: ElectronDownloadCacheMode.ReadOnly, + }); + expect(zipPath2).not.toEqual(zipPath); + expect(path.dirname(zipPath2).startsWith(cacheRoot)).toEqual(false); + expect(await fs.readFile(zipPath2, 'utf8')).toEqual('cached content'); + }); }); describe('downloadArtifact()', () => { @@ -143,6 +174,8 @@ describe('Public API', () => { expect(await fs.pathExists(dtsPath)).toEqual(true); expect(path.basename(dtsPath)).toEqual('electron.d.ts'); expect(await fs.readFile(dtsPath, 'utf8')).toContain('declare namespace Electron'); + expect(path.dirname(dtsPath).startsWith(cacheRoot)).toEqual(true); + expect(fs.readdirSync(cacheRoot).length).toBeGreaterThan(0); }); it('should work with the default platform/arch', async () => { @@ -231,6 +264,49 @@ describe('Public API', () => { expect(path.basename(zipPath)).toMatchInlineSnapshot(`"electron-v2.0.10-darwin-x64.zip"`); }); + it('should not put artifact in cache when cacheMode=ReadOnly', async () => { + const driverPath = await downloadArtifact({ + cacheRoot, + downloader, + version: '2.0.9', + artifactName: 'chromedriver', + platform: 'darwin', + arch: 'x64', + cacheMode: ElectronDownloadCacheMode.ReadOnly, + }); + expect(await fs.pathExists(driverPath)).toEqual(true); + expect(path.basename(driverPath)).toMatchInlineSnapshot( + `"chromedriver-v2.0.9-darwin-x64.zip"`, + ); + expect(path.extname(driverPath)).toEqual('.zip'); + expect(path.dirname(driverPath).startsWith(cacheRoot)).toEqual(false); + expect(fs.readdirSync(cacheRoot).length).toEqual(0); + }); + + it('should use cache hits when cacheMode=ReadOnly', async () => { + const driverPath = await downloadArtifact({ + cacheRoot, + downloader, + version: '2.0.9', + artifactName: 'chromedriver', + platform: 'darwin', + arch: 'x64', + }); + await fs.writeFile(driverPath, 'cached content'); + const driverPath2 = await downloadArtifact({ + cacheRoot, + downloader, + version: '2.0.9', + artifactName: 'chromedriver', + platform: 'darwin', + arch: 'x64', + cacheMode: ElectronDownloadCacheMode.ReadOnly, + }); + expect(driverPath2).not.toEqual(driverPath); + expect(path.dirname(driverPath2).startsWith(cacheRoot)).toEqual(false); + expect(await fs.readFile(driverPath2, 'utf8')).toEqual('cached content'); + }); + describe('sumchecker', () => { beforeEach(() => { jest.clearAllMocks(); diff --git a/test/utils.spec.ts b/test/utils.spec.ts index 8fddc0533..73dd02bba 100644 --- a/test/utils.spec.ts +++ b/test/utils.spec.ts @@ -9,6 +9,7 @@ import { isOfficialLinuxIA32Download, getEnv, setEnv, + TempDirCleanUpMode, } from '../src/utils'; describe('utils', () => { @@ -39,21 +40,19 @@ describe('utils', () => { await withTempDirectory(async dir => { expect(await fs.pathExists(dir)).toEqual(true); expect(await fs.readdir(dir)).toEqual([]); - }); + }, TempDirCleanUpMode.CLEAN); }); it('should return the value the function returns', async () => { - expect(await withTempDirectory(async () => 1234)).toEqual(1234); + expect(await withTempDirectory(async () => 1234, TempDirCleanUpMode.CLEAN)).toEqual(1234); }); it('should delete the directory when the function terminates', async () => { - let mDir: string | undefined = undefined; - await withTempDirectory(async dir => { - mDir = dir; - }); + const mDir = await withTempDirectory(async dir => { + return dir; + }, TempDirCleanUpMode.CLEAN); expect(mDir).not.toBeUndefined(); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - expect(await fs.pathExists(mDir!)).toEqual(false); + expect(await fs.pathExists(mDir)).toEqual(false); }); it('should delete the directory and reject correctly even if the function throws', async () => { @@ -62,12 +61,24 @@ describe('utils', () => { withTempDirectory(async dir => { mDir = dir; throw 'my error'; - }), + }, TempDirCleanUpMode.CLEAN), ).rejects.toEqual('my error'); expect(mDir).not.toBeUndefined(); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion expect(await fs.pathExists(mDir!)).toEqual(false); }); + + it('should not delete the directory if told to orphan the temp dir', async () => { + const mDir = await withTempDirectory(async dir => { + return dir; + }, TempDirCleanUpMode.ORPHAN); + expect(mDir).not.toBeUndefined(); + try { + expect(await fs.pathExists(mDir)).toEqual(true); + } finally { + await fs.remove(mDir); + } + }); }); describe('getHostArch()', () => {