diff --git a/src/registry/garbage-collector.ts b/src/registry/garbage-collector.ts index 79fdafe..d64bbfa 100644 --- a/src/registry/garbage-collector.ts +++ b/src/registry/garbage-collector.ts @@ -2,8 +2,9 @@ // Unreferenced will delete all blobs that are not referenced by any manifest. // Untagged will delete all blobs that are not referenced by any manifest and are not tagged. -import { ServerError } from "../errors"; import { ManifestSchema } from "../manifest"; +import { hexToDigest } from "../user"; +import {symlinkHeader} from "./r2"; export type GarbageCollectionMode = "unreferenced" | "untagged"; export type GCOptions = { @@ -147,7 +148,7 @@ export class GarbageCollector { } private async list(prefix: string, callback: (object: R2Object) => Promise): Promise { - const listed = await this.registry.list({ prefix }); + const listed = await this.registry.list({ prefix: prefix, include: ["customMetadata"] }); for (const object of listed.objects) { if ((await callback(object)) === false) { return false; @@ -182,61 +183,142 @@ export class GarbageCollector { private async collectInner(options: GCOptions): Promise { // We can run out of memory, this should be a bloom filter - let referencedBlobs = new Set(); + const manifestList: { [key: string]: Set } = {}; const mark = await this.getInsertionMark(options.name); + // List manifest from repo to be scanned await this.list(`${options.name}/manifests/`, async (manifestObject) => { - const tag = manifestObject.key.split("/").pop(); - if (!tag || (options.mode === "untagged" && tag.startsWith("sha256:"))) { - return true; + const currentHashFile = hexToDigest(manifestObject.checksums.sha256!); + if (manifestList[currentHashFile] === undefined) { + manifestList[currentHashFile] = new Set(); } - const manifest = await this.registry.get(manifestObject.key); - if (!manifest) { - return true; + manifestList[currentHashFile].add(manifestObject.key); + return true; + }); + + // In untagged mode, search for manifest to delete + if (options.mode === "untagged") { + const manifestToRemove = new Set(); + const referencedManifests = new Set(); + // List tagged manifest to find manifest-list + for (const [_, manifests] of Object.entries(manifestList)) { + const taggedManifest = [...manifests].filter((item) => !item.split("/").pop()?.startsWith("sha256:")); + for (const manifestPath of taggedManifest) { + // Tagged manifest some, load manifest content + const manifest = await this.registry.get(manifestPath); + if (!manifest) { + continue; + } + + const manifestData = (await manifest.json()) as ManifestSchema; + // Search for manifest list + if (manifestData.schemaVersion == 2 && "manifests" in manifestData) { + // Extract referenced manifests from manifest list + manifestData.manifests.forEach((manifest) => { + referencedManifests.add(manifest.digest); + }); + } + } } - const manifestData = (await manifest.json()) as ManifestSchema; - // TODO: garbage collect manifests. - if ("manifests" in manifestData) { - return true; + for (const [key, manifests] of Object.entries(manifestList)) { + if (referencedManifests.has(key)) { + continue; + } + if (![...manifests].some((item) => !item.split("/").pop()?.startsWith("sha256:"))) { + // Add untagged manifest that should be removed + manifests.forEach((manifest) => { + manifestToRemove.add(manifest); + }); + // Manifest to be removed shouldn't be parsed to search for referenced layers + delete manifestList[key]; + } + } + + // Deleting untagged manifest + if (manifestToRemove.size > 0) { + if (!(await this.checkIfGCCanContinue(options.name, mark))) { + throw new Error("there is a manifest insertion going, the garbage collection shall stop"); + } + + // GC will deleted untagged manifest + await this.registry.delete(manifestToRemove.values().toArray()); + } + } + + const referencedBlobs = new Set(); + // From manifest, extract referenced layers + for (const [_, manifests] of Object.entries(manifestList)) { + // Select only one manifest per unique manifest + const manifestPath = manifests.values().next().value; + if (manifestPath === undefined) { + continue; } + const manifest = await this.registry.get(manifestPath); + // Skip if manifest not found + if (!manifest) continue; + + const manifestData = (await manifest.json()) as ManifestSchema; if (manifestData.schemaVersion === 1) { manifestData.fsLayers.forEach((layer) => { referencedBlobs.add(layer.blobSum); }); } else { + // Skip manifest-list, they don't contain any layers references + if ("manifests" in manifestData) continue; + // Add referenced layers from current manifest manifestData.layers.forEach((layer) => { referencedBlobs.add(layer.digest); }); + // Add referenced config blob from current manifest + referencedBlobs.add(manifestData.config.digest); } + } + const unreferencedBlobs = new Set(); + // List blobs to be removed + await this.list(`${options.name}/blobs/`, async (object) => { + const blobHash = object.key.split("/").pop(); + if (blobHash && !referencedBlobs.has(blobHash)) { + unreferencedBlobs.add(object.key); + } return true; }); - let unreferencedKeys: string[] = []; - const deleteThreshold = 15; - await this.list(`${options.name}/blobs/`, async (object) => { - const hash = object.key.split("/").pop(); - if (hash && !referencedBlobs.has(hash)) { - unreferencedKeys.push(object.key); - if (unreferencedKeys.length > deleteThreshold) { - if (!(await this.checkIfGCCanContinue(options.name, mark))) { - throw new ServerError("there is a manifest insertion going, the garbage collection shall stop"); + // Check for symlink before removal + if (unreferencedBlobs.size >= 0) { + await this.list("", async (object) => { + const objectPath = object.key; + // Skip non-blobs object and from any other repository (symlink only target cross repository blobs) + if (objectPath.startsWith(`${options.name}/`) || !objectPath.includes("/blobs/sha256:")) { + return true; + } + if (object.customMetadata && object.customMetadata[symlinkHeader] !== undefined) { + // Check if the symlink target the current GC repository + if (object.customMetadata[symlinkHeader] !== options.name) return true; + // Get symlink blob to retrieve its target + const symlinkBlob = await this.registry.get(object.key); + // Skip if symlinkBlob not found + if (!symlinkBlob) return true; + // Get the path of the target blob from the symlink blob + const targetBlobPath = await symlinkBlob.text(); + if (unreferencedBlobs.has(targetBlobPath)) { + // This symlink target a layer that should be removed + unreferencedBlobs.delete(targetBlobPath); } - - await this.registry.delete(unreferencedKeys); - unreferencedKeys = []; } - } - return true; - }); - if (unreferencedKeys.length > 0) { + return unreferencedBlobs.size > 0; + }); + } + + if (unreferencedBlobs.size > 0) { if (!(await this.checkIfGCCanContinue(options.name, mark))) { throw new Error("there is a manifest insertion going, the garbage collection shall stop"); } - await this.registry.delete(unreferencedKeys); + // GC will delete unreferenced blobs + await this.registry.delete(unreferencedBlobs.values().toArray()); } return true; diff --git a/src/registry/http.ts b/src/registry/http.ts index 70cee55..6eac722 100644 --- a/src/registry/http.ts +++ b/src/registry/http.ts @@ -456,6 +456,14 @@ export class RegistryHTTPClient implements Registry { } } + mountExistingLayer( + _sourceName: string, + _digest: string, + _destinationName: string, + ): Promise { + throw new Error("unimplemented"); + } + putManifest( _namespace: string, _reference: string, diff --git a/src/registry/r2.ts b/src/registry/r2.ts index a024ead..46e921d 100644 --- a/src/registry/r2.ts +++ b/src/registry/r2.ts @@ -101,6 +101,8 @@ export async function encodeState(state: State, env: Env): Promise<{ jwt: string return { jwt: jwtSignature, hash: await getSHA256(jwtSignature, "") }; } +export const symlinkHeader = "X-Serverless-Registry-Symlink"; + export async function getUploadState( name: string, uploadId: string, @@ -150,14 +152,12 @@ export class R2Registry implements Registry { return { response: new ServerError("invalid checksum from R2 backend") }; } - const checkManifestResponse = { + return { exists: true, digest: hexToDigest(res.checksums.sha256!), contentType: res.httpMetadata!.contentType!, size: res.size, }; - - return checkManifestResponse; } async listRepositories(limit?: number, last?: string): Promise { @@ -377,6 +377,52 @@ export class R2Registry implements Registry { }; } + async mountExistingLayer( + sourceName: string, + digest: string, + destinationName: string, + ): Promise { + const sourceLayerPath = `${sourceName}/blobs/${digest}`; + const [res, err] = await wrap(this.env.REGISTRY.head(sourceLayerPath)); + if (err) { + return wrapError("mountExistingLayer", err); + } + if (!res) { + return wrapError("mountExistingLayer", "Layer not found"); + } else { + const destinationLayerPath = `${destinationName}/blobs/${digest}`; + if (sourceLayerPath === destinationLayerPath) { + // Bad request + throw new InternalError(); + } + // Prevent recursive symlink + if (res.customMetadata && symlinkHeader in res.customMetadata) { + return await this.mountExistingLayer(res.customMetadata[symlinkHeader], digest, destinationName); + } + // Trying to mount a layer from sourceLayerPath to destinationLayerPath + + // Create linked file with custom metadata + const [newFile, error] = await wrap( + this.env.REGISTRY.put(destinationLayerPath, sourceLayerPath, { + sha256: await getSHA256(sourceLayerPath, ""), + httpMetadata: res.httpMetadata, + customMetadata: { [symlinkHeader]: sourceName }, // Storing target repository name in metadata (to easily resolve recursive layer mounting) + }), + ); + if (error) { + return wrapError("mountExistingLayer", error); + } + if (newFile && "response" in newFile) { + return wrapError("mountExistingLayer", newFile.response); + } + + return { + digest: hexToDigest(res.checksums.sha256!), + location: `/v2/${destinationLayerPath}`, + }; + } + } + async layerExists(name: string, tag: string): Promise { const [res, err] = await wrap(this.env.REGISTRY.head(`${name}/blobs/${tag}`)); if (err) { @@ -408,6 +454,19 @@ export class R2Registry implements Registry { }; } + // Handle R2 symlink + if (res.customMetadata && symlinkHeader in res.customMetadata) { + const layerPath = await res.text(); + // Symlink detected! Will download layer from "layerPath" + const [linkName, linkDigest] = layerPath.split("/blobs/"); + if (linkName == name && linkDigest == digest) { + return { + response: new Response(JSON.stringify(BlobUnknownError), { status: 404 }), + }; + } + return await this.env.REGISTRY_CLIENT.getLayer(linkName, linkDigest); + } + return { stream: res.body!, digest: hexToDigest(res.checksums.sha256!), @@ -751,7 +810,6 @@ export class R2Registry implements Registry { } async garbageCollection(namespace: string, mode: GarbageCollectionMode): Promise { - const result = await this.gc.collect({ name: namespace, mode: mode }); - return result; + return await this.gc.collect({ name: namespace, mode: mode }); } } diff --git a/src/registry/registry.ts b/src/registry/registry.ts index 8c684b2..446ef45 100644 --- a/src/registry/registry.ts +++ b/src/registry/registry.ts @@ -114,6 +114,13 @@ export interface Registry { // gets the manifest by namespace + digest getManifest(namespace: string, digest: string): Promise; + // mount an existing layer from a repository to another + mountExistingLayer( + sourceName: string, + digest: string, + destinationName: string, + ): Promise; + // checks that a layer exists layerExists(namespace: string, digest: string): Promise; diff --git a/src/router.ts b/src/router.ts index 9c84840..3c45e16 100644 --- a/src/router.ts +++ b/src/router.ts @@ -42,7 +42,7 @@ v2Router.get("/_catalog", async (req, env: Env) => { }), { headers: { - Link: `${url.protocol}//${url.hostname}${url.pathname}?n=${n ?? 1000}&last=${response.cursor ?? ""}; rel=next`, + "Link": `${url.protocol}//${url.hostname}${url.pathname}?n=${n ?? 1000}&last=${response.cursor ?? ""}; rel=next`, "Content-Type": "application/json", }, }, @@ -331,6 +331,25 @@ v2Router.delete("/:name+/blobs/uploads/:id", async (req, env: Env) => { // this is the first thing that the client asks for in an upload v2Router.post("/:name+/blobs/uploads/", async (req, env: Env) => { const { name } = req.params; + const { from, mount } = req.query; + if (mount !== undefined && from !== undefined) { + // Try to create a new upload from an existing layer on another repository + const [finishedUploadObject, err] = await wrap( + env.REGISTRY_CLIENT.mountExistingLayer(from.toString(), mount.toString(), name), + ); + // If there is an error, fallback to the default layer upload system + if (!(err || (finishedUploadObject && "response" in finishedUploadObject))) { + return new Response(null, { + status: 201, + headers: { + "Content-Length": "0", + "Location": finishedUploadObject.location, + "Docker-Content-Digest": finishedUploadObject.digest, + }, + }); + } + } + // Upload a new layer const [uploadObject, err] = await wrap(env.REGISTRY_CLIENT.startUpload(name)); if (err) { diff --git a/test/index.test.ts b/test/index.test.ts index b2cf31c..85302df 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -13,33 +13,80 @@ import worker from "../index"; import { createExecutionContext, env, waitOnExecutionContext } from "cloudflare:test"; async function generateManifest(name: string, schemaVersion: 1 | 2 = 2): Promise { - const data = "bla"; - const sha256 = await getSHA256(data); + // Layer data + const layerData = Math.random().toString(36).substring(2); // Random string data + const layerSha256 = await getSHA256(layerData); + // Upload layer data const res = await fetch(createRequest("POST", `/v2/${name}/blobs/uploads/`, null, {})); expect(res.ok).toBeTruthy(); - const blob = new Blob([data]).stream(); - const stream = limit(blob, data.length); + const blob = new Blob([layerData]).stream(); + const stream = limit(blob, layerData.length); const res2 = await fetch(createRequest("PATCH", res.headers.get("location")!, stream, {})); expect(res2.ok).toBeTruthy(); - const last = await fetch(createRequest("PUT", res2.headers.get("location")! + "&digest=" + sha256, null, {})); + const last = await fetch(createRequest("PUT", res2.headers.get("location")! + "&digest=" + layerSha256, null, {})); expect(last.ok).toBeTruthy(); + // Config data + const configData = Math.random().toString(36).substring(2); // Random string data + const configSha256 = await getSHA256(configData); + if (schemaVersion === 2) { + // Upload config layer + const configRes = await fetch(createRequest("POST", `/v2/${name}/blobs/uploads/`, null, {})); + expect(configRes.ok).toBeTruthy(); + const configBlob = new Blob([configData]).stream(); + const configStream = limit(configBlob, configData.length); + const configRes2 = await fetch(createRequest("PATCH", configRes.headers.get("location")!, configStream, {})); + expect(configRes2.ok).toBeTruthy(); + const configLast = await fetch( + createRequest("PUT", configRes2.headers.get("location")! + "&digest=" + configSha256, null, {}), + ); + expect(configLast.ok).toBeTruthy(); + } return schemaVersion === 1 ? { schemaVersion, - fsLayers: [{ blobSum: sha256 }], + fsLayers: [{ blobSum: layerSha256 }], architecture: "amd64", } : { schemaVersion, layers: [ - { size: data.length, digest: sha256, mediaType: "shouldbeanything" }, - { size: data.length, digest: sha256, mediaType: "shouldbeanything" }, + { size: layerData.length, digest: layerSha256, mediaType: "shouldbeanything" }, + { size: layerData.length, digest: layerSha256, mediaType: "shouldbeanything" }, ], - config: { size: data.length, digest: sha256, mediaType: "configmediatypeshouldntbechecked" }, + config: { size: configData.length, digest: configSha256, mediaType: "configmediatypeshouldntbechecked" }, mediaType: "shouldalsobeanythingforretrocompatibility", }; } +async function generateManifestList(amdManifest: ManifestSchema, armManifest: ManifestSchema): Promise { + const amdManifestData = JSON.stringify(amdManifest); + const armManifestData = JSON.stringify(armManifest); + return { + schemaVersion: 2, + mediaType: "application/vnd.docker.distribution.manifest.list.v2+json", + manifests: [ + { + mediaType: "application/vnd.docker.distribution.manifest.v2+json", + size: amdManifestData.length, + digest: await getSHA256(amdManifestData), + platform: { + architecture: "amd64", + os: "linux", + }, + }, + { + mediaType: "application/vnd.docker.distribution.manifest.v2+json", + size: armManifestData.length, + digest: await getSHA256(armManifestData), + platform: { + architecture: "arm64", + os: "linux", + }, + }, + ], + }; +} + function createRequest(method: string, path: string, body: ReadableStream | null, headers = {}) { return new Request(new URL("https://registry.com" + path), { method, body: body, headers }); } @@ -154,6 +201,39 @@ async function createManifest(name: string, schema: ManifestSchema, tag?: string return { sha256 }; } +function getLayersFromManifest(schema: ManifestSchema): string[] { + const layersDigest = []; + if (schema.schemaVersion === 1) { + for (const layer of schema.fsLayers) { + layersDigest.push(layer.blobSum); + } + } else if (schema.schemaVersion === 2 && !("manifests" in schema)) { + layersDigest.push(schema.config.digest); + for (const layer of schema.layers) { + layersDigest.push(layer.digest); + } + } + return layersDigest; +} + +async function mountLayersFromManifest(from: string, schema: ManifestSchema, name: string): Promise { + const layersDigest = getLayersFromManifest(schema); + + for (const layerDigest of layersDigest) { + const res = await fetch( + createRequest("POST", `/v2/${name}/blobs/uploads/?from=${from}&mount=${layerDigest}`, null, {}), + ); + if (!res.ok) { + throw new Error(await res.text()); + } + expect(res.ok).toBeTruthy(); + expect(res.status).toEqual(201); + expect(res.headers.get("docker-content-digest")).toEqual(layerDigest); + } + + return layersDigest.length; +} + describe("v2 manifests", () => { test("HEAD /v2/:name/manifests/:reference NOT FOUND", async () => { const response = await fetch(createRequest("GET", "/v2/notfound/manifests/reference", null)); @@ -198,7 +278,7 @@ describe("v2 manifests", () => { { const listObjects = await bindings.REGISTRY.list({ prefix: "hello-world/blobs/" }); - expect(listObjects.objects.length).toEqual(1); + expect(listObjects.objects.length).toEqual(2); const gcRes = await fetch(new Request("http://registry.com/v2/hello-world/gc", { method: "POST" })); if (!gcRes.ok) { @@ -206,7 +286,7 @@ describe("v2 manifests", () => { } const listObjectsAfterGC = await bindings.REGISTRY.list({ prefix: "hello-world/blobs/" }); - expect(listObjectsAfterGC.objects.length).toEqual(1); + expect(listObjectsAfterGC.objects.length).toEqual(2); } expect(await bindings.REGISTRY.head(`hello-world/manifests/hello`)).toBeTruthy(); @@ -216,7 +296,7 @@ describe("v2 manifests", () => { expect(await bindings.REGISTRY.head(`hello-world/manifests/hello`)).toBeNull(); const listObjects = await bindings.REGISTRY.list({ prefix: "hello-world/blobs/" }); - expect(listObjects.objects.length).toEqual(1); + expect(listObjects.objects.length).toEqual(2); const listObjectsManifests = await bindings.REGISTRY.list({ prefix: "hello-world/manifests/" }); expect(listObjectsManifests.objects.length).toEqual(0); @@ -241,30 +321,93 @@ describe("v2 manifests", () => { }); test("PUT then list tags with GET /v2/:name/tags/list", async () => { + const manifestList = new Set(); const { sha256 } = await createManifest("hello-world-list", await generateManifest("hello-world-list"), `hello`); + manifestList.add(sha256); const expectedRes = ["hello"]; - for (let i = 0; i < 50; i++) { + for (let i = 0; i < 40; i++) { expectedRes.push(`hello-${i}`); } expectedRes.sort(); const shuffledRes = shuffleArray([...expectedRes]); for (const tag of shuffledRes) { - await createManifest("hello-world-list", await generateManifest("hello-world-list"), tag); + const { sha256 } = await createManifest("hello-world-list", await generateManifest("hello-world-list"), tag); + manifestList.add(sha256); } const tagsRes = await fetch(createRequest("GET", `/v2/hello-world-list/tags/list?n=1000`, null)); const tags = (await tagsRes.json()) as TagsList; expect(tags.name).toEqual("hello-world-list"); expect(tags.tags).toEqual(expectedRes); - expect(tags.tags).not.contain(sha256) - const res = await fetch(createRequest("DELETE", `/v2/hello-world-list/manifests/${sha256}`, null)); - expect(res.ok).toBeTruthy(); + for (const manifestSha256 of manifestList) { + const res = await fetch(createRequest("DELETE", `/v2/hello-world-list/manifests/${manifestSha256}`, null)); + expect(res.ok).toBeTruthy(); + } const tagsResEmpty = await fetch(createRequest("GET", `/v2/hello-world-list/tags/list`, null)); const tagsEmpty = (await tagsResEmpty.json()) as TagsList; expect(tagsEmpty.tags).toHaveLength(0); }); + + test("Upload manifests with recursive layer mounting", async () => { + const repoA = "app-a"; + const repoB = "app-b"; + const repoC = "app-c"; + + // Generate manifest + const appManifest = await generateManifest(repoA); + // Create architecture specific repository + await createManifest(repoA, appManifest, `latest`); + + // Upload app from repoA to repoB + await mountLayersFromManifest(repoA, appManifest, repoB); + await createManifest(repoB, appManifest, `latest`); + + // Upload app from repoB to repoC + await mountLayersFromManifest(repoB, appManifest, repoC); + await createManifest(repoC, appManifest, `latest`); + + const bindings = env as Env; + // Check manifest count + { + const manifestCountA = (await bindings.REGISTRY.list({ prefix: `${repoA}/manifests/` })).objects.length; + const manifestCountB = (await bindings.REGISTRY.list({ prefix: `${repoB}/manifests/` })).objects.length; + const manifestCountC = (await bindings.REGISTRY.list({ prefix: `${repoC}/manifests/` })).objects.length; + expect(manifestCountA).toEqual(manifestCountB); + expect(manifestCountA).toEqual(manifestCountC); + } + // Check blobs count + { + const layersCountA = (await bindings.REGISTRY.list({ prefix: `${repoA}/blobs/` })).objects.length; + const layersCountB = (await bindings.REGISTRY.list({ prefix: `${repoB}/blobs/` })).objects.length; + const layersCountC = (await bindings.REGISTRY.list({ prefix: `${repoC}/blobs/` })).objects.length; + expect(layersCountA).toEqual(layersCountB); + expect(layersCountA).toEqual(layersCountC); + } + // Check symlink direct layer target + for (const layer of getLayersFromManifest(appManifest)) { + const repoLayerB = await bindings.REGISTRY.get(`${repoB}/blobs/${layer}`); + const repoLayerC = await bindings.REGISTRY.get(`${repoC}/blobs/${layer}`); + expect(repoLayerB).not.toBeNull(); + expect(repoLayerC).not.toBeNull(); + if (repoLayerB !== null && repoLayerC !== null) { + // Check if both symlink target the same original blob + expect(await repoLayerB.text()).toEqual(`${repoA}/blobs/${layer}`); + expect(await repoLayerC.text()).toEqual(`${repoA}/blobs/${layer}`); + // Check layer download follow symlink + const layerSource = await fetch(createRequest("GET", `/v2/${repoA}/blobs/${layer}`, null)); + expect(layerSource.ok).toBeTruthy(); + const sourceData = await layerSource.bytes(); + const layerB = await fetch(createRequest("GET", `/v2/${repoB}/blobs/${layer}`, null)); + expect(layerB.ok).toBeTruthy(); + const layerC = await fetch(createRequest("GET", `/v2/${repoC}/blobs/${layer}`, null)); + expect(layerC.ok).toBeTruthy(); + expect(await layerB.bytes()).toEqual(sourceData); + expect(await layerC.bytes()).toEqual(sourceData); + } + } + }); }); describe("tokens", async () => { @@ -516,7 +659,6 @@ describe("push and catalog", () => { "hello-2", "latest", ]); - expect(tags.tags).not.contain("sha256:a8a29b609fa044cf3ee9a79b57a6fbfb59039c3e9c4f38a57ecb76238bf0dec6"); const repositoryBuildUp: string[] = []; let currentPath = "/v2/_catalog?n=1"; @@ -535,6 +677,21 @@ describe("push and catalog", () => { } expect(repositoryBuildUp).toEqual(expectedRepositories); + + // Check blobs count + const bindings = env as Env; + { + const listObjects = await bindings.REGISTRY.list({ prefix: "hello-world-main/blobs/" }); + expect(listObjects.objects.length).toEqual(6); + } + { + const listObjects = await bindings.REGISTRY.list({ prefix: "hello/blobs/" }); + expect(listObjects.objects.length).toEqual(2); + } + { + const listObjects = await bindings.REGISTRY.list({ prefix: "hello/hello/blobs/" }); + expect(listObjects.objects.length).toEqual(2); + } }); test("(v1) push and then use the catalog", async () => { @@ -559,7 +716,6 @@ describe("push and catalog", () => { "hello-2", "latest", ]); - expect(tags.tags).not.contain("sha256:a70525d2dd357c6ece8d9e0a5a232e34ca3bbceaa1584d8929cdbbfc81238210"); const repositoryBuildUp: string[] = []; let currentPath = "/v2/_catalog?n=1"; @@ -578,5 +734,380 @@ describe("push and catalog", () => { } expect(repositoryBuildUp).toEqual(expectedRepositories); + + // Check blobs count + const bindings = env as Env; + { + const listObjects = await bindings.REGISTRY.list({ prefix: "hello-world-main/blobs/" }); + expect(listObjects.objects.length).toEqual(3); + } + { + const listObjects = await bindings.REGISTRY.list({ prefix: "hello/blobs/" }); + expect(listObjects.objects.length).toEqual(1); + } + { + const listObjects = await bindings.REGISTRY.list({ prefix: "hello/hello/blobs/" }); + expect(listObjects.objects.length).toEqual(1); + } + }); +}); + +async function createManifestList(name: string, tag?: string): Promise { + // Generate manifest + const amdManifest = await generateManifest(name); + const armManifest = await generateManifest(name); + const manifestList = await generateManifestList(amdManifest, armManifest); + + if (!tag) { + const manifestListData = JSON.stringify(manifestList); + tag = await getSHA256(manifestListData); + } + const { sha256: amdSha256 } = await createManifest(name, amdManifest); + const { sha256: armSha256 } = await createManifest(name, armManifest); + const { sha256 } = await createManifest(name, manifestList, tag); + return [amdSha256, armSha256, sha256]; +} + +describe("v2 manifest-list", () => { + test("Upload manifest-list", async () => { + const name = "m-arch"; + const tag = "app"; + const manifestsSha256 = await createManifestList(name, tag); + + const bindings = env as Env; + expect(await bindings.REGISTRY.head(`${name}/manifests/${tag}`)).toBeTruthy(); + for (const digest of manifestsSha256) { + expect(await bindings.REGISTRY.head(`${name}/manifests/${digest}`)).toBeTruthy(); + } + + // Delete tag only + const res = await fetch(createRequest("DELETE", `/v2/${name}/manifests/${tag}`, null)); + expect(res.status).toEqual(202); + expect(await bindings.REGISTRY.head(`${name}/manifests/${tag}`)).toBeNull(); + for (const digest of manifestsSha256) { + expect(await bindings.REGISTRY.head(`${name}/manifests/${digest}`)).toBeTruthy(); + } + + // Check blobs count (2 config and 2 layer) + { + const listObjects = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listObjects.objects.length).toEqual(4); + } + + for (const digest of manifestsSha256) { + const res = await fetch(createRequest("DELETE", `/v2/${name}/manifests/${digest}`, null)); + expect(res.status).toEqual(202); + } + for (const digest of manifestsSha256) { + expect(await bindings.REGISTRY.head(`${name}/manifests/${digest}`)).toBeNull(); + } + }); + + test("Upload manifest-list with layer mounting", async () => { + const preprodName = "m-arch-pp"; + const prodName = "m-arch"; + const tag = "app"; + // Generate manifest + const amdManifest = await generateManifest(preprodName); + const armManifest = await generateManifest(preprodName); + // Create architecture specific repository + await createManifest(preprodName, amdManifest, `${tag}-amd`); + await createManifest(preprodName, armManifest, `${tag}-arm`); + + // Create manifest-list on prod repository + const bindings = env as Env; + // Step 1 mount blobs + await mountLayersFromManifest(preprodName, amdManifest, prodName); + await mountLayersFromManifest(preprodName, armManifest, prodName); + // Check blobs count (2 config and 2 layer) + { + const listObjects = await bindings.REGISTRY.list({ prefix: `${preprodName}/blobs/` }); + expect(listObjects.objects.length).toEqual(4); + } + { + const listObjects = await bindings.REGISTRY.list({ prefix: `${prodName}/blobs/` }); + expect(listObjects.objects.length).toEqual(4); + } + + // Step 2 create manifest + const { sha256: amdSha256 } = await createManifest(prodName, amdManifest); + const { sha256: armSha256 } = await createManifest(prodName, armManifest); + + // Step 3 create manifest list + const manifestList = await generateManifestList(amdManifest, armManifest); + const { sha256 } = await createManifest(prodName, manifestList, tag); + + expect(await bindings.REGISTRY.head(`${prodName}/manifests/${tag}`)).toBeTruthy(); + expect(await bindings.REGISTRY.head(`${prodName}/manifests/${sha256}`)).toBeTruthy(); + expect(await bindings.REGISTRY.head(`${prodName}/manifests/${amdSha256}`)).toBeTruthy(); + expect(await bindings.REGISTRY.head(`${prodName}/manifests/${armSha256}`)).toBeTruthy(); + + // Check symlink binding + expect(amdManifest.schemaVersion === 2).toBeTruthy(); + expect("manifests" in amdManifest).toBeFalsy(); + if (amdManifest.schemaVersion === 2 && !("manifests" in amdManifest)) { + const layerDigest = amdManifest.layers[0].digest; + const layerSource = await fetch(createRequest("GET", `/v2/${preprodName}/blobs/${layerDigest}`, null)); + expect(layerSource.ok).toBeTruthy(); + const layerLinked = await fetch(createRequest("GET", `/v2/${prodName}/blobs/${layerDigest}`, null)); + expect(layerLinked.ok).toBeTruthy(); + expect(await layerLinked.text()).toEqual(await layerSource.text()); + } + }); +}); + +async function runGarbageCollector(name: string, mode: "unreferenced" | "untagged" | "both"): Promise { + if (mode === "unreferenced" || mode === "both") { + const gcRes = await fetch(createRequest("POST", `/v2/${name}/gc?mode=unreferenced`, null)); + if (!gcRes.ok) { + throw new Error(`${gcRes.status}: ${await gcRes.text()}`); + } + expect(gcRes.status).toEqual(200); + const response: { success: boolean } = await gcRes.json(); + expect(response.success).toBeTruthy(); + } + if (mode === "untagged" || mode === "both") { + const gcRes = await fetch(createRequest("POST", `/v2/${name}/gc?mode=untagged`, null)); + if (!gcRes.ok) { + throw new Error(`${gcRes.status}: ${await gcRes.text()}`); + } + expect(gcRes.status).toEqual(200); + const response: { success: boolean } = await gcRes.json(); + expect(response.success).toBeTruthy(); + } +} + +describe("garbage collector", () => { + test("Single arch image", async () => { + const name = "hello"; + const manifestOld = await generateManifest(name); + await createManifest(name, manifestOld, "v1"); + const manifestLatest = await generateManifest(name); + await createManifest(name, manifestLatest, "v2"); + await createManifest(name, manifestLatest, "app"); + const bindings = env as Env; + // Check no action needed + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(5); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + await runGarbageCollector(name, "both"); + + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(5); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + // Removing manifest tag - GC untagged mode will clean image + const res = await fetch(createRequest("DELETE", `/v2/${name}/manifests/v1`, null)); + expect(res.status).toEqual(202); + await runGarbageCollector(name, "unreferenced"); + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(4); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + await runGarbageCollector(name, "untagged"); + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(3); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(2); + } + // Add an unreferenced blobs + { + const { sha256: tempSha256 } = await createManifest(name, await generateManifest(name)); + const res = await fetch(createRequest("DELETE", `/v2/${name}/manifests/${tempSha256}`, null)); + expect(res.status).toEqual(202); + } + // Removed manifest - GC unreferenced mode will clean blobs + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(3); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + await runGarbageCollector(name, "unreferenced"); + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(3); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(2); + } + }); + + test("Multi-arch image", async () => { + const name = "hello"; + await createManifestList(name, "app"); + const bindings = env as Env; + // Check no action needed + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(4); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + await runGarbageCollector(name, "both"); + + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(4); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + // Add unreferenced blobs + + { + const manifests = await createManifestList(name, "bis"); + for (const manifest of manifests) { + const res = await fetch(createRequest("DELETE", `/v2/${name}/manifests/${manifest}`, null)); + expect(res.status).toEqual(202); + } + } + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(4); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(8); + } + + await runGarbageCollector(name, "unreferenced"); + + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(4); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + // Add untagged manifest + { + const res = await fetch(createRequest("DELETE", `/v2/${name}/manifests/app`, null)); + expect(res.status).toEqual(202); + } + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(3); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + await runGarbageCollector(name, "unreferenced"); + + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(3); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + await runGarbageCollector(name, "untagged"); + + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${name}/manifests/` }); + expect(listManifests.objects.length).toEqual(0); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${name}/blobs/` }); + expect(listBlobs.objects.length).toEqual(0); + } + }); + + test("Multi-arch image with symlink layers", async () => { + // Deploy multi-repo multi-arch image + const preprodBame = "m-arch-pp"; + const prodName = "m-arch"; + const tag = "app"; + // Generate manifest + const amdManifest = await generateManifest(preprodBame); + const armManifest = await generateManifest(preprodBame); + // Create architecture specific repository + await createManifest(preprodBame, amdManifest, `${tag}-amd`); + await createManifest(preprodBame, armManifest, `${tag}-arm`); + + // Create manifest-list on prod repository + const bindings = env as Env; + // Step 1 mount blobs + await mountLayersFromManifest(preprodBame, amdManifest, prodName); + await mountLayersFromManifest(preprodBame, armManifest, prodName); + + // Step 2 create manifest + await createManifest(prodName, amdManifest); + await createManifest(prodName, armManifest); + + // Step 3 create manifest list + const manifestList = await generateManifestList(amdManifest, armManifest); + await createManifest(prodName, manifestList, tag); + + // Check no action needed + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${preprodBame}/manifests/` }); + expect(listManifests.objects.length).toEqual(4); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${preprodBame}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + await runGarbageCollector(preprodBame, "both"); + + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${preprodBame}/manifests/` }); + expect(listManifests.objects.length).toEqual(4); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${preprodBame}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + // Untagged preprod repo + { + const res = await fetch(createRequest("DELETE", `/v2/${preprodBame}/manifests/${tag}-amd`, null)); + expect(res.status).toEqual(202); + const res2 = await fetch(createRequest("DELETE", `/v2/${preprodBame}/manifests/${tag}-arm`, null)); + expect(res2.status).toEqual(202); + } + await runGarbageCollector(preprodBame, "unreferenced"); + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${preprodBame}/manifests/` }); + expect(listManifests.objects.length).toEqual(2); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${preprodBame}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + await runGarbageCollector(preprodBame, "untagged"); + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${preprodBame}/manifests/` }); + expect(listManifests.objects.length).toEqual(0); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${preprodBame}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + + // Untagged prod repo + { + const res = await fetch(createRequest("DELETE", `/v2/${prodName}/manifests/${tag}`, null)); + expect(res.status).toEqual(202); + } + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${prodName}/manifests/` }); + expect(listManifests.objects.length).toEqual(3); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${prodName}/blobs/` }); + expect(listBlobs.objects.length).toEqual(4); + } + await runGarbageCollector(prodName, "untagged"); + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${prodName}/manifests/` }); + expect(listManifests.objects.length).toEqual(0); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${prodName}/blobs/` }); + expect(listBlobs.objects.length).toEqual(0); + } + await runGarbageCollector(preprodBame, "unreferenced"); + { + const listManifests = await bindings.REGISTRY.list({ prefix: `${prodName}/manifests/` }); + expect(listManifests.objects.length).toEqual(0); + const listBlobs = await bindings.REGISTRY.list({ prefix: `${prodName}/blobs/` }); + expect(listBlobs.objects.length).toEqual(0); + } }); }); diff --git a/tsconfig.base.json b/tsconfig.base.json index 132c4d0..dd8da68 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -5,7 +5,7 @@ "experimentalDecorators": true, "module": "esnext", "moduleResolution": "node", - "types": ["@cloudflare/workers-types", "@cloudflare/vitest-pool-workers"], + "types": ["@cloudflare/workers-types/2023-07-01", "@cloudflare/vitest-pool-workers"], "resolveJsonModule": true, "allowJs": true, "noEmit": true,